sf-m3-cli 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of sf-m3-cli might be problematic. Click here for more details.
- m3_cli/__init__.py +5 -0
- m3_cli/__main__.py +5 -0
- m3_cli/branding.py +10 -0
- m3_cli/control_plane.py +278 -0
- m3_cli/doctor.py +376 -0
- m3_cli/errors.py +10 -0
- m3_cli/init.py +236 -0
- m3_cli/main.py +271 -0
- m3_cli/runtime.py +123 -0
- m3_cli/setup.py +353 -0
- m3_cli/supervisor.py +1336 -0
- m3_cli/ui/assets/geist-mono-latin-wght-normal-XN7g48iV.woff2 +0 -0
- m3_cli/ui/assets/index-BB1JIVWj.css +1 -0
- m3_cli/ui/assets/index-DRJt176-.js +95 -0
- m3_cli/ui/assets/inter-latin-standard-normal-BwkfbSeq.woff2 +0 -0
- m3_cli/ui/index.html +34 -0
- m3_cli/web.py +166 -0
- sf_m3_cli-0.2.0.dist-info/METADATA +395 -0
- sf_m3_cli-0.2.0.dist-info/RECORD +22 -0
- sf_m3_cli-0.2.0.dist-info/WHEEL +4 -0
- sf_m3_cli-0.2.0.dist-info/entry_points.txt +2 -0
- sf_m3_cli-0.2.0.dist-info/licenses/LICENSE +197 -0
|
Binary file
|
m3_cli/ui/index.html
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
<!doctype html>
|
|
2
|
+
<html lang="en">
|
|
3
|
+
<head>
|
|
4
|
+
<meta charset="UTF-8" />
|
|
5
|
+
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
6
|
+
<meta name="color-scheme" content="dark light" />
|
|
7
|
+
<meta name="theme-color" content="#22262d" />
|
|
8
|
+
<meta
|
|
9
|
+
name="description"
|
|
10
|
+
content="MCP Pal is a lightweight MCP development kit for correlated Runs, readable responses, and causal debugging."
|
|
11
|
+
/>
|
|
12
|
+
<title>MCP Pal</title>
|
|
13
|
+
<script type="module" crossorigin src="/assets/index-DRJt176-.js"></script>
|
|
14
|
+
<link rel="stylesheet" crossorigin href="/assets/index-BB1JIVWj.css">
|
|
15
|
+
</head>
|
|
16
|
+
<body>
|
|
17
|
+
<!--
|
|
18
|
+
THESIS: MCP Pal is a Run flight recorder: response and causal timing share one identity; it refuses the all-panels-open inspector.
|
|
19
|
+
OWN-WORLD: graphite instrument surfaces, single-pixel structure, measured rails, compact sans labels, mono only for payloads and timing, cool blue selection with amber/green evidence states.
|
|
20
|
+
STORY: choose tools, submit a draft, read the result, then follow any activity directly to its correlated evidence.
|
|
21
|
+
FIRST VIEWPORT: narrow navigation, offset request tabs, wide center transcript with bottom composer, resizable right waterfall; Send is the primary action and row-to-block navigation is the signature interaction.
|
|
22
|
+
FORM: flight-recorder workbench, grounded direction 5, seed 6e2782a2.
|
|
23
|
+
FINISH: unreviewed and undocumented is unfinished; this build ends with the finish review, the verdict, DESIGN.md, and every shipping raster carrying its provenance
|
|
24
|
+
|
|
25
|
+
PUBLIC THESIS: MCP Pal moves an MCP developer from a readable result to the exact causal evidence; it refuses the generic AI-inspector promise.
|
|
26
|
+
PUBLIC OWN-WORLD: an editorial graphite field with restrained system typography, one-pixel instrument frames, and the correlated Run demo as proof rather than decoration.
|
|
27
|
+
PUBLIC STORY: understand the Run model, interact with response-to-activity correlation, learn the debugging workflow, choose Playground or Tool Debugger, then open evidence or the workbench.
|
|
28
|
+
PUBLIC FIRST VIEWPORT: compact navigation, result-to-root-cause promise, one primary action, and an interactive fixture Run with response and waterfall visible together.
|
|
29
|
+
PUBLIC FORM: user-pinned competitor-informed developer-tool canon, filtered through MCP Pal's Linear/Notion cleanliness and Network/Postman interaction model.
|
|
30
|
+
PUBLIC FINISH: the landing surface is tested, reviewed at desktop and mobile widths, documented as product truth, and shipped with honest fixture and planned-capability labels.
|
|
31
|
+
-->
|
|
32
|
+
<div id="root"></div>
|
|
33
|
+
</body>
|
|
34
|
+
</html>
|
m3_cli/web.py
ADDED
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
"""The bundled local web application used by ``m3 test --ui``."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import os
|
|
7
|
+
import re
|
|
8
|
+
import socket
|
|
9
|
+
import sys
|
|
10
|
+
from importlib import resources
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import NoReturn
|
|
13
|
+
|
|
14
|
+
from fastapi import FastAPI, Request
|
|
15
|
+
from fastapi.responses import FileResponse, JSONResponse
|
|
16
|
+
from fastapi.staticfiles import StaticFiles
|
|
17
|
+
from starlette.responses import Response
|
|
18
|
+
|
|
19
|
+
_AUTH_TOKEN_MAX_BYTES = 256
|
|
20
|
+
_AUTH_TOKEN_PATTERN = re.compile(rb"^[A-Za-z0-9_-]{32,256}$")
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _fail(message: str) -> NoReturn:
|
|
24
|
+
raise ValueError(message)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _safe_startup_error(error: Exception, auth_token: str | None = None) -> str:
|
|
28
|
+
"""Keep startup failures useful without echoing ambient credentials."""
|
|
29
|
+
|
|
30
|
+
message = str(error)[:1000]
|
|
31
|
+
if auth_token:
|
|
32
|
+
message = message.replace(auth_token, "<redacted>")
|
|
33
|
+
for key, value in os.environ.items():
|
|
34
|
+
if (
|
|
35
|
+
value
|
|
36
|
+
and len(value) >= 4
|
|
37
|
+
and any(
|
|
38
|
+
part in key.upper() for part in ("KEY", "TOKEN", "SECRET", "PASSWORD")
|
|
39
|
+
)
|
|
40
|
+
):
|
|
41
|
+
message = message.replace(value, "<redacted>")
|
|
42
|
+
message = re.sub(
|
|
43
|
+
r"(?i)(\b(?:api[_-]?key|token|secret|password|credential)\b\s*(?:=|:)\s*)[^\s,;]+",
|
|
44
|
+
r"\1<redacted>",
|
|
45
|
+
message,
|
|
46
|
+
)
|
|
47
|
+
return f"{type(error).__name__}: {message}" if message else type(error).__name__
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def ui_directory(value: str | os.PathLike[str] | None = None) -> Path:
|
|
51
|
+
"""Return the packaged UI directory, or a test-only directory override."""
|
|
52
|
+
|
|
53
|
+
if value is not None:
|
|
54
|
+
root = Path(value).expanduser()
|
|
55
|
+
else:
|
|
56
|
+
root = Path(str(resources.files("m3_cli").joinpath("ui")))
|
|
57
|
+
index = root / "index.html"
|
|
58
|
+
assets = root / "assets"
|
|
59
|
+
if not index.is_file() or not assets.is_dir():
|
|
60
|
+
_fail("bundled M3 UI assets are unavailable")
|
|
61
|
+
return root
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def create_web_app(
|
|
65
|
+
database: str | os.PathLike[str],
|
|
66
|
+
*,
|
|
67
|
+
auth_token: str,
|
|
68
|
+
ui_dir: str | os.PathLike[str] | None = None,
|
|
69
|
+
) -> FastAPI:
|
|
70
|
+
"""Create the full application and mount the production SPA last."""
|
|
71
|
+
|
|
72
|
+
if not isinstance(auth_token, str) or not re.fullmatch(
|
|
73
|
+
r"[A-Za-z0-9_-]{32,256}", auth_token
|
|
74
|
+
):
|
|
75
|
+
raise ValueError("invalid local UI credentials")
|
|
76
|
+
|
|
77
|
+
from m3_app.api import create_app
|
|
78
|
+
from m3_app.settings import Settings
|
|
79
|
+
|
|
80
|
+
root = ui_directory(ui_dir)
|
|
81
|
+
application = create_app(
|
|
82
|
+
Settings(database_path=str(Path(database).absolute())),
|
|
83
|
+
v2_embedded_worker=True,
|
|
84
|
+
auth_token=auth_token,
|
|
85
|
+
)
|
|
86
|
+
application.mount(
|
|
87
|
+
"/assets", StaticFiles(directory=str(root / "assets")), name="assets"
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
@application.api_route("/{path:path}", methods=["GET", "HEAD"], response_model=None)
|
|
91
|
+
async def spa_fallback(request: Request, path: str) -> Response:
|
|
92
|
+
if path == "api" or path.startswith("api/"):
|
|
93
|
+
return JSONResponse({"detail": "Not Found"}, status_code=404)
|
|
94
|
+
return FileResponse(root / "index.html")
|
|
95
|
+
|
|
96
|
+
return application
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
class _SafeParser(argparse.ArgumentParser):
|
|
100
|
+
def error(self, _message: str) -> NoReturn:
|
|
101
|
+
raise ValueError("invalid web arguments")
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _parser() -> argparse.ArgumentParser:
|
|
105
|
+
parser = _SafeParser(prog="m3_cli.web")
|
|
106
|
+
parser.add_argument("--database-path", required=True)
|
|
107
|
+
parser.add_argument("--port", required=True, type=int)
|
|
108
|
+
return parser
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _read_auth_token(stream: object) -> str:
|
|
112
|
+
"""Read one bounded token line from the supervisor's private stdin pipe."""
|
|
113
|
+
|
|
114
|
+
read = getattr(stream, "readline", None)
|
|
115
|
+
if not callable(read):
|
|
116
|
+
raise ValueError("missing local UI credentials")
|
|
117
|
+
raw = read(_AUTH_TOKEN_MAX_BYTES + 2)
|
|
118
|
+
if not isinstance(raw, bytes):
|
|
119
|
+
raise ValueError("invalid local UI credentials")
|
|
120
|
+
token = raw.rstrip(b"\r\n")
|
|
121
|
+
if (
|
|
122
|
+
not raw.endswith(b"\n")
|
|
123
|
+
or len(raw) > _AUTH_TOKEN_MAX_BYTES + 1
|
|
124
|
+
or not _AUTH_TOKEN_PATTERN.fullmatch(token)
|
|
125
|
+
):
|
|
126
|
+
raise ValueError("invalid local UI credentials")
|
|
127
|
+
if read(1) != b"":
|
|
128
|
+
raise ValueError("invalid local UI credentials")
|
|
129
|
+
return token.decode("ascii")
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def main(argv: list[str] | None = None) -> int:
|
|
133
|
+
auth_token: str | None = None
|
|
134
|
+
try:
|
|
135
|
+
args = _parser().parse_args(argv)
|
|
136
|
+
if not 1 <= args.port <= 65535:
|
|
137
|
+
raise ValueError("port must be between 1 and 65535")
|
|
138
|
+
import uvicorn
|
|
139
|
+
|
|
140
|
+
auth_token = _read_auth_token(sys.stdin.buffer)
|
|
141
|
+
application = create_web_app(args.database_path, auth_token=auth_token)
|
|
142
|
+
|
|
143
|
+
class _ReadinessServer(uvicorn.Server):
|
|
144
|
+
async def startup(self, sockets: list[socket.socket] | None = None) -> None:
|
|
145
|
+
await super().startup(sockets=sockets)
|
|
146
|
+
if self.started:
|
|
147
|
+
sys.stdout.write("\x00M3_UI_SERVER_READY\x00\n")
|
|
148
|
+
sys.stdout.flush()
|
|
149
|
+
|
|
150
|
+
config = uvicorn.Config(
|
|
151
|
+
application, host="127.0.0.1", port=args.port, log_level="warning"
|
|
152
|
+
)
|
|
153
|
+
_ReadinessServer(config).run()
|
|
154
|
+
return 0
|
|
155
|
+
except SystemExit as exc:
|
|
156
|
+
return exc.code if isinstance(exc.code, int) else 2
|
|
157
|
+
except Exception as exc:
|
|
158
|
+
print(
|
|
159
|
+
f"m3 web: unable to start local UI ({_safe_startup_error(exc, auth_token)})",
|
|
160
|
+
file=sys.stderr,
|
|
161
|
+
)
|
|
162
|
+
return 2
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
if __name__ == "__main__":
|
|
166
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,395 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: sf-m3-cli
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Standalone command line interface for M3
|
|
5
|
+
License-Expression: Apache-2.0
|
|
6
|
+
License-File: LICENSE
|
|
7
|
+
Requires-Python: >=3.10
|
|
8
|
+
Requires-Dist: sf-m3-app==0.2.0
|
|
9
|
+
Requires-Dist: sf-m3[storage]==0.2.0
|
|
10
|
+
Description-Content-Type: text/markdown
|
|
11
|
+
|
|
12
|
+
# M3 CLI
|
|
13
|
+
|
|
14
|
+
Use `m3 test --env-file .env` to pass selected variables to pytest.
|
|
15
|
+
`--credential-env` maps agent harness credentials, or judge credentials with
|
|
16
|
+
the `judge:` scope. The judge reads `M3_JUDGE_API_KEY` by default; use
|
|
17
|
+
`LLMJudge(api_key_env=...)` to select another target variable.
|
|
18
|
+
`--judge-max-requests N` caps attempts including retries. Custom endpoints
|
|
19
|
+
require explicit `api_key_env` and
|
|
20
|
+
`response_mode`; loopback `auth="none"` reads no key and sends no
|
|
21
|
+
`Authorization` header. Judges use Chat Completions, so the endpoint and model
|
|
22
|
+
must support the selected response mode.
|
|
23
|
+
|
|
24
|
+
`m3` is a standalone test runner for projects that use the M3 SDK.
|
|
25
|
+
The command is distributed with the production web UI, so users do not need a
|
|
26
|
+
checkout of this repository, Node.js, Vite, or `uv` in the project
|
|
27
|
+
being tested.
|
|
28
|
+
|
|
29
|
+
## Install
|
|
30
|
+
|
|
31
|
+
Install a final release from PyPI:
|
|
32
|
+
|
|
33
|
+
```sh
|
|
34
|
+
uv tool install sf-m3-cli
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
To try a prerelease, select and pin it explicitly:
|
|
38
|
+
|
|
39
|
+
```sh
|
|
40
|
+
uv tool install --prerelease allow "sf-m3-cli==0.2.0a13"
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
The shell installer selects the highest final version, falling back to the highest prerelease until a final exists:
|
|
44
|
+
|
|
45
|
+
```sh
|
|
46
|
+
curl -fsSL https://raw.githubusercontent.com/sineframe/m3/main/scripts/install-latest.sh | sh
|
|
47
|
+
# For an explicit alpha, use: sh -s -- --tag v0.2.0a13
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
The installer prefers `uv tool install`, and otherwise creates a dedicated
|
|
51
|
+
virtual environment. The machine-level CLI and bundled UI stay isolated from
|
|
52
|
+
global Python and every project environment.
|
|
53
|
+
|
|
54
|
+
## Set up a project
|
|
55
|
+
|
|
56
|
+
After installing the CLI, initialize the project and answer its two short
|
|
57
|
+
questions:
|
|
58
|
+
|
|
59
|
+
```sh
|
|
60
|
+
cd my-project
|
|
61
|
+
m3 init
|
|
62
|
+
m3 setup
|
|
63
|
+
m3 doctor
|
|
64
|
+
# Replace the starter TODO with a real assertion and remove its skip.
|
|
65
|
+
m3 test --suite mcp-behavior -- tests/test_m3_starter.py
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
`init` defaults the project name to the repository name and the suite name to
|
|
69
|
+
`mcp-behavior`. It creates `m3.toml` (the stable project identity) and a
|
|
70
|
+
single skipped starter test at `tests/test_m3_starter.py`. Running `init`
|
|
71
|
+
also creates `.env.example` with blank `OPENCODE_API_KEY`, `OPENAI_API_KEY`,
|
|
72
|
+
`ANTHROPIC_API_KEY`, and `M3_JUDGE_API_KEY` entries. Copy it to `.env` if you
|
|
73
|
+
do not already have one; otherwise add only the keys your tests need. Add
|
|
74
|
+
`.env` to `.gitignore` if needed and pass `--env-file .env` to `m3 test`. Running
|
|
75
|
+
`init` again preserves existing files and adds `.env.example` if it is
|
|
76
|
+
missing. The first skipped run confirms collection but exits 1 under `m3 test`
|
|
77
|
+
because no test executed. Replace the starter before using the run as a CI gate.
|
|
78
|
+
|
|
79
|
+
`m3 setup` installs the matching SDK with pytest, storage, and judge support
|
|
80
|
+
into the project environment. It selects `--python`, then an active `VIRTUAL_ENV` or
|
|
81
|
+
`CONDA_PREFIX`, then `.venv`, creating `.venv` when needed. It never installs
|
|
82
|
+
the CLI or bundled app there, never edits dependency manifests or lockfiles,
|
|
83
|
+
and verifies the exact SDK version installed from PyPI. Judge support is
|
|
84
|
+
included. If you recreate or sync the environment, run setup again. Rerunning setup also adds judge support to an older
|
|
85
|
+
project environment.
|
|
86
|
+
|
|
87
|
+
For an isolated pip environment without the `m3` command, install the SDK directly:
|
|
88
|
+
|
|
89
|
+
```sh
|
|
90
|
+
python3 -m venv .venv
|
|
91
|
+
. .venv/bin/activate
|
|
92
|
+
python -m pip install "sf-m3[pytest,judge]"
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
## Commands
|
|
97
|
+
|
|
98
|
+
There are four public commands:
|
|
99
|
+
|
|
100
|
+
```text
|
|
101
|
+
m3 init
|
|
102
|
+
m3 setup [options]
|
|
103
|
+
m3 doctor
|
|
104
|
+
m3 test [options] -- [pytest arguments]
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
There is no `m3 ui` command. The UI is a mode of `m3 test` because it
|
|
108
|
+
shows the runs produced by that test command.
|
|
109
|
+
|
|
110
|
+
### `init`
|
|
111
|
+
|
|
112
|
+
Run `m3 init` from the repository you want to test. It asks for the
|
|
113
|
+
project name, then the suite name, showing a default for each. It creates the
|
|
114
|
+
identity file, skipped pytest starter, and `.env.example` template. If the
|
|
115
|
+
identity and starter already exist, it reports the existing project and adds
|
|
116
|
+
the template only when missing. If one of those two required files is
|
|
117
|
+
missing or the identity is invalid, it reports the partial state for repair.
|
|
118
|
+
|
|
119
|
+
### `doctor`
|
|
120
|
+
|
|
121
|
+
`doctor` reports the standalone CLI and project environment independently. A
|
|
122
|
+
missing project environment is a normal `not ready` result with `m3 setup`
|
|
123
|
+
as remediation; invalid interpreters and invalid configuration are operational
|
|
124
|
+
errors. It also checks explicitly requested capabilities:
|
|
125
|
+
|
|
126
|
+
```sh
|
|
127
|
+
m3 doctor
|
|
128
|
+
m3 doctor --require storage:sqlite
|
|
129
|
+
m3 doctor --python .venv/bin/python --project-root . --json
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
### `test`
|
|
133
|
+
|
|
134
|
+
Agent selection flags:
|
|
135
|
+
|
|
136
|
+
| Flag | Meaning |
|
|
137
|
+
| --- | --- |
|
|
138
|
+
| `--harness KIND[@VERSION]=MODEL[,MODEL...]` | repeatable harness, optional version, and model selection |
|
|
139
|
+
| `--runtime=system\|managed` | use the installed harness (default) or the managed cache |
|
|
140
|
+
| `--harness-cache-dir PATH` | override the per-user harness cache for this run |
|
|
141
|
+
| `--trials N` | independent executions per combination |
|
|
142
|
+
| `--execution-timeout SECONDS` | full deadline for each selected execution; each case has its own deadline |
|
|
143
|
+
| `--env-file PATH` | explicitly load provider variables for the pytest child |
|
|
144
|
+
| `--credential-env [KIND:]TARGET=SOURCE` | map an agent credential; use `judge:TARGET=SOURCE` for the judge |
|
|
145
|
+
| `--suite NAME` or `--suite=NAME` | select tests whose inherited `m3` marker has this suite name |
|
|
146
|
+
|
|
147
|
+
Suite selection happens during pytest collection, before agent expansion. Put
|
|
148
|
+
the same existing marker in each file belonging to one suite:
|
|
149
|
+
|
|
150
|
+
```python
|
|
151
|
+
import pytest
|
|
152
|
+
pytestmark = pytest.mark.m3(suite_name="catalog")
|
|
153
|
+
```
|
|
154
|
+
|
|
155
|
+
Combine `--suite` with `--harness`, `--trials`, paths, `-k`, and `-m`; every
|
|
156
|
+
selector must match. An unknown suite exits 5 and a blank value exits 2. The
|
|
157
|
+
selected run ID is printed and can be passed to `--baseline RUN_ID`.
|
|
158
|
+
|
|
159
|
+
For example, two selections and two trials produce four agent executions:
|
|
160
|
+
|
|
161
|
+
```sh
|
|
162
|
+
m3 test --env-file .env --harness opencode=opencode/big-pickle \
|
|
163
|
+
--harness codex=gpt-5.6-sol --trials 2 -- tests/test_shipping.py
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
Mark a test with `@pytest.mark.m3` and request the `agent` fixture. The
|
|
167
|
+
CLI supplies its harnesses and models. A marker may instead set defaults with
|
|
168
|
+
`@pytest.mark.m3(agents=[...], trials=2)`. CLI `--harness` replaces those
|
|
169
|
+
defaults; CLI `--trials` replaces the marker's trial count. Ordinary tests
|
|
170
|
+
without an `agent` fixture still run once. For an agent test, the item count is
|
|
171
|
+
ordinary pytest cases × selected harness/model choices × trials.
|
|
172
|
+
|
|
173
|
+
### Managed harness runtimes
|
|
174
|
+
|
|
175
|
+
The default `--runtime=system` launches the harness already installed on the
|
|
176
|
+
machine. A version selector requires `--runtime=managed`. With managed mode,
|
|
177
|
+
an unversioned harness requests `latest`; M3 resolves it once for the run and
|
|
178
|
+
shares that pinned version across pytest workers. Explicit versions are
|
|
179
|
+
reused until their verified cache entry is removed.
|
|
180
|
+
|
|
181
|
+
```sh
|
|
182
|
+
m3 test --runtime=managed \
|
|
183
|
+
--harness opencode@1.18.30=opencode/big-pickle \
|
|
184
|
+
--harness opencode@1.18.31=opencode/big-pickle \
|
|
185
|
+
-- tests/test_shipping.py
|
|
186
|
+
```
|
|
187
|
+
|
|
188
|
+
Harness setup starts when the selected test runs. The test waits for the
|
|
189
|
+
requested release to be resolved, downloaded, checked, and installed, or for
|
|
190
|
+
an existing cache entry to be checked. The CLI prints resolution, download,
|
|
191
|
+
verification, and ready states; cache hits are shown as loaded from cache.
|
|
192
|
+
Setup failures fail the affected test. Each execution records the selected
|
|
193
|
+
model, requested harness selector, resolved harness version, target, and
|
|
194
|
+
download digest. Matrix reports distinguish different resolved versions.
|
|
195
|
+
|
|
196
|
+
The default cache root is `~/Library/Caches/m3/harnesses` on macOS,
|
|
197
|
+
`${XDG_CACHE_HOME:-~/.cache}/m3/harnesses` on Linux, and
|
|
198
|
+
`%LOCALAPPDATA%/m3/harnesses` on Windows (falling back to
|
|
199
|
+
`~/AppData/Local/m3/harnesses`). Entries are grouped by harness,
|
|
200
|
+
version, target, and digest. Set `M3_HARNESS_CACHE_DIR`, pass
|
|
201
|
+
`--harness-cache-dir PATH`, or use the SDK's `harness_cache_dir` argument to
|
|
202
|
+
override the root. The SDK constructor takes precedence for SDK calls; the
|
|
203
|
+
CLI flag takes precedence over the environment variable for CLI runs. Keep
|
|
204
|
+
the cache outside the tested repository and system executable directories.
|
|
205
|
+
M3 runs the cached executable by absolute path without a global install or
|
|
206
|
+
PATH change.
|
|
207
|
+
|
|
208
|
+
Inspect or remove cached releases with `m3 runtime cache list` and
|
|
209
|
+
`m3 runtime cache prune`. Pruning waits for active leases for up to 10 seconds
|
|
210
|
+
and returns an operational error if a release remains in use.
|
|
211
|
+
|
|
212
|
+
Known provider variable names are:
|
|
213
|
+
|
|
214
|
+
| Route | Variable name |
|
|
215
|
+
| --- | --- |
|
|
216
|
+
| OpenCode or Pi with `opencode/` models | `OPENCODE_API_KEY` |
|
|
217
|
+
| Codex, OpenCode, or Pi with `openai/` models | `OPENAI_API_KEY` |
|
|
218
|
+
| Claude Code, OpenCode, or Pi with `anthropic/` models | `ANTHROPIC_API_KEY` |
|
|
219
|
+
| Pi with `openai-codex/` models | `PI_CODING_AGENT_DIR` |
|
|
220
|
+
| `m3.judges.LLMJudge` | `M3_JUDGE_API_KEY` by default, or explicit `api_key_env` |
|
|
221
|
+
|
|
222
|
+
For example, put `OPENAI_API_KEY` for an agent using OpenAI and
|
|
223
|
+
`M3_JUDGE_API_KEY` for the judge in the same `.env` file, then run
|
|
224
|
+
`m3 test --env-file .env`. The judge does not fall back to the agent key.
|
|
225
|
+
If the judge key is already named `MY_JUDGE_KEY`, use
|
|
226
|
+
`--credential-env judge:M3_JUDGE_API_KEY=MY_JUDGE_KEY`. The explicit mapping
|
|
227
|
+
overrides `M3_JUDGE_API_KEY` for that test run. A missing source stops the run
|
|
228
|
+
before tests execute.
|
|
229
|
+
|
|
230
|
+
Custom providers use names only, for example
|
|
231
|
+
`--credential-env VENDOR_API_KEY=MY_VENDOR_KEY`; use
|
|
232
|
+
`--credential-env opencode:VENDOR_API_KEY=MY_VENDOR_KEY` to scope a mapping.
|
|
233
|
+
Unscoped mappings apply to harnesses, not judges. Only names appear in flags
|
|
234
|
+
and test code. `.env` is read only when
|
|
235
|
+
`--env-file` is supplied, and ambient variables take precedence. `doctor
|
|
236
|
+
--env-file` checks configuration and does not provide credentials to a later
|
|
237
|
+
test command.
|
|
238
|
+
|
|
239
|
+
The CLI runs pytest using the project Python. It discovers that Python in this
|
|
240
|
+
order:
|
|
241
|
+
|
|
242
|
+
1. `--python PATH`, when supplied;
|
|
243
|
+
2. the `VIRTUAL_ENV` Python;
|
|
244
|
+
3. the `CONDA_PREFIX` Python;
|
|
245
|
+
4. `<project-root>/.venv/bin/python` (or `Scripts/python.exe` on Windows);
|
|
246
|
+
5. `python3`, then `python` on `PATH`.
|
|
247
|
+
|
|
248
|
+
The selected environment must contain `pytest`, the SDK pytest plugin, SQLite
|
|
249
|
+
storage, judge support, and `m3` at exactly the same version as the CLI's SDK.
|
|
250
|
+
Run setup in the project to install these together:
|
|
251
|
+
|
|
252
|
+
```sh
|
|
253
|
+
m3 setup
|
|
254
|
+
```
|
|
255
|
+
|
|
256
|
+
The default results database is `.m3/executions.sqlite` below the project
|
|
257
|
+
root. Override it when needed:
|
|
258
|
+
|
|
259
|
+
```sh
|
|
260
|
+
m3 test --results-db /tmp/my-runs.sqlite -- -q tests
|
|
261
|
+
m3 test --python .venv/bin/python -- -q --maxfail=1
|
|
262
|
+
m3 test --baseline run-123 -- -q tests
|
|
263
|
+
```
|
|
264
|
+
|
|
265
|
+
`--baseline RUN_ID` is the only feedback comparison option. It validates the
|
|
266
|
+
explicit run ID in the selected results database before pytest starts. A run
|
|
267
|
+
through the plugin writes `.m3/reports/<run-id>/feedback.json`; the bundle
|
|
268
|
+
contains the saved test manifest and references to detailed executions and
|
|
269
|
+
catalogs. Existing pytest output, including test prints and logs, remains
|
|
270
|
+
diagnostic output and is not interpreted as a score.
|
|
271
|
+
|
|
272
|
+
Use `--execution-timeout 30` to bound startup, the harness turn, and cleanup for
|
|
273
|
+
each selected execution. A timeout writes a partial trace with the observed
|
|
274
|
+
stage and operation into the execution and trace JSON files under the feedback
|
|
275
|
+
bundle. `handle.result(timeout=...)` in SDK code remains a wait-only timeout.
|
|
276
|
+
Provider credentials still come from the child test environment: use
|
|
277
|
+
`--env-file .env` or ambient variables, and use `--credential-env TARGET=SOURCE`
|
|
278
|
+
when the provider variable has a project-specific name. Values are never put in
|
|
279
|
+
the timeout summary or feedback paths.
|
|
280
|
+
|
|
281
|
+
Everything after `--` is passed to pytest unchanged. The CLI adds its storage
|
|
282
|
+
plugin and `--results-db` option itself. In other words, SQLite
|
|
283
|
+
execution persistence is automatic whenever tests run through `m3 test`;
|
|
284
|
+
tests run directly with pytest use in-memory SDK storage unless they pass an
|
|
285
|
+
explicit `SQLiteExecutionStore` or install the plugin and flag themselves.
|
|
286
|
+
|
|
287
|
+
Profiles saved in the UI are stored in that same results database. A later
|
|
288
|
+
CLI-managed test can reference one directly; no additional CLI option is
|
|
289
|
+
needed. Use the profile ID shown by the UI and explicitly select the server
|
|
290
|
+
inside its MCP document:
|
|
291
|
+
|
|
292
|
+
```python
|
|
293
|
+
from m3 import MCPTestKit
|
|
294
|
+
from m3.types import RevisionSelection, ServerBinding, ServerProfileRef
|
|
295
|
+
|
|
296
|
+
saved_server = ServerBinding(
|
|
297
|
+
profile=ServerProfileRef(
|
|
298
|
+
profile_id="profile-from-ui",
|
|
299
|
+
server_name="orders",
|
|
300
|
+
revision=RevisionSelection(mode="latest"),
|
|
301
|
+
)
|
|
302
|
+
)
|
|
303
|
+
|
|
304
|
+
def test_saved_server_profile():
|
|
305
|
+
with MCPTestKit() as kit, kit.direct(saved_server) as client:
|
|
306
|
+
assert client.list_all_tools()
|
|
307
|
+
```
|
|
308
|
+
|
|
309
|
+
At execution start, `latest` resolves to one immutable revision and its profile
|
|
310
|
+
and revision IDs are retained with the execution. Use a pinned
|
|
311
|
+
`RevisionSelection` when the test must name an exact revision. The same shared
|
|
312
|
+
store resolves `HarnessProfileRef` in agent specifications. This automatic
|
|
313
|
+
store selection applies to `m3 test` (including `--ui`); plain `pytest`
|
|
314
|
+
must be given the same `SQLiteExecutionStore` explicitly.
|
|
315
|
+
|
|
316
|
+
The database records SDK executions, specifications, recorded events and
|
|
317
|
+
traces, sessions/turns, saved artifacts/evidence, evaluations attached to
|
|
318
|
+
those executions, and internal pytest run records. Direct SDK evaluations are saved only with
|
|
319
|
+
`store=SQLiteExecutionStore(path)`; `m3 test` selects the equivalent
|
|
320
|
+
store through `--results-db`. In-memory SDK storage is temporary.
|
|
321
|
+
When the M3 pytest plugin is active, pytest item outcomes, phase
|
|
322
|
+
diagnostics, and execution associations are saved in internal run records.
|
|
323
|
+
M3 matcher checks are saved as evaluation records on their associated
|
|
324
|
+
executions. Ordinary `print()` and logging output remain diagnostic text; they
|
|
325
|
+
are never parsed into a score. Aggregate matrix/trial trends are calculated from
|
|
326
|
+
saved evaluations with the SDK store or the API v2 aggregate route. A
|
|
327
|
+
completed execution is still not by itself a saved test-pass result.
|
|
328
|
+
|
|
329
|
+
Each run also writes an agent-readable JSON bundle to
|
|
330
|
+
`.m3/reports/<run-id>/feedback.json`. Pass `--baseline RUN_ID` to add a
|
|
331
|
+
read-only comparison. The JSON is deterministic for the saved run, and normal
|
|
332
|
+
pytest results remain visible alongside the M3 run ID and feedback path.
|
|
333
|
+
|
|
334
|
+
### Control-plane upload status
|
|
335
|
+
|
|
336
|
+
`m3 test` only writes local results. It never uploads a report, even when
|
|
337
|
+
`M3_CONTROL_PLANE_URL` or `M3_CONTROL_PLANE_TOKEN` is present. There is no
|
|
338
|
+
`m3 upload` command yet. The CLI package contains a report uploader module
|
|
339
|
+
for a future explicit command; it is not registered with command dispatch.
|
|
340
|
+
That module prepares the same public v2 responses as the local app, sends the
|
|
341
|
+
feedback summary and complete current-run execution reports, then publishes
|
|
342
|
+
the run. The transport contract is documented in the control-plane repository.
|
|
343
|
+
|
|
344
|
+
### `--ui`
|
|
345
|
+
|
|
346
|
+
Add `--ui` to keep a local viewer open after pytest finishes:
|
|
347
|
+
|
|
348
|
+
```sh
|
|
349
|
+
m3 test --ui
|
|
350
|
+
```
|
|
351
|
+
|
|
352
|
+
This starts one FastAPI server and one loopback port. The production UI is
|
|
353
|
+
bundled inside the CLI wheel; Node.js, npm, and Vite are not run at runtime.
|
|
354
|
+
The command prints a home link and report links like this:
|
|
355
|
+
|
|
356
|
+
```text
|
|
357
|
+
UI: http://127.0.0.1:8000/#m3_token=<token>
|
|
358
|
+
Run: http://127.0.0.1:8000/reports/runs/<runId>#m3_token=<token>
|
|
359
|
+
```
|
|
360
|
+
|
|
361
|
+
The `<runId>` in the direct link is the pytest run ID stored in the test-run
|
|
362
|
+
manifest and returned by `/api/v2/feedback/<runId>`. The CLI stays open so the
|
|
363
|
+
browser can load results; press Ctrl+C to stop it. A normal pytest failure
|
|
364
|
+
still opens the UI and keeps that pytest exit code. Collection/configuration
|
|
365
|
+
errors, interruption, server startup errors, and invalid configuration return
|
|
366
|
+
an operational failure.
|
|
367
|
+
|
|
368
|
+
Open a link printed by the current CLI process to authorize the browser. The
|
|
369
|
+
browser removes the token fragment from its address bar and keeps the token in
|
|
370
|
+
the current tab's session storage for API requests. A new CLI launch uses a new
|
|
371
|
+
token, so old links stop working. The home link is printed even when the test
|
|
372
|
+
run saves no results. Treat the printed links as credentials while the server
|
|
373
|
+
is running.
|
|
374
|
+
|
|
375
|
+
When stdout is an interactive terminal, the SDK plugin shows a compact test
|
|
376
|
+
progress bar. It is disabled for non-TTY output and for verbose pytest modes,
|
|
377
|
+
where pytest's normal output remains available.
|
|
378
|
+
|
|
379
|
+
The server binds only to `127.0.0.1` and requires the launch token for API
|
|
380
|
+
requests. It is intended for local use; do not expose it through a public
|
|
381
|
+
interface or reverse proxy without suitable network controls.
|
|
382
|
+
|
|
383
|
+
## Troubleshooting
|
|
384
|
+
|
|
385
|
+
- `project Python ... does not match`: install the same `m3` version as
|
|
386
|
+
the CLI, including the `[pytest,storage,judge]` extras, by running `m3 setup`
|
|
387
|
+
in the project.
|
|
388
|
+
- `pytest`, SQLite, or judge support is missing: run `m3 setup` in the project
|
|
389
|
+
to install the matching SDK and extras.
|
|
390
|
+
- `port is already in use`: select another port with `--port 8123`.
|
|
391
|
+
- No UI bundle is available: reinstall the CLI release; development checkouts
|
|
392
|
+
do not contain generated UI files.
|
|
393
|
+
- The UI shows no runs: confirm the test uses `MCPTestKit` and that the CLI
|
|
394
|
+
results database is the same file passed to the API. Runs are written by
|
|
395
|
+
the SDK's default storage plugin; the CLI never writes test results itself.
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
m3_cli/__init__.py,sha256=4pM-eskysxG5ZKNO7uUFnzJEx-bKv5MBiNxfdX9hjiA,92
|
|
2
|
+
m3_cli/__main__.py,sha256=gsvDApbFhDV1dC6Zu2giioJ3vq66xh47CCWaKrl9YFY,101
|
|
3
|
+
m3_cli/branding.py,sha256=w-2k0HMYYagy8W3DAO0IWO6fZkBZdLITonLQ22YVTrw,452
|
|
4
|
+
m3_cli/control_plane.py,sha256=hjdch74SmNSCekILHcunDBBFqDz6GN4y6gl4xBpTKw8,9820
|
|
5
|
+
m3_cli/doctor.py,sha256=smwgBaXYZWhg7drWgx0EoMYFr-VGz3on4XuKDHPfHm8,12901
|
|
6
|
+
m3_cli/errors.py,sha256=8CZf8YkeqPcGN0TVYKi023pFy6h69CHk4CPlDKWoFW0,223
|
|
7
|
+
m3_cli/init.py,sha256=nrM8VZHll6_TjHvzloFritx7WlQ1nx8lM4ric-smeog,8115
|
|
8
|
+
m3_cli/main.py,sha256=B1-ea7ESaNsNRFBXp015HJ-IxDIiAnkH1ab0xeUR6PM,9463
|
|
9
|
+
m3_cli/runtime.py,sha256=PHg062fRp-N5Pj1_6Hj2uwLOX9KYfCtx-5siZB-FwpQ,3914
|
|
10
|
+
m3_cli/setup.py,sha256=rbYhAWdI4JVgMdzPNhmsAnUpS4O3vxtnoKCG_tY3ffI,11752
|
|
11
|
+
m3_cli/supervisor.py,sha256=nWdZwqSOPE5Hww2PXkHyzNU_IisSz24tu7jWwLmALlU,45423
|
|
12
|
+
m3_cli/web.py,sha256=V_AliHWpcF60Utdyky0Mou56kNHmyBg5ugwnNDUcwC8,5379
|
|
13
|
+
m3_cli/ui/index.html,sha256=uqYHYrlKYeYFOpb91aXFwZDlqb7UTnnlcn8vGhFIPP0,2650
|
|
14
|
+
m3_cli/ui/assets/geist-mono-latin-wght-normal-XN7g48iV.woff2,sha256=aErVtTH4HUPB6McDgmLV23zcH2gAbgTWx3ae-o0zyMw,23128
|
|
15
|
+
m3_cli/ui/assets/index-BB1JIVWj.css,sha256=YWoU9AXF3bNwyswPoNRMk6iaEcWWZTLHTrdbJfWufyg,161842
|
|
16
|
+
m3_cli/ui/assets/index-DRJt176-.js,sha256=F7JRcctInqoDaINfuLaBl9sM79UPhFOkg5vFgpbAeO0,802920
|
|
17
|
+
m3_cli/ui/assets/inter-latin-standard-normal-BwkfbSeq.woff2,sha256=LCldmeJtzzV9TQG88nD9aSS2AMmhPdjDY-8RT0xpdvo,72920
|
|
18
|
+
sf_m3_cli-0.2.0.dist-info/METADATA,sha256=Dz3Jaz6Ww5d8NF0XyXIUwkFI63oDQl2hoWvar-4WQRY,17325
|
|
19
|
+
sf_m3_cli-0.2.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
|
|
20
|
+
sf_m3_cli-0.2.0.dist-info/entry_points.txt,sha256=4d_zQx6QCbxXbj86tOhzWvCrgKGau--LXdt0HXSHHeA,40
|
|
21
|
+
sf_m3_cli-0.2.0.dist-info/licenses/LICENSE,sha256=i36d8zJQbRprn-7qlbgO-jLgKe5lyXktrEpv3oqPOGE,11151
|
|
22
|
+
sf_m3_cli-0.2.0.dist-info/RECORD,,
|