newmeasure-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cli/__init__.py +0 -0
- cli/api_client.py +19 -0
- cli/commands/__init__.py +0 -0
- cli/commands/admin.py +51 -0
- cli/commands/artifacts.py +48 -0
- cli/commands/login.py +16 -0
- cli/commands/result.py +46 -0
- cli/commands/run.py +99 -0
- cli/commands/runs.py +44 -0
- cli/commands/tasks.py +36 -0
- cli/commands/upload.py +140 -0
- cli/config.py +29 -0
- cli/main.py +98 -0
- newmeasure_cli-0.1.0.dist-info/METADATA +105 -0
- newmeasure_cli-0.1.0.dist-info/RECORD +17 -0
- newmeasure_cli-0.1.0.dist-info/WHEEL +4 -0
- newmeasure_cli-0.1.0.dist-info/entry_points.txt +2 -0
cli/__init__.py
ADDED
|
File without changes
|
cli/api_client.py
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import httpx
|
|
4
|
+
import typer
|
|
5
|
+
|
|
6
|
+
from cli.config import get
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def client() -> httpx.Client:
|
|
10
|
+
api_url = get("api_url")
|
|
11
|
+
token = get("token")
|
|
12
|
+
if not token:
|
|
13
|
+
typer.echo("Not logged in. Run: newm login")
|
|
14
|
+
raise typer.Exit(1)
|
|
15
|
+
return httpx.Client(
|
|
16
|
+
base_url=api_url,
|
|
17
|
+
headers={"Authorization": f"Bearer {token}"},
|
|
18
|
+
timeout=300,
|
|
19
|
+
)
|
cli/commands/__init__.py
ADDED
|
File without changes
|
cli/commands/admin.py
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Optional
|
|
4
|
+
|
|
5
|
+
import typer
|
|
6
|
+
from rich.console import Console
|
|
7
|
+
|
|
8
|
+
from cli import config
|
|
9
|
+
|
|
10
|
+
console = Console()
|
|
11
|
+
|
|
12
|
+
admin_app = typer.Typer(help="Admin commands")
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@admin_app.command("create-token")
|
|
16
|
+
def create_token(
|
|
17
|
+
customer: str = typer.Option(..., "--customer", help="Customer name"),
|
|
18
|
+
agent_url: str = typer.Option(..., "--agent-url", help="Customer's agent URL"),
|
|
19
|
+
agent_protocol: str = typer.Option(..., "--agent-protocol", help="Protocol (eve, mastra)"),
|
|
20
|
+
agent_auth_user: Optional[str] = typer.Option(None, "--agent-auth-user"),
|
|
21
|
+
agent_auth_pass: Optional[str] = typer.Option(None, "--agent-auth-pass"),
|
|
22
|
+
mcp_token: Optional[str] = typer.Option(None, "--mcp-token", help="Comma-separated MCP tokens for routing"),
|
|
23
|
+
):
|
|
24
|
+
"""Create a new customer API token (requires admin auth)."""
|
|
25
|
+
import httpx
|
|
26
|
+
|
|
27
|
+
cfg = config.load()
|
|
28
|
+
admin_token = typer.prompt("Admin token", hide_input=True)
|
|
29
|
+
|
|
30
|
+
body = {
|
|
31
|
+
"customer_name": customer,
|
|
32
|
+
"agent_url": agent_url,
|
|
33
|
+
"agent_protocol": agent_protocol,
|
|
34
|
+
"agent_auth_user": agent_auth_user or "",
|
|
35
|
+
"agent_auth_pass": agent_auth_pass or "",
|
|
36
|
+
"mcp_token": mcp_token or "",
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
c = httpx.Client(
|
|
40
|
+
base_url=cfg["api_url"],
|
|
41
|
+
headers={"Authorization": f"Bearer {admin_token}"},
|
|
42
|
+
timeout=30,
|
|
43
|
+
)
|
|
44
|
+
resp = c.post("/api/v1/admin/tokens", json=body)
|
|
45
|
+
resp.raise_for_status()
|
|
46
|
+
data = resp.json()
|
|
47
|
+
|
|
48
|
+
console.print(f"Customer: [bold]{data['customer_name']}[/bold]")
|
|
49
|
+
console.print(f"Customer ID: {data['customer_id']}")
|
|
50
|
+
console.print(f"Token: [bold]{data['token']}[/bold]")
|
|
51
|
+
console.print("\nGive this token to the customer. It cannot be retrieved again.")
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import hashlib
|
|
2
|
+
import json
|
|
3
|
+
from pathlib import Path, PurePosixPath
|
|
4
|
+
|
|
5
|
+
import httpx
|
|
6
|
+
import typer
|
|
7
|
+
from rich.console import Console
|
|
8
|
+
|
|
9
|
+
from cli.api_client import client
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def artifacts(run_id, trial_id, output=None):
|
|
13
|
+
base = f"/api/v1/runs/{run_id}/trials/{trial_id}/artifacts"
|
|
14
|
+
with client() as api:
|
|
15
|
+
response = api.get(base)
|
|
16
|
+
response.raise_for_status()
|
|
17
|
+
manifest = response.json()
|
|
18
|
+
Console().print(
|
|
19
|
+
f"Status: {manifest.get('status')} | Stage: {manifest.get('stage')} | Cleanup: {manifest.get('cleanup', {}).get('status')}"
|
|
20
|
+
)
|
|
21
|
+
if output:
|
|
22
|
+
output = Path(output).resolve()
|
|
23
|
+
output.mkdir(parents=True, exist_ok=True)
|
|
24
|
+
(output / "manifest.json").write_text(json.dumps(manifest, indent=2))
|
|
25
|
+
for item in manifest.get("artifacts", []):
|
|
26
|
+
name = item["path"]
|
|
27
|
+
Console().print(f"{name} ({item['size_bytes']} bytes)")
|
|
28
|
+
if not output:
|
|
29
|
+
continue
|
|
30
|
+
relative = PurePosixPath(name)
|
|
31
|
+
destination = (output / name).resolve()
|
|
32
|
+
if (
|
|
33
|
+
relative.is_absolute()
|
|
34
|
+
or ".." in relative.parts
|
|
35
|
+
or not destination.is_relative_to(output)
|
|
36
|
+
):
|
|
37
|
+
raise typer.BadParameter("Unsafe artifact path")
|
|
38
|
+
link = api.get(base + "/download", params={"path": name})
|
|
39
|
+
link.raise_for_status()
|
|
40
|
+
# Separate client: never send the NewMeasure bearer token to S3.
|
|
41
|
+
data = httpx.get(link.json()["url"], timeout=120).raise_for_status().content
|
|
42
|
+
if (
|
|
43
|
+
len(data) != item["size_bytes"]
|
|
44
|
+
or hashlib.sha256(data).hexdigest() != item["sha256"]
|
|
45
|
+
):
|
|
46
|
+
raise typer.BadParameter(f"Artifact checksum mismatch: {name}")
|
|
47
|
+
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
48
|
+
destination.write_bytes(data)
|
cli/commands/login.py
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import typer
|
|
4
|
+
from rich.prompt import Prompt
|
|
5
|
+
|
|
6
|
+
from cli import config
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def login():
|
|
10
|
+
"""Store API token."""
|
|
11
|
+
cfg = config.load()
|
|
12
|
+
|
|
13
|
+
cfg["token"] = Prompt.ask("API Token", password=True)
|
|
14
|
+
|
|
15
|
+
config.save(cfg)
|
|
16
|
+
typer.echo(f"Saved to {config.CONFIG_PATH}")
|
cli/commands/result.py
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import typer
|
|
4
|
+
from rich.console import Console
|
|
5
|
+
from rich.table import Table
|
|
6
|
+
|
|
7
|
+
from cli.api_client import client
|
|
8
|
+
|
|
9
|
+
console = Console()
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def result(run_id: str):
|
|
13
|
+
"""Get detailed results for a run."""
|
|
14
|
+
c = client()
|
|
15
|
+
resp = c.get(f"/api/v1/runs/{run_id}")
|
|
16
|
+
resp.raise_for_status()
|
|
17
|
+
data = resp.json()
|
|
18
|
+
|
|
19
|
+
console.print(f"\nRun: [bold]{data['id']}[/bold]")
|
|
20
|
+
console.print(f"Task: {data['task_id']}")
|
|
21
|
+
console.print(f"Status: {data['status']}")
|
|
22
|
+
console.print(f"Agent: {data.get('agent_url', '-')}")
|
|
23
|
+
|
|
24
|
+
trials = data.get("trials", [])
|
|
25
|
+
if not trials:
|
|
26
|
+
console.print("No trial results yet.")
|
|
27
|
+
return
|
|
28
|
+
|
|
29
|
+
table = Table(title="Trials")
|
|
30
|
+
table.add_column("Trial")
|
|
31
|
+
table.add_column("Status")
|
|
32
|
+
table.add_column("Reward")
|
|
33
|
+
table.add_column("Duration")
|
|
34
|
+
table.add_column("Error")
|
|
35
|
+
|
|
36
|
+
for t in trials:
|
|
37
|
+
reward = f"{t['reward']:.2f}" if t.get("reward") is not None else "-"
|
|
38
|
+
dur = f"{t['duration_sec']:.1f}s" if t.get("duration_sec") else "-"
|
|
39
|
+
err = (t.get("error") or "")[:50]
|
|
40
|
+
table.add_row(t["id"], t["status"], reward, dur, err)
|
|
41
|
+
|
|
42
|
+
console.print(table)
|
|
43
|
+
|
|
44
|
+
avg = data.get("avg_reward")
|
|
45
|
+
if avg is not None:
|
|
46
|
+
console.print(f"\nAvg reward: [bold]{avg:.2f}[/bold]")
|
cli/commands/run.py
ADDED
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import time
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from urllib.parse import urlparse
|
|
6
|
+
|
|
7
|
+
import typer
|
|
8
|
+
from dotenv import dotenv_values
|
|
9
|
+
from rich.console import Console
|
|
10
|
+
|
|
11
|
+
from cli.api_client import client
|
|
12
|
+
from cli.commands.upload import upload
|
|
13
|
+
|
|
14
|
+
console = Console()
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def run(
|
|
18
|
+
target: str,
|
|
19
|
+
n_trials: int = 1,
|
|
20
|
+
credentials_file: Path | None = None,
|
|
21
|
+
wait: bool = True,
|
|
22
|
+
):
|
|
23
|
+
"""Run an eval. TARGET is a task ID (t_...) or a local task directory path."""
|
|
24
|
+
task_id = target
|
|
25
|
+
target_path = Path(target)
|
|
26
|
+
if target_path.is_dir():
|
|
27
|
+
upload(target_path)
|
|
28
|
+
import tomllib
|
|
29
|
+
|
|
30
|
+
toml_text = (target_path / "task.toml").read_text()
|
|
31
|
+
task_id = tomllib.loads(toml_text).get("task_id")
|
|
32
|
+
if not task_id:
|
|
33
|
+
typer.echo("Failed to get task_id after upload")
|
|
34
|
+
raise typer.Exit(1)
|
|
35
|
+
|
|
36
|
+
c = client()
|
|
37
|
+
body = {
|
|
38
|
+
"task_id": task_id,
|
|
39
|
+
"n_trials": n_trials,
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
task_response = c.get(f"/api/v1/tasks/{task_id}")
|
|
43
|
+
task_response.raise_for_status()
|
|
44
|
+
task = task_response.json()
|
|
45
|
+
if task.get("runner") == "browser":
|
|
46
|
+
url = urlparse(str(c.base_url))
|
|
47
|
+
if url.scheme != "https" and url.hostname not in {
|
|
48
|
+
"127.0.0.1",
|
|
49
|
+
"localhost",
|
|
50
|
+
"::1",
|
|
51
|
+
}:
|
|
52
|
+
raise typer.BadParameter(
|
|
53
|
+
"Application credentials require an HTTPS NewMeasure API URL"
|
|
54
|
+
)
|
|
55
|
+
if credentials_file is None or not credentials_file.is_file():
|
|
56
|
+
raise typer.BadParameter(
|
|
57
|
+
"Browser runs require --credentials-file pointing to a local environment file"
|
|
58
|
+
)
|
|
59
|
+
values = dotenv_values(credentials_file, interpolate=False)
|
|
60
|
+
fields = task.get("credential_fields", [])
|
|
61
|
+
if any(not values.get(field) for field in fields):
|
|
62
|
+
raise typer.BadParameter(
|
|
63
|
+
f"Credential file must define: {', '.join(fields)}"
|
|
64
|
+
)
|
|
65
|
+
body["credentials"] = {field: values[field] for field in fields}
|
|
66
|
+
elif credentials_file:
|
|
67
|
+
raise typer.BadParameter(
|
|
68
|
+
"This task does not accept per-run browser credentials"
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
resp = c.post("/api/v1/runs", json=body)
|
|
72
|
+
body.pop("credentials", None)
|
|
73
|
+
resp.raise_for_status()
|
|
74
|
+
run_id = resp.json()["run_id"]
|
|
75
|
+
console.print(f"Run started: [bold]{run_id}[/bold]")
|
|
76
|
+
if not wait:
|
|
77
|
+
return
|
|
78
|
+
|
|
79
|
+
console.print("Waiting for results...")
|
|
80
|
+
while True:
|
|
81
|
+
time.sleep(3)
|
|
82
|
+
resp = c.get(f"/api/v1/runs/{run_id}")
|
|
83
|
+
resp.raise_for_status()
|
|
84
|
+
data = resp.json()
|
|
85
|
+
status = data["status"]
|
|
86
|
+
|
|
87
|
+
if status == "completed":
|
|
88
|
+
n = len(data.get("trials", []))
|
|
89
|
+
avg = data.get("avg_reward")
|
|
90
|
+
if avg is not None:
|
|
91
|
+
console.print(
|
|
92
|
+
f"Done. {n} trial{'s' if n != 1 else ''}, avg reward = {avg:.2f}"
|
|
93
|
+
)
|
|
94
|
+
else:
|
|
95
|
+
console.print(f"Done. {n} trial{'s' if n != 1 else ''}.")
|
|
96
|
+
break
|
|
97
|
+
elif status == "failed":
|
|
98
|
+
console.print("[red]Run failed.[/red]")
|
|
99
|
+
raise typer.Exit(1)
|
cli/commands/runs.py
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from rich.console import Console
|
|
4
|
+
from rich.table import Table
|
|
5
|
+
|
|
6
|
+
from cli.api_client import client
|
|
7
|
+
|
|
8
|
+
console = Console()
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def list_runs(task_id: str | None = None):
|
|
12
|
+
"""List runs, optionally filtered by task ID."""
|
|
13
|
+
c = client()
|
|
14
|
+
url = "/api/v1/runs"
|
|
15
|
+
if task_id:
|
|
16
|
+
url += f"?task_id={task_id}"
|
|
17
|
+
resp = c.get(url)
|
|
18
|
+
resp.raise_for_status()
|
|
19
|
+
runs = resp.json()
|
|
20
|
+
|
|
21
|
+
if not runs:
|
|
22
|
+
console.print("No runs found.")
|
|
23
|
+
return
|
|
24
|
+
|
|
25
|
+
table = Table()
|
|
26
|
+
table.add_column("ID")
|
|
27
|
+
table.add_column("Task")
|
|
28
|
+
table.add_column("Status")
|
|
29
|
+
table.add_column("Trials")
|
|
30
|
+
table.add_column("Avg Reward")
|
|
31
|
+
table.add_column("Created")
|
|
32
|
+
|
|
33
|
+
for r in runs:
|
|
34
|
+
avg = f"{r['avg_reward']:.2f}" if r.get("avg_reward") is not None else "-"
|
|
35
|
+
table.add_row(
|
|
36
|
+
r["id"],
|
|
37
|
+
r["task_id"],
|
|
38
|
+
r["status"],
|
|
39
|
+
str(r["n_trials"]),
|
|
40
|
+
avg,
|
|
41
|
+
str(r.get("created_at", ""))[:19],
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
console.print(table)
|
cli/commands/tasks.py
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from rich.console import Console
|
|
4
|
+
from rich.table import Table
|
|
5
|
+
|
|
6
|
+
from cli.api_client import client
|
|
7
|
+
|
|
8
|
+
console = Console()
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def list_tasks():
|
|
12
|
+
"""List uploaded tasks."""
|
|
13
|
+
c = client()
|
|
14
|
+
resp = c.get("/api/v1/tasks")
|
|
15
|
+
resp.raise_for_status()
|
|
16
|
+
tasks = resp.json()
|
|
17
|
+
|
|
18
|
+
if not tasks:
|
|
19
|
+
console.print("No tasks found.")
|
|
20
|
+
return
|
|
21
|
+
|
|
22
|
+
table = Table()
|
|
23
|
+
table.add_column("ID")
|
|
24
|
+
table.add_column("Name")
|
|
25
|
+
table.add_column("Services")
|
|
26
|
+
table.add_column("Created")
|
|
27
|
+
|
|
28
|
+
for t in tasks:
|
|
29
|
+
table.add_row(
|
|
30
|
+
t["id"],
|
|
31
|
+
t["name"],
|
|
32
|
+
", ".join(t.get("services", [])),
|
|
33
|
+
str(t.get("created_at", ""))[:19],
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
console.print(table)
|
cli/commands/upload.py
ADDED
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import io
|
|
4
|
+
import tomllib
|
|
5
|
+
import zipfile
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
import typer
|
|
9
|
+
from rich.console import Console
|
|
10
|
+
|
|
11
|
+
from cli.api_client import client
|
|
12
|
+
|
|
13
|
+
console = Console()
|
|
14
|
+
|
|
15
|
+
REQUIRED_FILES = ["task.toml", "instruction.md", "seed.json"]
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def upload(task_dir: Path):
|
|
19
|
+
"""Validate and upload a task directory."""
|
|
20
|
+
if not task_dir.is_dir():
|
|
21
|
+
typer.echo(f"Not a directory: {task_dir}")
|
|
22
|
+
raise typer.Exit(1)
|
|
23
|
+
|
|
24
|
+
task_file = task_dir / "task.toml"
|
|
25
|
+
if task_file.exists():
|
|
26
|
+
config = tomllib.loads(task_file.read_text())
|
|
27
|
+
if config.get("metadata", {}).get("newmeasure", {}).get("runner") == "browser":
|
|
28
|
+
return upload_browser(task_dir, config)
|
|
29
|
+
|
|
30
|
+
for name in REQUIRED_FILES:
|
|
31
|
+
if not (task_dir / name).exists():
|
|
32
|
+
typer.echo(f"Missing required file: {name}")
|
|
33
|
+
raise typer.Exit(1)
|
|
34
|
+
|
|
35
|
+
test_sh = task_dir / "tests" / "test.sh"
|
|
36
|
+
if not test_sh.exists():
|
|
37
|
+
typer.echo("Missing required file: tests/test.sh")
|
|
38
|
+
raise typer.Exit(1)
|
|
39
|
+
|
|
40
|
+
console.print("Validating task structure... [green]✓[/green]")
|
|
41
|
+
|
|
42
|
+
task_toml_text = (task_dir / "task.toml").read_text()
|
|
43
|
+
task_id = None
|
|
44
|
+
try:
|
|
45
|
+
config = tomllib.loads(task_toml_text)
|
|
46
|
+
task_id = config.get("task_id")
|
|
47
|
+
except Exception:
|
|
48
|
+
pass
|
|
49
|
+
|
|
50
|
+
c = client()
|
|
51
|
+
|
|
52
|
+
files = {
|
|
53
|
+
"task_toml": ("task.toml", open(task_dir / "task.toml", "rb")),
|
|
54
|
+
"instruction_md": ("instruction.md", open(task_dir / "instruction.md", "rb")),
|
|
55
|
+
"seed_json": ("seed.json", open(task_dir / "seed.json", "rb")),
|
|
56
|
+
"test_sh": ("test.sh", open(test_sh, "rb")),
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
try:
|
|
60
|
+
if task_id:
|
|
61
|
+
resp = c.put(f"/api/v1/tasks/{task_id}", files=files)
|
|
62
|
+
else:
|
|
63
|
+
resp = c.post("/api/v1/tasks", files=files)
|
|
64
|
+
|
|
65
|
+
resp.raise_for_status()
|
|
66
|
+
data = resp.json()
|
|
67
|
+
tid = data.get("task_id", task_id)
|
|
68
|
+
|
|
69
|
+
if not task_id and tid:
|
|
70
|
+
_write_task_id(task_dir / "task.toml", tid)
|
|
71
|
+
|
|
72
|
+
action = "updated" if task_id else "created"
|
|
73
|
+
console.print(f"Task {action}: [bold]{tid}[/bold]")
|
|
74
|
+
finally:
|
|
75
|
+
for _, f in files.values():
|
|
76
|
+
f.close()
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _write_task_id(toml_path: Path, task_id: str) -> None:
|
|
80
|
+
"""Write task_id back into task.toml."""
|
|
81
|
+
text = toml_path.read_text()
|
|
82
|
+
if "task_id" in text:
|
|
83
|
+
lines = text.splitlines(keepends=True)
|
|
84
|
+
new_lines = []
|
|
85
|
+
for line in lines:
|
|
86
|
+
if line.strip().startswith("task_id"):
|
|
87
|
+
new_lines.append(f'task_id = "{task_id}"\n')
|
|
88
|
+
else:
|
|
89
|
+
new_lines.append(line)
|
|
90
|
+
toml_path.write_text("".join(new_lines))
|
|
91
|
+
else:
|
|
92
|
+
lines = text.splitlines(keepends=True)
|
|
93
|
+
inserted = False
|
|
94
|
+
for i, line in enumerate(lines):
|
|
95
|
+
key = line.strip().split("=")[0].strip() if "=" in line else ""
|
|
96
|
+
if key in ("version", "schema_version"):
|
|
97
|
+
lines.insert(i + 1, f'task_id = "{task_id}"\n')
|
|
98
|
+
inserted = True
|
|
99
|
+
break
|
|
100
|
+
if not inserted:
|
|
101
|
+
lines.insert(0, f'task_id = "{task_id}"\n')
|
|
102
|
+
toml_path.write_text("".join(lines))
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def upload_browser(task_dir: Path, config: dict):
|
|
106
|
+
names = [
|
|
107
|
+
"task.toml",
|
|
108
|
+
"instruction.md",
|
|
109
|
+
"scenario.yaml",
|
|
110
|
+
"product.md",
|
|
111
|
+
"environment/Dockerfile",
|
|
112
|
+
]
|
|
113
|
+
if not (task_dir / "instruction.md").is_file():
|
|
114
|
+
raise typer.BadParameter("instruction.md is required")
|
|
115
|
+
data = io.BytesIO()
|
|
116
|
+
with zipfile.ZipFile(data, "w", zipfile.ZIP_DEFLATED) as archive:
|
|
117
|
+
for name in names:
|
|
118
|
+
path = task_dir / name
|
|
119
|
+
if path.is_file():
|
|
120
|
+
if path.is_symlink():
|
|
121
|
+
raise typer.BadParameter("Task files must not be symlinks")
|
|
122
|
+
archive.writestr(name, path.read_bytes())
|
|
123
|
+
if not (task_dir / "product.md").exists():
|
|
124
|
+
# Shared task context is packaged by the CLI, never resolved on the server.
|
|
125
|
+
shared = task_dir.parent.parent / "shared" / "product.md"
|
|
126
|
+
if shared.is_file() and not shared.is_symlink():
|
|
127
|
+
archive.writestr("product.md", shared.read_bytes())
|
|
128
|
+
task_id = config.get("task_id")
|
|
129
|
+
with client() as c:
|
|
130
|
+
url = f"/api/v1/task-bundles/{task_id}" if task_id else "/api/v1/task-bundles"
|
|
131
|
+
response = c.request(
|
|
132
|
+
"PUT" if task_id else "POST",
|
|
133
|
+
url,
|
|
134
|
+
files={"bundle": ("task.zip", data.getvalue(), "application/zip")},
|
|
135
|
+
)
|
|
136
|
+
response.raise_for_status()
|
|
137
|
+
result = response.json()
|
|
138
|
+
if not task_id:
|
|
139
|
+
_write_task_id(task_dir / "task.toml", result["task_id"])
|
|
140
|
+
console.print(f"Task uploaded: [bold]{result['task_id']}[/bold]")
|
cli/config.py
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
CONFIG_DIR = Path.home() / ".newm"
|
|
7
|
+
CONFIG_PATH = CONFIG_DIR / "config.json"
|
|
8
|
+
|
|
9
|
+
DEFAULTS = {
|
|
10
|
+
"api_url": "https://api.newmeasure.cc",
|
|
11
|
+
"token": "",
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def load() -> dict:
|
|
16
|
+
if CONFIG_PATH.exists():
|
|
17
|
+
data = json.loads(CONFIG_PATH.read_text())
|
|
18
|
+
return {**DEFAULTS, **data}
|
|
19
|
+
return dict(DEFAULTS)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def save(config: dict) -> None:
|
|
23
|
+
CONFIG_DIR.mkdir(parents=True, exist_ok=True)
|
|
24
|
+
CONFIG_PATH.write_text(json.dumps(config, indent=2) + "\n")
|
|
25
|
+
CONFIG_PATH.chmod(0o600)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def get(key: str) -> str:
|
|
29
|
+
return load().get(key, "")
|
cli/main.py
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import subprocess
|
|
4
|
+
import sys
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Optional
|
|
7
|
+
|
|
8
|
+
import typer
|
|
9
|
+
|
|
10
|
+
from cli.commands.admin import admin_app
|
|
11
|
+
from cli.commands.artifacts import artifacts as artifacts_cmd
|
|
12
|
+
from cli.commands.login import login as login_cmd
|
|
13
|
+
from cli.commands.result import result as result_cmd
|
|
14
|
+
from cli.commands.run import run as run_cmd
|
|
15
|
+
from cli.commands.runs import list_runs
|
|
16
|
+
from cli.commands.tasks import list_tasks
|
|
17
|
+
from cli.commands.upload import upload as upload_cmd
|
|
18
|
+
|
|
19
|
+
app = typer.Typer(
|
|
20
|
+
name="newm",
|
|
21
|
+
help="NewMeasure eval CLI",
|
|
22
|
+
no_args_is_help=True,
|
|
23
|
+
)
|
|
24
|
+
app.add_typer(admin_app, name="admin")
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@app.command()
|
|
28
|
+
def login():
|
|
29
|
+
"""Store API token."""
|
|
30
|
+
login_cmd()
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@app.command()
|
|
34
|
+
def upload(task_dir: Path = typer.Argument(..., help="Path to task directory")):
|
|
35
|
+
"""Validate and upload a task."""
|
|
36
|
+
upload_cmd(task_dir)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@app.command()
|
|
40
|
+
def run(
|
|
41
|
+
target: str = typer.Argument(..., help="Task ID (t_...) or local task directory"),
|
|
42
|
+
n_trials: int = typer.Option(1, "--n-trials", help="Number of trials"),
|
|
43
|
+
credentials_file: Optional[Path] = typer.Option(
|
|
44
|
+
None,
|
|
45
|
+
"--credentials-file",
|
|
46
|
+
help="Per-run application credentials (not uploaded as a file)",
|
|
47
|
+
),
|
|
48
|
+
wait: bool = typer.Option(
|
|
49
|
+
True, "--wait/--no-wait", help="Wait for results after submission"
|
|
50
|
+
),
|
|
51
|
+
):
|
|
52
|
+
"""Run an eval against your agent."""
|
|
53
|
+
run_cmd(
|
|
54
|
+
target=target, n_trials=n_trials, credentials_file=credentials_file, wait=wait
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
@app.command()
|
|
59
|
+
def artifacts(
|
|
60
|
+
run_id: str,
|
|
61
|
+
trial_id: str,
|
|
62
|
+
output: Optional[Path] = typer.Option(
|
|
63
|
+
None, "--output", help="Download artifacts to this directory"
|
|
64
|
+
),
|
|
65
|
+
):
|
|
66
|
+
"""List or download a trial's stored artifacts."""
|
|
67
|
+
artifacts_cmd(run_id, trial_id, output)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
@app.command()
|
|
71
|
+
def tasks():
|
|
72
|
+
"""List uploaded tasks."""
|
|
73
|
+
list_tasks()
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
@app.command()
|
|
77
|
+
def runs(task_id: Optional[str] = typer.Argument(None, help="Filter by task ID")):
|
|
78
|
+
"""List runs."""
|
|
79
|
+
list_runs(task_id)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
@app.command()
|
|
83
|
+
def result(run_id: str = typer.Argument(..., help="Run ID")):
|
|
84
|
+
"""Get detailed run results."""
|
|
85
|
+
result_cmd(run_id)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
@app.command(
|
|
89
|
+
context_settings={"allow_extra_args": True, "ignore_unknown_options": True},
|
|
90
|
+
)
|
|
91
|
+
def harbor(ctx: typer.Context):
|
|
92
|
+
"""Passthrough to Harbor CLI (power users)."""
|
|
93
|
+
cmd = ["harbor", *ctx.args]
|
|
94
|
+
sys.exit(subprocess.call(cmd))
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
if __name__ == "__main__":
|
|
98
|
+
app()
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: newmeasure-cli
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: NewMeasure eval CLI
|
|
5
|
+
Requires-Python: >=3.11
|
|
6
|
+
Requires-Dist: httpx>=0.28
|
|
7
|
+
Requires-Dist: python-dotenv>=1.0
|
|
8
|
+
Requires-Dist: rich>=13.0
|
|
9
|
+
Requires-Dist: typer>=0.16
|
|
10
|
+
Description-Content-Type: text/markdown
|
|
11
|
+
|
|
12
|
+
# NewMeasure CLI
|
|
13
|
+
|
|
14
|
+
Upload evaluation tasks, start cloud runs, check results, and download artifacts
|
|
15
|
+
from NewMeasure. Requires Python 3.11 or later and a NewMeasure customer API key.
|
|
16
|
+
|
|
17
|
+
## Install
|
|
18
|
+
|
|
19
|
+
```sh
|
|
20
|
+
uv tool install newmeasure-cli
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
Alternatively, use `pipx install newmeasure-cli`, or install with
|
|
24
|
+
`pip install newmeasure-cli` inside a virtual environment. The executable is `newm`.
|
|
25
|
+
|
|
26
|
+
## Sign in
|
|
27
|
+
|
|
28
|
+
```sh
|
|
29
|
+
newm login
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
Enter your customer API key when prompted. The default API is
|
|
33
|
+
`https://api.newmeasure.cc`. Local configuration is stored in
|
|
34
|
+
`~/.newm/config.json` with owner-only permissions.
|
|
35
|
+
|
|
36
|
+
## Run a task
|
|
37
|
+
|
|
38
|
+
```sh
|
|
39
|
+
# Upload and run a local task directory
|
|
40
|
+
newm run ./my-task --no-wait
|
|
41
|
+
|
|
42
|
+
# Or run an already uploaded task
|
|
43
|
+
newm run t_abc123 --no-wait
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
`--no-wait` returns the run ID immediately; execution continues in the cloud.
|
|
47
|
+
Without it, the CLI waits for completion. Check status with:
|
|
48
|
+
|
|
49
|
+
```sh
|
|
50
|
+
newm result <run-id>
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
For browser tasks, provide application credentials in a local environment file
|
|
54
|
+
using the field names declared by the task, for example:
|
|
55
|
+
|
|
56
|
+
```dotenv
|
|
57
|
+
username=YOUR_APPLICATION_EMAIL
|
|
58
|
+
password=YOUR_APPLICATION_PASSWORD
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
Keep that file out of version control. Submit it for the run:
|
|
62
|
+
|
|
63
|
+
```sh
|
|
64
|
+
newm run ./my-task --credentials-file credentials.env --no-wait
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
The CLI sends the declared credential values over HTTPS for that run. Cloud
|
|
68
|
+
provider and model credentials are managed by the NewMeasure service.
|
|
69
|
+
|
|
70
|
+
## Download browser artifacts
|
|
71
|
+
|
|
72
|
+
Use the trial ID printed by `newm result`:
|
|
73
|
+
|
|
74
|
+
```sh
|
|
75
|
+
newm artifacts <run-id> <trial-id> --output ./artifacts/<trial-id>
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
Downloads include the available exports, evidence, and manifest. The CLI verifies
|
|
79
|
+
file sizes and SHA-256 checksums. Browser smoke completion is an operational
|
|
80
|
+
result, not a content-quality score. Inspect the manifest for cleanup status
|
|
81
|
+
before repeating a failed task.
|
|
82
|
+
|
|
83
|
+
## Other commands
|
|
84
|
+
|
|
85
|
+
```sh
|
|
86
|
+
newm upload ./my-task
|
|
87
|
+
newm tasks
|
|
88
|
+
newm runs
|
|
89
|
+
newm runs <task-id>
|
|
90
|
+
newm --help
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
Task IDs persist across uploads; each execution gets a fresh trial ID. Browser
|
|
94
|
+
task bundles include the instruction, task settings, and optional scenario and
|
|
95
|
+
product context. HTTP-agent tasks also require their seed and verifier files.
|
|
96
|
+
|
|
97
|
+
## Development
|
|
98
|
+
|
|
99
|
+
From the CLI source directory:
|
|
100
|
+
|
|
101
|
+
```sh
|
|
102
|
+
uv sync
|
|
103
|
+
uv run newm --help
|
|
104
|
+
uv run pytest
|
|
105
|
+
```
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
+
cli/api_client.py,sha256=aNSBCu915MLBn5g0LGq8usKh3kwPZ2uerGzm9DMAnbA,407
|
|
3
|
+
cli/config.py,sha256=GJHPB2ySO5p3b0PiwTMypQqthsBDxRnez8mcmPZ3tK4,627
|
|
4
|
+
cli/main.py,sha256=wgxIvMOU-s673VUuDkpyvHWmZwV0VByzopHHZ54S8Js,2456
|
|
5
|
+
cli/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
6
|
+
cli/commands/admin.py,sha256=XaCRw_8OGQYSH6d6lxSTcJWn3EHnWnAxJ2RpkQaLOew,1740
|
|
7
|
+
cli/commands/artifacts.py,sha256=UBwYmkpX6dk3Tp25SvuObeRRh-iLq8ZzVYNOryA6TC4,1918
|
|
8
|
+
cli/commands/login.py,sha256=D846YkdqNHSakFaV0hsJBjBuqMnAJCatPKrPjg0my38,300
|
|
9
|
+
cli/commands/result.py,sha256=vWO6gbdRO3ubxWNFT4hybMU3HdFKj8OSTjaqwTrO6_s,1289
|
|
10
|
+
cli/commands/run.py,sha256=zWq13PTsKtiSjff7XdDCx0H589rQQ5QjJt9gXpwhYXY,3115
|
|
11
|
+
cli/commands/runs.py,sha256=CYR1m84liAuqL3sBYN5zi_UfarwreLcSrp15UMJkSpw,1029
|
|
12
|
+
cli/commands/tasks.py,sha256=lL9zdDmaVJHFzJdV9Kowo96LlcT7ROwUBPebspR-6IQ,742
|
|
13
|
+
cli/commands/upload.py,sha256=v9J6ge7iz6cbnLGjBU8CkvyXv9w9nvSLDUIUqa09Wt4,4730
|
|
14
|
+
newmeasure_cli-0.1.0.dist-info/METADATA,sha256=jQwvwbaI5G-k8887SV7Pw9iiQtSIervCUDU8Ro5DRe0,2561
|
|
15
|
+
newmeasure_cli-0.1.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
|
|
16
|
+
newmeasure_cli-0.1.0.dist-info/entry_points.txt,sha256=9cErJMpXIUirw7EG3DWPYXqbeUfOK1eEIA_2HTyM-Tw,38
|
|
17
|
+
newmeasure_cli-0.1.0.dist-info/RECORD,,
|