newmeasure-cli 0.2.0__tar.gz → 0.3.0__tar.gz

{newmeasure_cli-0.2.0 → newmeasure_cli-0.3.0}/.gitignore

@@ -12,3 +12,7 @@ credentials.env
 tasks/
 build-docs/
 docs/
+TASKS.md
+.claude/
+# Leftover local data of the retired review app (expert reviews, sandbox archives, builds).
+/review/

{newmeasure_cli-0.2.0 → newmeasure_cli-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: newmeasure-cli
-Version: 0.2.0
+Version: 0.3.0
 Summary: NewMeasure eval CLI
 Requires-Python: >=3.11
 Requires-Dist: httpx>=0.28
@@ -16,7 +16,7 @@ Description-Content-Type: text/markdown
 
 *Upload eval tasks, start cloud runs, and download results with `newm`*
 
-[Install](#install) • [Usage](#usage) • [Browser tasks](#browser-tasks) • [Commands](#commands) • [Development](#development)
+[Install](#install) • [Usage](#usage) • [Browser tasks](#browser-tasks) • [Edit tasks](#edit-tasks) • [Commands](#commands) • [Development](#development)
 
 </div>
 
@@ -68,6 +68,31 @@ newm result <run-id>
 > [!NOTE]
 > Task IDs persist across uploads (the first upload writes `task_id` into `task.toml`). Each execution gets a fresh trial ID.
 
+## Preview environments and saved logins
+
+In the app, open **Manage logins** from the Run dialog to create a personal CLI token, then:
+
+```bash
+newm login --evals
+newm logins
+newm run t_abc123 --url https://my-branch.solstice.co --n-trials 3
+newm run tasks/ --url https://my-branch.solstice.co --login "Preview team"
+newm run t_abc123 --prompt-credentials --save-login "My preview"
+# Add --share-login to make the saved login available to your organization.
+newm run t_abc123 --from-run e_previous --login "My preview"
+```
+
+The personal CLI token supplements your customer API key (`newm login`), identifying whose saved
+logins to use. Tokens expire after 90 days and can be revoked from Manage logins. Omit `--login`
+to use the NewMeasure account. Saved logins are personal unless shared; `newm logins` shows who
+saved each one. Passwords are prompted privately and never accepted as command arguments.
+
+Trials are limited to **1–5 per task** in both the app and CLI. Every launch creates a separate
+entry on Runs. `--from-run` links the original and inherits its environment unless `--url` is set;
+it does not reuse the original login. Without either option the task's staging URL is used.
+Solstice previews must use HTTPS on a subdomain of `solstice.co`, without user info, query strings
+or fragments. The override applies only to the prepared run, never to your task files.
+
 ## Browser tasks
 
 Browser tasks test a web product: an AI agent uses it in a cloud browser. Put the application login in a local env file, using the field names the task declares in `credential_fields`:
@@ -95,6 +120,28 @@ Downloads include `outputs/` (the files the product generated, such as PDF and H
 
 A browser task bundle contains `task.toml`, `instruction.md`, and, when present, `scenario.yaml`, `product.md` (or `../../shared/product.md`), and the `environment/` and `tests/` folders.
 
+## Edit tasks
+
+Turn a Solstice edit session into an Edit-suite task in the app. This uses the personal CLI token
+from `newm login --evals`. Pass the session's `operation_id` or its asset URL
+(`https://<tenant>.solstice.co/home/assets/<operation_id>`):
+
+```bash
+# List the operation's turns, with their surface, whether starting HTML is ready, and the request
+newm edit-task 0b7f2a7e-1111-4222-8333-444455556666
+
+# Create a task from turn 2 (a turn number or turn ID)
+newm edit-task 0b7f2a7e-1111-4222-8333-444455556666 --turn 2
+
+# Merge consecutive turns into one task, with your own title
+newm edit-task https://acme.solstice.co/home/assets/0b7f2a7e-1111-4222-8333-444455556666 \
+  -t 2 -t 3 --title "Swap headline and CTA"
+```
+
+The task's checks and title are drafted from the chosen turns; refine them in the app's Edits tab.
+Turns listed as `awaiting` have no starting HTML yet: the task is created, but stays
+`awaiting_html` until the HTML is imported.
+
 ## Commands
 
 ```bash
@@ -105,6 +152,8 @@ newm result <run-id>            # run details and trial results
 newm artifacts <run-id> <trial-id> [--output DIR]
 newm tasks                      # list uploaded tasks
 newm runs [<task-id>]           # list runs
+newm edit-task <operation> [-t <turn> ...]  # list turns or create an edit task
+newm logins                     # list saved logins
 newm harbor ...                 # pass through to the Harbor CLI
 newm --help
 ```

{newmeasure_cli-0.2.0 → newmeasure_cli-0.3.0}/README.md

@@ -5,7 +5,7 @@
 
 *Upload eval tasks, start cloud runs, and download results with `newm`*
 
-[Install](#install) • [Usage](#usage) • [Browser tasks](#browser-tasks) • [Commands](#commands) • [Development](#development)
+[Install](#install) • [Usage](#usage) • [Browser tasks](#browser-tasks) • [Edit tasks](#edit-tasks) • [Commands](#commands) • [Development](#development)
 
 </div>
 
@@ -57,6 +57,31 @@ newm result <run-id>
 > [!NOTE]
 > Task IDs persist across uploads (the first upload writes `task_id` into `task.toml`). Each execution gets a fresh trial ID.
 
+## Preview environments and saved logins
+
+In the app, open **Manage logins** from the Run dialog to create a personal CLI token, then:
+
+```bash
+newm login --evals
+newm logins
+newm run t_abc123 --url https://my-branch.solstice.co --n-trials 3
+newm run tasks/ --url https://my-branch.solstice.co --login "Preview team"
+newm run t_abc123 --prompt-credentials --save-login "My preview"
+# Add --share-login to make the saved login available to your organization.
+newm run t_abc123 --from-run e_previous --login "My preview"
+```
+
+The personal CLI token supplements your customer API key (`newm login`), identifying whose saved
+logins to use. Tokens expire after 90 days and can be revoked from Manage logins. Omit `--login`
+to use the NewMeasure account. Saved logins are personal unless shared; `newm logins` shows who
+saved each one. Passwords are prompted privately and never accepted as command arguments.
+
+Trials are limited to **1–5 per task** in both the app and CLI. Every launch creates a separate
+entry on Runs. `--from-run` links the original and inherits its environment unless `--url` is set;
+it does not reuse the original login. Without either option the task's staging URL is used.
+Solstice previews must use HTTPS on a subdomain of `solstice.co`, without user info, query strings
+or fragments. The override applies only to the prepared run, never to your task files.
+
 ## Browser tasks
 
 Browser tasks test a web product: an AI agent uses it in a cloud browser. Put the application login in a local env file, using the field names the task declares in `credential_fields`:
@@ -84,6 +109,28 @@ Downloads include `outputs/` (the files the product generated, such as PDF and H
 
 A browser task bundle contains `task.toml`, `instruction.md`, and, when present, `scenario.yaml`, `product.md` (or `../../shared/product.md`), and the `environment/` and `tests/` folders.
 
+## Edit tasks
+
+Turn a Solstice edit session into an Edit-suite task in the app. This uses the personal CLI token
+from `newm login --evals`. Pass the session's `operation_id` or its asset URL
+(`https://<tenant>.solstice.co/home/assets/<operation_id>`):
+
+```bash
+# List the operation's turns, with their surface, whether starting HTML is ready, and the request
+newm edit-task 0b7f2a7e-1111-4222-8333-444455556666
+
+# Create a task from turn 2 (a turn number or turn ID)
+newm edit-task 0b7f2a7e-1111-4222-8333-444455556666 --turn 2
+
+# Merge consecutive turns into one task, with your own title
+newm edit-task https://acme.solstice.co/home/assets/0b7f2a7e-1111-4222-8333-444455556666 \
+  -t 2 -t 3 --title "Swap headline and CTA"
+```
+
+The task's checks and title are drafted from the chosen turns; refine them in the app's Edits tab.
+Turns listed as `awaiting` have no starting HTML yet: the task is created, but stays
+`awaiting_html` until the HTML is imported.
+
 ## Commands
 
 ```bash
@@ -94,6 +141,8 @@ newm result <run-id>            # run details and trial results
 newm artifacts <run-id> <trial-id> [--output DIR]
 newm tasks                      # list uploaded tasks
 newm runs [<task-id>]           # list runs
+newm edit-task <operation> [-t <turn> ...]  # list turns or create an edit task
+newm logins                     # list saved logins
 newm harbor ...                 # pass through to the Harbor CLI
 newm --help
 ```

newmeasure_cli-0.3.0/cli/api_client.py

@@ -0,0 +1,38 @@
+from __future__ import annotations
+
+import httpx
+import typer
+
+from cli.config import get
+
+
+def client() -> httpx.Client:
+    api_url = get("api_url")
+    token = get("token")
+    if not token:
+        typer.echo("Not logged in. Run: newm login")
+        raise typer.Exit(1)
+    return httpx.Client(
+        base_url=api_url,
+        headers={"Authorization": f"Bearer {token}"},
+        timeout=300,
+    )
+
+
+def evals_client() -> httpx.Client:
+    token = get("evals_token")
+    if not token:
+        raise typer.BadParameter(
+            "Connect your app account with newm login --evals (create a token at /settings/runs in the app)"
+        )
+    url = get("evals_url")
+    from urllib.parse import urlsplit
+
+    p = urlsplit(url)
+    if p.scheme != "https" and p.hostname not in ("localhost", "127.0.0.1", "::1"):
+        raise typer.BadParameter("CLI access requires HTTPS")
+    return httpx.Client(
+        base_url=url.rstrip("/") + "/",
+        headers={"Authorization": f"Bearer {token}"},
+        timeout=300,
+    )

{newmeasure_cli-0.2.0 → newmeasure_cli-0.3.0}/cli/commands/admin.py

@@ -16,10 +16,14 @@ admin_app = typer.Typer(help="Admin commands")
 def create_token(
     customer: str = typer.Option(..., "--customer", help="Customer name"),
     agent_url: str = typer.Option(..., "--agent-url", help="Customer's agent URL"),
-    agent_protocol: str = typer.Option(..., "--agent-protocol", help="Protocol (eve, mastra)"),
+    agent_protocol: str = typer.Option(
+        ..., "--agent-protocol", help="Protocol (eve, mastra)"
+    ),
     agent_auth_user: Optional[str] = typer.Option(None, "--agent-auth-user"),
     agent_auth_pass: Optional[str] = typer.Option(None, "--agent-auth-pass"),
-    mcp_token: Optional[str] = typer.Option(None, "--mcp-token", help="Comma-separated MCP tokens for routing"),
+    mcp_token: Optional[str] = typer.Option(
+        None, "--mcp-token", help="Comma-separated MCP tokens for routing"
+    ),
 ):
     """Create a new customer API token (requires admin auth)."""
     import httpx

newmeasure_cli-0.3.0/cli/commands/edit_task.py

@@ -0,0 +1,73 @@
+from __future__ import annotations
+
+import json
+
+import typer
+from rich.console import Console
+from rich.table import Table
+
+from cli.api_client import evals_client
+
+console = Console()
+
+
+def _turn_ids(turns: list[dict], picks: list[str]) -> list[str]:
+    """Map 1-based turn numbers to turn IDs; pass turn IDs through."""
+    by_seq = {str(t["seq"]): t["turn_id"] for t in turns}
+    known = {t["turn_id"] for t in turns}
+    out = []
+    for pick in picks:
+        tid = by_seq.get(pick, pick)
+        if tid not in known:
+            raise typer.BadParameter(
+                f"Turn {pick!r} is not in this operation; run without --turn to list turns"
+            )
+        out.append(tid)
+    return out
+
+
+def _show_turns(inspected: dict) -> None:
+    console.print(
+        f"Operation {inspected['operation_id']}  {inspected.get('brand_label') or ''}"
+        f"  {inspected.get('kind') or ''}"
+    )
+    table = Table()
+    for col in ("#", "Turn ID", "Surface", "HTML", "Request"):
+        table.add_column(col)
+    for t in inspected["turns"]:
+        request = " ".join((t.get("user_request") or "").split())
+        table.add_row(
+            str(t["seq"]),
+            t["turn_id"],
+            t.get("surface") or "",
+            ("ready" if t.get("html_ready") else "awaiting")
+            + (" (reverted)" if t.get("reverted") else ""),
+            request[:80] + ("…" if len(request) > 80 else ""),
+        )
+    console.print(table)
+    for task in inspected.get("existing_tasks") or []:
+        console.print(
+            f"Existing task {task['id']}: {task['title']} (turns {', '.join(task['turn_ids'])})"
+        )
+    console.print("Create a task with: newm edit-task <operation> --turn <#>")
+
+
+def edit_task(operation: str, turns: list[str], title: str) -> None:
+    """List an operation's turns, or create an edit task from the chosen turns."""
+    with evals_client() as c:
+        r = c.post("edit-tasks/inspect", json={"operation": operation})
+        r.raise_for_status()
+        inspected = r.json()
+        if not turns:
+            _show_turns(inspected)
+            return
+        r = c.post(
+            "edit-tasks",
+            json={
+                "operation": inspected["operation_id"],
+                "turn_ids": _turn_ids(inspected["turns"], turns),
+                "title": title,
+            },
+        )
+        r.raise_for_status()
+        typer.echo(json.dumps(r.json(), indent=1))

{newmeasure_cli-0.2.0 → newmeasure_cli-0.3.0}/cli/commands/login.py

@@ -6,11 +6,14 @@ from rich.prompt import Prompt
 from cli import config
 
 
-def login():
+def login(evals: bool = False):
     """Store API token."""
     cfg = config.load()
 
-    cfg["token"] = Prompt.ask("API Token", password=True)
+    cfg["evals_token" if evals else "token"] = Prompt.ask(
+        "Personal CLI token from Manage logins in the app" if evals else "API Token",
+        password=True,
+    )
 
     config.save(cfg)
     typer.echo(f"Saved to {config.CONFIG_PATH}")

{newmeasure_cli-0.2.0 → newmeasure_cli-0.3.0}/cli/commands/result.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import typer
 from rich.console import Console
 from rich.table import Table
 
@@ -44,7 +43,9 @@ def result(run_id: str):
         reward = f"{t['reward']:.2f}" if t.get("reward") is not None else "-"
         dur = f"{t['duration_sec']:.1f}s" if t.get("duration_sec") else "-"
         err = (t.get("error") or "")[:50]
-        table.add_row(t["id"], t["status"], reward, dur, t.get("last_stage") or "-", err)
+        table.add_row(
+            t["id"], t["status"], reward, dur, t.get("last_stage") or "-", err
+        )
 
     console.print(table)
 

newmeasure_cli-0.3.0/cli/commands/run.py

@@ -0,0 +1,281 @@
+from __future__ import annotations
+
+import subprocess
+import time
+import tomllib
+from pathlib import Path
+from urllib.parse import urlparse
+
+import typer
+from dotenv import dotenv_values
+from rich.console import Console
+
+from cli.api_client import client, evals_client
+from cli.commands.upload import upload
+from cli.progress import progress_lines
+
+console = Console()
+
+
+def task_dirs(target: Path) -> list[Path]:
+    """The task folders TARGET names: itself, or every task folder in it (or in its tasks/)."""
+    if not target.is_dir():
+        return []
+    if (target / "task.toml").is_file():
+        return [target]
+    for root in (target, target / "tasks"):
+        found = sorted(p.parent for p in root.glob("*/task.toml"))
+        if found:
+            return found
+    raise typer.BadParameter(f"No task folders (with task.toml) in {target}")
+
+
+def _task_id(task_dir: Path) -> str:
+    upload(task_dir)
+    task_id = tomllib.loads((task_dir / "task.toml").read_text()).get("task_id")
+    if not task_id:
+        typer.echo(f"Failed to get task_id after uploading {task_dir}")
+        raise typer.Exit(1)
+    return task_id
+
+
+def _credentials(c, tasks: list[dict], credentials_file: Path | None) -> dict:
+    """The fields the tasks need, from the local file; never sent over remote plain HTTP."""
+    url = urlparse(str(c.base_url))
+    if url.scheme != "https" and url.hostname not in {"127.0.0.1", "localhost", "::1"}:
+        raise typer.BadParameter(
+            "Application credentials require an HTTPS NewMeasure API URL"
+        )
+    if credentials_file is None or not credentials_file.is_file():
+        raise typer.BadParameter(
+            "Browser runs require --credentials-file pointing to a local environment file"
+        )
+    values = dotenv_values(credentials_file, interpolate=False)
+    fields = sorted({f for t in tasks for f in t.get("credential_fields", [])})
+    if any(not values.get(field) for field in fields):
+        raise typer.BadParameter(f"Credential file must define: {', '.join(fields)}")
+    return {field: values[field] for field in fields}
+
+
+def _who(where: Path) -> str:
+    """Who is starting the run, for the evals app: git's user.email where the tasks are."""
+    try:
+        got = subprocess.run(
+            ["git", "-C", str(where), "config", "user.email"],
+            check=False,
+            capture_output=True,
+            text=True,
+            timeout=5,
+        )
+    except (OSError, subprocess.SubprocessError):
+        return ""
+    return got.stdout.strip()[:200]
+
+
+def task_kind(task_dir: Path) -> str:
+    """The task's kind from [metadata.evals] kind in task.toml: "banner", or "email" when unset."""
+    config = tomllib.loads((task_dir / "task.toml").read_text())
+    return config.get("metadata", {}).get("evals", {}).get("kind") or "email"
+
+
+def run(
+    target: str,
+    n_trials: int = 1,
+    credentials_file: Path | None = None,
+    wait: bool = True,
+    kind: str | None = None,
+    url: str | None = None,
+    saved_login: str | None = None,
+    prompt_credentials: bool = False,
+    save_login: str | None = None,
+    share_login: bool = False,
+    parent: str | None = None,
+):
+    """Run an eval. TARGET is a task ID (t_...), a task folder, or a folder of task folders (such
+    as tasks/). Browser tasks run together as one eval run, graded and shown in the evals app.
+    KIND keeps only task folders of that kind (email or banner)."""
+    if not 1 <= n_trials <= 5:
+        raise typer.BadParameter("Trials must be 1 to 5")
+    if sum(bool(x) for x in (credentials_file, saved_login, prompt_credentials)) > 1:
+        raise typer.BadParameter("Choose only one login source")
+    if save_login and not (credentials_file or prompt_credentials):
+        raise typer.BadParameter("Saving a login requires credentials")
+    if share_login and not save_login:
+        raise typer.BadParameter("--share-login requires --save-login")
+    dirs = task_dirs(Path(target))
+    if kind:
+        if not dirs:
+            raise typer.BadParameter(
+                "--kind needs a task folder or a folder of task folders"
+            )
+        dirs = [d for d in dirs if task_kind(d) == kind]
+        if not dirs:
+            raise typer.BadParameter(f"No {kind} tasks in {target}")
+    task_ids = [_task_id(d) for d in dirs] if dirs else [target]
+    c = client()
+    tasks = []
+    for task_id in task_ids:
+        response = c.get(f"/api/v1/tasks/{task_id}")
+        response.raise_for_status()
+        tasks.append(response.json())
+    if not all(t.get("runner") == "browser" for t in tasks):
+        if len(tasks) > 1:
+            raise typer.BadParameter("Only browser tasks can run together")
+        if (
+            credentials_file
+            or url
+            or saved_login
+            or prompt_credentials
+            or save_login
+            or parent
+        ):
+            raise typer.BadParameter(
+                "This task does not accept per-run browser credentials"
+            )
+        return _run_one(c, task_ids[0], n_trials, wait)
+
+    use_app = bool(
+        not credentials_file
+        or saved_login
+        or prompt_credentials
+        or save_login
+        or parent
+    )
+    values = _credentials(c, tasks, credentials_file) if credentials_file else {}
+    if prompt_credentials:
+        from rich.prompt import Prompt
+
+        values = {
+            "username": Prompt.ask("Username"),
+            "password": Prompt.ask("Password", password=True),
+        }
+    body = {"task_ids": task_ids, "n_trials": n_trials, "target_url": url or ""}
+    if use_app:
+        body.update(
+            login=saved_login or ("custom" if values else "default"),
+            **values,
+            save_login=save_login or "",
+            share_login=share_login,
+            parent_eval_id=parent or "",
+        )
+        with evals_client() as ec:
+            resp = ec.post("cli/evals", json=body)
+    else:
+        # Existing scripts using explicit files continue to work without a personal token.
+        body.update(
+            credentials=values,
+            source="cli",
+            created_by=_who(dirs[0] if dirs else Path.cwd()),
+        )
+        resp = c.post("/api/v1/evals", json=body)
+    values.clear()
+    body.clear()
+    resp.raise_for_status()
+    started = resp.json()
+    eval_id, runs = started["eval_id"], started["runs"]
+    plural = "s" if len(runs) != 1 else ""
+    console.print(
+        f"Eval run started: [bold]{eval_id}[/bold] ({len(runs)} task{plural})"
+    )
+    url = started.get("results_url")
+    if not wait:
+        if url:
+            console.print(f"Results: {url}")
+        return
+    if len(runs) == 1:
+        try:
+            _wait_run(c, runs[0]["run_id"])
+        finally:
+            if url:
+                console.print(f"Results: {url}")
+        return
+    _wait_eval(c, eval_id, url)
+
+
+def _run_one(c, task_id: str, n_trials: int, wait: bool) -> None:
+    """One task of a hosted HTTP agent (not graded by the evals app)."""
+    resp = c.post("/api/v1/runs", json={"task_id": task_id, "n_trials": n_trials})
+    resp.raise_for_status()
+    run_id = resp.json()["run_id"]
+    console.print(f"Run started: [bold]{run_id}[/bold]")
+    if wait:
+        _wait_run(c, run_id)
+
+
+def _wait_run(c, run_id: str) -> None:
+    console.print("Waiting for results...")
+    shown = 0
+    while True:
+        time.sleep(3)
+        resp = c.get(f"/api/v1/runs/{run_id}")
+        resp.raise_for_status()
+        data = resp.json()
+        status = data["status"]
+        lines = progress_lines(data.get("progress", []))
+        for line in lines[shown:]:
+            console.print(line)
+        shown = len(lines)
+
+        if status == "completed":
+            lines, all_failed = summarize(data)
+            for line in lines:
+                console.print(line)
+            if all_failed:
+                raise typer.Exit(1)
+            break
+        elif status == "failed":
+            console.print("[red]Run failed.[/red]")
+            raise typer.Exit(1)
+
+
+def _wait_eval(c, eval_id: str, url: str | None) -> None:
+    """Print each task as it finishes, then every task's result and where to see the scores."""
+    console.print("Waiting for results (each task is printed as it finishes)...")
+    done: set[str] = set()
+    while True:
+        time.sleep(10)
+        resp = c.get(f"/api/v1/evals/{eval_id}")
+        resp.raise_for_status()
+        data = resp.json()
+        for r in data["runs"]:
+            if r["status"] in ("completed", "failed") and r["id"] not in done:
+                done.add(r["id"])
+                console.print(eval_line(r))
+        if data["status"] in ("completed", "failed"):
+            break
+    ok = sum(r["status"] == "completed" for r in data["runs"])
+    console.print(f"Done. {ok} of {len(data['runs'])} tasks completed.")
+    if url:
+        console.print(f"Scores: {url}")
+    if ok == 0:
+        raise typer.Exit(1)
+
+
+def eval_line(r: dict) -> str:
+    name = r["task"].rsplit("/", 1)[-1]
+    if r["status"] == "failed":
+        return f"  [red]{name}: failed[/red]"
+    reward = r.get("avg_reward")
+    return f"  {name}: " + (
+        f"reward {reward:.2f}" if reward is not None else "no reward"
+    )
+
+
+def summarize(data: dict) -> tuple[list[str], bool]:
+    """Final lines for a completed run, and whether every trial failed."""
+    trials = data.get("trials", [])
+    n = len(trials)
+    failed = [t for t in trials if t.get("status") == "failed"]
+    plural = "s" if n != 1 else ""
+    avg = data.get("avg_reward")
+    head = f"Done. {n} trial{plural}" + (
+        f", avg reward = {avg:.2f}" if avg is not None else ""
+    )
+    if not failed:
+        return [head + "."], False
+    lines = [f"[red]{head}. {len(failed)} of {n} trial{plural} failed:[/red]"]
+    for t in failed:
+        why = t.get("error") or "no error recorded"
+        stage = f" (last stage: {t['last_stage']})" if t.get("last_stage") else ""
+        lines.append(f"  {t['id']}: {why}{stage}")
+    return lines, len(failed) == n

{newmeasure_cli-0.2.0 → newmeasure_cli-0.3.0}/cli/commands/upload.py

@@ -116,7 +116,9 @@ def upload_browser(task_dir: Path, config: dict):
             if path.is_symlink():
                 raise typer.BadParameter("Task files must not be symlinks")
             info = zipfile.ZipInfo(path.relative_to(task_dir).as_posix())
-            info.external_attr = (path.stat().st_mode & 0o777) << 16  # keep test.sh executable
+            info.external_attr = (
+                path.stat().st_mode & 0o777
+            ) << 16  # keep test.sh executable
             archive.writestr(info, path.read_bytes(), zipfile.ZIP_DEFLATED)
         if not (task_dir / "product.md").exists():
             # Shared task context is packaged by the CLI, never resolved on the server.

{newmeasure_cli-0.2.0 → newmeasure_cli-0.3.0}/cli/config.py

@@ -9,6 +9,8 @@ CONFIG_PATH = CONFIG_DIR / "config.json"
 DEFAULTS = {
     "api_url": "https://api.newmeasure.cc",
     "token": "",
+    "evals_url": "https://review.mocks.newmeasure.cc/evals-api",
+    "evals_token": "",
 }
 
 

newmeasure_cli-0.3.0/cli/errors.py

@@ -0,0 +1,45 @@
+from __future__ import annotations
+
+import httpx
+
+
+def describe(response: httpx.Response) -> str:
+    """The server's reason for an error response, as one line."""
+    try:
+        body = response.json()
+    except ValueError:
+        return f"{response.reason_phrase or 'Request failed'} from {response.url.host}"
+    detail = body.get("detail") if isinstance(body, dict) else None
+    if isinstance(detail, str) and detail:
+        return detail
+    if isinstance(detail, list) and detail:
+        parts = []
+        for item in detail:
+            if not isinstance(item, dict):
+                parts.append(str(item))
+                continue
+            loc = ".".join(str(p) for p in item.get("loc", []) if p != "body")
+            msg = item.get("msg", "invalid")
+            parts.append(f"{loc}: {msg}" if loc else msg)
+        return "; ".join(parts)
+    return response.reason_phrase or "Request failed"
+
+
+def http_error_message(exc: httpx.HTTPStatusError) -> str:
+    response = exc.response
+    request = exc.request
+    message = (
+        f"Error: {describe(response)} "
+        f"(HTTP {response.status_code} on {request.method} {request.url.path})"
+    )
+    if response.status_code == 401:
+        message += "\nCheck your API token: newm login"
+    return message
+
+
+def transport_error_message(exc: httpx.TransportError) -> str:
+    try:
+        target = f"{exc.request.url.scheme}://{exc.request.url.host}"
+    except RuntimeError:
+        target = "the NewMeasure API"
+    return f"Error: couldn't reach {target} ({type(exc).__name__}: {exc})"

newmeasure_cli-0.3.0/cli/main.py

@@ -0,0 +1,187 @@
+from __future__ import annotations
+
+import subprocess
+import sys
+from pathlib import Path
+
+import httpx
+import typer
+
+from cli.commands.admin import admin_app
+from cli.commands.artifacts import artifacts as artifacts_cmd
+from cli.commands.edit_task import edit_task as edit_task_cmd
+from cli.commands.login import login as login_cmd
+from cli.commands.result import result as result_cmd
+from cli.commands.run import run as run_cmd
+from cli.commands.runs import list_runs
+from cli.commands.tasks import list_tasks
+from cli.commands.upload import upload as upload_cmd
+from cli.errors import http_error_message, transport_error_message
+
+app = typer.Typer(
+    name="newm",
+    help="NewMeasure eval CLI",
+    no_args_is_help=True,
+)
+app.add_typer(admin_app, name="admin")
+
+
+@app.command()
+def login(
+    evals: bool = typer.Option(
+        False, "--evals", help="Connect your personal app account"
+    ),
+):
+    """Store API token."""
+    login_cmd(evals)
+
+
+@app.command()
+def upload(task_dir: Path = typer.Argument(..., help="Path to task directory")):
+    """Validate and upload a task."""
+    upload_cmd(task_dir)
+
+
+@app.command()
+def run(
+    target: str = typer.Argument(
+        ...,
+        help="Task ID (t_...), a task folder, or a folder of task folders (e.g. tasks/)",
+    ),
+    n_trials: int = typer.Option(
+        1, "--n-trials", min=1, max=5, help="Trials per task (1–5)"
+    ),
+    credentials_file: Path | None = typer.Option(
+        None,
+        "--credentials-file",
+        help="Per-run application credentials (not uploaded as a file)",
+    ),
+    url: str | None = typer.Option(None, "--url", help="HTTPS target URL for this run"),
+    saved_login: str | None = typer.Option(
+        None, "--login", help="Saved login name or ID"
+    ),
+    prompt_credentials: bool = typer.Option(
+        False,
+        "--prompt-credentials",
+        help="Prompt for application username and password",
+    ),
+    save_login: str | None = typer.Option(
+        None, "--save-login", help="Save the supplied login under this name"
+    ),
+    share_login: bool = typer.Option(
+        False, "--share-login", help="Share the saved login with your organization"
+    ),
+    parent: str | None = typer.Option(
+        None,
+        "--from-run",
+        help="Link to an original run; reuse its environment unless --url is set",
+    ),
+    wait: bool = typer.Option(
+        True, "--wait/--no-wait", help="Wait for results after submission"
+    ),
+    kind: str | None = typer.Option(
+        None,
+        "--kind",
+        help="Only tasks of this kind: email or banner (kind in task.toml under metadata.evals)",
+    ),
+):
+    """Run an eval. Several browser tasks run together as one eval run."""
+    run_cmd(
+        target=target,
+        n_trials=n_trials,
+        credentials_file=credentials_file,
+        wait=wait,
+        kind=kind,
+        url=url,
+        saved_login=saved_login,
+        prompt_credentials=prompt_credentials,
+        save_login=save_login,
+        share_login=share_login,
+        parent=parent,
+    )
+
+
+@app.command()
+def artifacts(
+    run_id: str,
+    trial_id: str,
+    output: Path | None = typer.Option(
+        None, "--output", help="Download artifacts to this directory"
+    ),
+):
+    """List or download a trial's stored artifacts."""
+    artifacts_cmd(run_id, trial_id, output)
+
+
+@app.command()
+def tasks():
+    """List uploaded tasks."""
+    list_tasks()
+
+
+@app.command()
+def runs(task_id: str | None = typer.Argument(None, help="Filter by task ID")):
+    """List runs."""
+    list_runs(task_id)
+
+
+@app.command()
+def result(run_id: str = typer.Argument(..., help="Run ID")):
+    """Get detailed run results."""
+    result_cmd(run_id)
+
+
+@app.command("edit-task")
+def edit_task(
+    operation: str = typer.Argument(
+        ..., help="Solstice operation ID or /home/assets/<id> URL"
+    ),
+    turn: list[str] = typer.Option(
+        [],
+        "--turn",
+        "-t",
+        help="Turn number or turn ID to convert; repeat to merge turns. Omit to list turns",
+    ),
+    title: str = typer.Option("", "--title", help="Task title (default: drafted)"),
+):
+    """Create an Edit-suite task from a Solstice operation."""
+    edit_task_cmd(operation, turn, title)
+
+
+@app.command(
+    context_settings={"allow_extra_args": True, "ignore_unknown_options": True},
+)
+def harbor(ctx: typer.Context):
+    """Passthrough to Harbor CLI (power users)."""
+    cmd = ["harbor", *ctx.args]
+    sys.exit(subprocess.call(cmd))
+
+
+def main() -> None:
+    """Entry point: show the server's reason for a failed request, not a traceback."""
+    try:
+        app()
+    except httpx.HTTPStatusError as exc:
+        typer.secho(http_error_message(exc), err=True, fg=typer.colors.RED)
+        raise SystemExit(1) from None
+    except httpx.TransportError as exc:
+        typer.secho(transport_error_message(exc), err=True, fg=typer.colors.RED)
+        raise SystemExit(1) from None
+
+
+@app.command("logins")
+def logins():
+    """List the same personal and shared logins as the app."""
+    from cli.api_client import evals_client
+
+    with evals_client() as c:
+        r = c.get("run-logins")
+        r.raise_for_status()
+        for entry in r.json()["logins"]:
+            typer.echo(
+                f"{entry['id']}  {entry['name']}  {'Organization' if entry['shared'] else 'Personal'}  Saved by {entry['saved_by']}"
+            )
+
+
+if __name__ == "__main__":
+    main()

{newmeasure_cli-0.2.0 → newmeasure_cli-0.3.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "newmeasure-cli"
-version = "0.2.0"
+version = "0.3.0"
 description = "NewMeasure eval CLI"
 readme = "README.md"
 requires-python = ">=3.11"
@@ -25,4 +25,4 @@ packages = ["cli"]
 include = ["/cli", "/README.md", "/pyproject.toml"]
 
 [project.scripts]
-newm = "cli.main:app"
+newm = "cli.main:main"

newmeasure_cli-0.2.0/cli/api_client.py

@@ -1,19 +0,0 @@
-from __future__ import annotations
-
-import httpx
-import typer
-
-from cli.config import get
-
-
-def client() -> httpx.Client:
-    api_url = get("api_url")
-    token = get("token")
-    if not token:
-        typer.echo("Not logged in. Run: newm login")
-        raise typer.Exit(1)
-    return httpx.Client(
-        base_url=api_url,
-        headers={"Authorization": f"Bearer {token}"},
-        timeout=300,
-    )

newmeasure_cli-0.2.0/cli/commands/run.py

@@ -1,120 +0,0 @@
-from __future__ import annotations
-
-import time
-from pathlib import Path
-from urllib.parse import urlparse
-
-import typer
-from dotenv import dotenv_values
-from rich.console import Console
-
-from cli.api_client import client
-from cli.progress import progress_lines
-from cli.commands.upload import upload
-
-console = Console()
-
-
-def run(
-    target: str,
-    n_trials: int = 1,
-    credentials_file: Path | None = None,
-    wait: bool = True,
-):
-    """Run an eval. TARGET is a task ID (t_...) or a local task directory path."""
-    task_id = target
-    target_path = Path(target)
-    if target_path.is_dir():
-        upload(target_path)
-        import tomllib
-
-        toml_text = (target_path / "task.toml").read_text()
-        task_id = tomllib.loads(toml_text).get("task_id")
-        if not task_id:
-            typer.echo("Failed to get task_id after upload")
-            raise typer.Exit(1)
-
-    c = client()
-    body = {
-        "task_id": task_id,
-        "n_trials": n_trials,
-    }
-
-    task_response = c.get(f"/api/v1/tasks/{task_id}")
-    task_response.raise_for_status()
-    task = task_response.json()
-    if task.get("runner") == "browser":
-        url = urlparse(str(c.base_url))
-        if url.scheme != "https" and url.hostname not in {
-            "127.0.0.1",
-            "localhost",
-            "::1",
-        }:
-            raise typer.BadParameter(
-                "Application credentials require an HTTPS NewMeasure API URL"
-            )
-        if credentials_file is None or not credentials_file.is_file():
-            raise typer.BadParameter(
-                "Browser runs require --credentials-file pointing to a local environment file"
-            )
-        values = dotenv_values(credentials_file, interpolate=False)
-        fields = task.get("credential_fields", [])
-        if any(not values.get(field) for field in fields):
-            raise typer.BadParameter(
-                f"Credential file must define: {', '.join(fields)}"
-            )
-        body["credentials"] = {field: values[field] for field in fields}
-    elif credentials_file:
-        raise typer.BadParameter(
-            "This task does not accept per-run browser credentials"
-        )
-
-    resp = c.post("/api/v1/runs", json=body)
-    body.pop("credentials", None)
-    resp.raise_for_status()
-    run_id = resp.json()["run_id"]
-    console.print(f"Run started: [bold]{run_id}[/bold]")
-    if not wait:
-        return
-
-    console.print("Waiting for results...")
-    shown = 0
-    while True:
-        time.sleep(3)
-        resp = c.get(f"/api/v1/runs/{run_id}")
-        resp.raise_for_status()
-        data = resp.json()
-        status = data["status"]
-        lines = progress_lines(data.get("progress", []))
-        for line in lines[shown:]:
-            console.print(line)
-        shown = len(lines)
-
-        if status == "completed":
-            lines, all_failed = summarize(data)
-            for line in lines:
-                console.print(line)
-            if all_failed:
-                raise typer.Exit(1)
-            break
-        elif status == "failed":
-            console.print("[red]Run failed.[/red]")
-            raise typer.Exit(1)
-
-
-def summarize(data: dict) -> tuple[list[str], bool]:
-    """Final lines for a completed run, and whether every trial failed."""
-    trials = data.get("trials", [])
-    n = len(trials)
-    failed = [t for t in trials if t.get("status") == "failed"]
-    plural = "s" if n != 1 else ""
-    avg = data.get("avg_reward")
-    head = f"Done. {n} trial{plural}" + (f", avg reward = {avg:.2f}" if avg is not None else "")
-    if not failed:
-        return [head + "."], False
-    lines = [f"[red]{head}. {len(failed)} of {n} trial{plural} failed:[/red]"]
-    for t in failed:
-        why = t.get("error") or "no error recorded"
-        stage = f" (last stage: {t['last_stage']})" if t.get("last_stage") else ""
-        lines.append(f"  {t['id']}: {why}{stage}")
-    return lines, len(failed) == n

newmeasure_cli-0.2.0/cli/main.py

@@ -1,98 +0,0 @@
-from __future__ import annotations
-
-import subprocess
-import sys
-from pathlib import Path
-from typing import Optional
-
-import typer
-
-from cli.commands.admin import admin_app
-from cli.commands.artifacts import artifacts as artifacts_cmd
-from cli.commands.login import login as login_cmd
-from cli.commands.result import result as result_cmd
-from cli.commands.run import run as run_cmd
-from cli.commands.runs import list_runs
-from cli.commands.tasks import list_tasks
-from cli.commands.upload import upload as upload_cmd
-
-app = typer.Typer(
-    name="newm",
-    help="NewMeasure eval CLI",
-    no_args_is_help=True,
-)
-app.add_typer(admin_app, name="admin")
-
-
-@app.command()
-def login():
-    """Store API token."""
-    login_cmd()
-
-
-@app.command()
-def upload(task_dir: Path = typer.Argument(..., help="Path to task directory")):
-    """Validate and upload a task."""
-    upload_cmd(task_dir)
-
-
-@app.command()
-def run(
-    target: str = typer.Argument(..., help="Task ID (t_...) or local task directory"),
-    n_trials: int = typer.Option(1, "--n-trials", help="Number of trials"),
-    credentials_file: Optional[Path] = typer.Option(
-        None,
-        "--credentials-file",
-        help="Per-run application credentials (not uploaded as a file)",
-    ),
-    wait: bool = typer.Option(
-        True, "--wait/--no-wait", help="Wait for results after submission"
-    ),
-):
-    """Run an eval against your agent."""
-    run_cmd(
-        target=target, n_trials=n_trials, credentials_file=credentials_file, wait=wait
-    )
-
-
-@app.command()
-def artifacts(
-    run_id: str,
-    trial_id: str,
-    output: Optional[Path] = typer.Option(
-        None, "--output", help="Download artifacts to this directory"
-    ),
-):
-    """List or download a trial's stored artifacts."""
-    artifacts_cmd(run_id, trial_id, output)
-
-
-@app.command()
-def tasks():
-    """List uploaded tasks."""
-    list_tasks()
-
-
-@app.command()
-def runs(task_id: Optional[str] = typer.Argument(None, help="Filter by task ID")):
-    """List runs."""
-    list_runs(task_id)
-
-
-@app.command()
-def result(run_id: str = typer.Argument(..., help="Run ID")):
-    """Get detailed run results."""
-    result_cmd(run_id)
-
-
-@app.command(
-    context_settings={"allow_extra_args": True, "ignore_unknown_options": True},
-)
-def harbor(ctx: typer.Context):
-    """Passthrough to Harbor CLI (power users)."""
-    cmd = ["harbor", *ctx.args]
-    sys.exit(subprocess.call(cmd))
-
-
-if __name__ == "__main__":
-    app()