avp-cli 0.1.0__py3-none-any.whl

avp_cli/brand.py

@@ -0,0 +1,46 @@
+"""AVP branding for the terminal: the ship logo + the palette.
+
+The logo is the same monospace ASCII ship as `assets/logo.svg` (the `<avp>`
+sail on a mast, over a hull). Palette matches the logo and the constellation
+viz: sail gold, mast/light, hull, keel.
+"""
+
+from __future__ import annotations
+
+from rich.text import Text
+
+# Brand palette (hex), shared with the trajectory constellation.
+SAIL = "#f3d28a"
+MAST = "#e6eef3"
+HULL = "#c98b4a"
+KEEL = "#8a5b2c"
+SKY = "#9fd6e7"
+
+# A compact sailboat mark for prompts (the brand ship's one-glyph proxy).
+SAILBOAT = "⛵"
+
+# (text, color) per line, reconstructed from assets/logo.svg's row layout.
+_SHIP: list[list[tuple[str, str]]] = [
+    [("           _", SAIL)],
+    [("        |", MAST), ("<avp>", SAIL)],
+    [("        |", MAST), ("  ‾", SAIL)],
+    [("        |", MAST), ("\\", SAIL)],
+    [("        |", MAST), ("_\\", SAIL)],
+    [("        |", MAST), ("__\\", SAIL)],
+    [("        |", MAST), ("___\\", SAIL)],
+    [("        |", MAST), ("____\\", SAIL)],
+    [("        |", MAST), ("_____\\", SAIL)],
+    [("\\______________/", HULL)],
+    [(" \\____________/", KEEL)],
+]
+
+
+def logo() -> Text:
+    """The AVP ship as a colored rich Text block."""
+    out = Text()
+    for i, line in enumerate(_SHIP):
+        for span, color in line:
+            out.append(span, style=color)
+        if i < len(_SHIP) - 1:
+            out.append("\n")
+    return out

avp_cli/broker.py

@@ -0,0 +1,227 @@
+"""The vault broker: a host-side credential-injecting reverse proxy.
+
+The vault's promise is "the agent can *use* a credential it can never *read*."
+The wire keeps secrets off the Commission (credentials travel as handles); this
+broker keeps the resolved value out of the sandbox entirely.
+
+How it fits together. OpenSandbox filters egress by DNS only, so a sandboxed
+agent can reach a host process at `host.docker.internal` once that name is in
+the egress allowlist (verified empirically). avp points the agent's provider
+base_url and MCP urls at this broker and hands the agent only sentinels; the
+broker, running on the host where the real secret lives, overwrites the auth
+header with the real value and forwards over TLS to the real upstream. The
+secret never crosses into the sandbox; only the broker (host) and the upstream
+ever see it.
+
+The broker is per-run: started in `run_agent`, its routes built from the
+Commission + resolved vault handles, torn down in the run's `finally`. Secrets
+live only in the in-memory route table; nothing is written to disk.
+
+This is also the real egress boundary for secret-bearing traffic: a request
+that matches no route is refused, so the broker can only ever reach the
+commission-declared upstreams (tighter than the DNS allowlist, and
+destination-specific).
+"""
+
+from __future__ import annotations
+
+import threading
+from dataclasses import dataclass
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from urllib.parse import urlsplit, urlunsplit
+
+import httpx
+
+__all__ = ["Broker", "Route"]
+
+# The host alias an OpenSandbox bridge container uses to reach the host. avp
+# adds this to the egress allowlist for broker-mode runs. Docker Desktop /
+# OrbStack inject it; on plain Linux Docker the broker preflight maps it to the
+# default-route gateway (the host) in the sandbox's /etc/hosts, so this single
+# address reaches the broker on every host.
+SANDBOX_HOST_ALIAS = "host.docker.internal"
+
+# Hop-by-hop / recomputed headers we never forward verbatim.
+_DROP_REQUEST_HEADERS = frozenset(
+    {"host", "content-length", "connection", "keep-alive", "transfer-encoding", "te", "upgrade"}
+)
+# Hop-by-hop, plus `content-length` (we re-frame the body as chunked, so the
+# upstream length no longer applies). `content-encoding` is deliberately kept:
+# we forward the body raw (`iter_raw`, still gzip/br-encoded if the upstream
+# compressed it), so the client needs the header to decode it.
+_DROP_RESPONSE_HEADERS = frozenset(
+    {"connection", "keep-alive", "transfer-encoding", "te", "upgrade", "content-length"}
+)
+
+
+@dataclass(frozen=True)
+class Route:
+    """One destination the broker injects credentials for.
+
+    `upstream` is where the request is forwarded. For a provider it is the
+    origin (`https://api.anthropic.com`) and the agent's SDK supplies the path;
+    for an MCP server it is the full real url. `header`/`prefix` name the auth
+    header to overwrite (e.g. `authorization` + `Bearer `, or `x-api-key` + ``);
+    `secret` is the resolved value, held only here on the host.
+    """
+
+    upstream: str
+    header: str
+    prefix: str
+    secret: str
+
+
+class Broker:
+    """A per-run credential-injecting reverse proxy bound on the host.
+
+    Add routes keyed by the first two path segments (`llm/<id>`, `mcp/<id>`),
+    then `start()`. The sandbox reaches it at `base_url()`.
+    """
+
+    def __init__(self) -> None:
+        self._routes: dict[str, Route] = {}
+        self._server: ThreadingHTTPServer | None = None
+        self._thread: threading.Thread | None = None
+        self._client = httpx.Client(
+            timeout=httpx.Timeout(600.0, connect=15.0), follow_redirects=False
+        )
+
+    def add_route(self, key: str, route: Route) -> None:
+        """Register a route. `key` is the path prefix after the leading slash,
+        e.g. `llm/anthropic` or `mcp/network`."""
+        self._routes[key.strip("/")] = route
+
+    @property
+    def port(self) -> int:
+        if self._server is None:
+            raise RuntimeError("broker not started")
+        return self._server.server_address[1]
+
+    def base_url(self) -> str:
+        """The URL the sandboxed agent uses to reach the broker."""
+        return f"http://{SANDBOX_HOST_ALIAS}:{self.port}"
+
+    def route_url(self, key: str) -> str:
+        """The full broker URL for a given route key."""
+        return f"{self.base_url()}/{key.strip('/')}"
+
+    def start(self) -> None:
+        # Bind on all interfaces so the bridge can reach us; port 0 = ephemeral.
+        handler = _make_handler(self)
+        self._server = ThreadingHTTPServer(("0.0.0.0", 0), handler)
+        self._thread = threading.Thread(target=self._server.serve_forever, daemon=True)
+        self._thread.start()
+
+    def stop(self) -> None:
+        if self._server is not None:
+            self._server.shutdown()
+            self._server.server_close()
+            self._server = None
+        self._client.close()
+
+    def __enter__(self) -> Broker:
+        self.start()
+        return self
+
+    def __exit__(self, *exc: object) -> None:
+        self.stop()
+
+    # ── request handling ─────────────────────────────────────────────────────
+
+    def _match(self, path: str) -> tuple[Route, str] | None:
+        """Resolve a request path to (route, remainder-after-key)."""
+        parts = path.lstrip("/").split("/", 2)
+        if len(parts) < 2:
+            return None
+        key = f"{parts[0]}/{parts[1]}"
+        route = self._routes.get(key)
+        if route is None:
+            return None
+        remainder = f"/{parts[2]}" if len(parts) == 3 else ""
+        return route, remainder
+
+    def _target_url(self, route: Route, remainder: str, query: str) -> str:
+        base = route.upstream.rstrip("/")
+        url = base + remainder if remainder else base
+        if query:
+            url = f"{url}?{query}"
+        return url
+
+    def _forward_headers(self, route: Route, headers: dict[str, str]) -> dict[str, str]:
+        out: dict[str, str] = {}
+        for name, value in headers.items():
+            low = name.lower()
+            if low in _DROP_REQUEST_HEADERS or low == route.header.lower():
+                continue
+            out[name] = value
+        # Overwrite (never append) the auth header with the real secret.
+        out[route.header] = f"{route.prefix}{route.secret}"
+        # Pin Host to the upstream so the upstream's TLS/vhost routing is correct.
+        out["Host"] = urlsplit(self._target_url(route, "", "")).netloc
+        return out
+
+
+def _make_handler(broker: Broker) -> type[BaseHTTPRequestHandler]:
+    class Handler(BaseHTTPRequestHandler):
+        protocol_version = "HTTP/1.1"
+
+        def log_message(self, *args: object) -> None:  # silence default logging
+            pass
+
+        def _health(self) -> bool:
+            if self.path.rstrip("/") == "/health":
+                self.send_response(200)
+                self.send_header("Content-Length", "2")
+                self.end_headers()
+                self.wfile.write(b"ok")
+                return True
+            return False
+
+        def _handle(self) -> None:
+            if self._health():
+                return
+            split = urlsplit(self.path)
+            matched = broker._match(split.path)
+            if matched is None:
+                self.send_error(404, "no broker route")
+                return
+            route, remainder = matched
+            target = broker._target_url(route, remainder, split.query)
+            length = int(self.headers.get("Content-Length") or 0)
+            body = self.rfile.read(length) if length else None
+            headers = broker._forward_headers(route, dict(self.headers.items()))
+            try:
+                with broker._client.stream(
+                    self.command, target, headers=headers, content=body
+                ) as upstream:
+                    self.send_response(upstream.status_code)
+                    for name, value in upstream.headers.items():
+                        if name.lower() in _DROP_RESPONSE_HEADERS:
+                            continue
+                        self.send_header(name, value)
+                    self.send_header("Transfer-Encoding", "chunked")
+                    self.end_headers()
+                    for chunk in upstream.iter_raw():
+                        if chunk:
+                            self.wfile.write(f"{len(chunk):X}\r\n".encode())
+                            self.wfile.write(chunk)
+                            self.wfile.write(b"\r\n")
+                            self.wfile.flush()
+                    self.wfile.write(b"0\r\n\r\n")
+            except Exception as exc:  # upstream unreachable / stream error
+                self.send_error(502, f"broker upstream error: {exc}")
+
+        do_GET = _handle
+        do_POST = _handle
+        do_PUT = _handle
+        do_DELETE = _handle
+        do_PATCH = _handle
+
+    return Handler
+
+
+def origin_of(url: str) -> str:
+    """The scheme://host[:port] origin of a url (provider routes forward here;
+    the agent SDK supplies the path)."""
+    s = urlsplit(url)
+    return urlunsplit((s.scheme, s.netloc, "", "", ""))

avp_cli/catalog/__init__.py

@@ -0,0 +1,128 @@
+"""The `avp init` catalog: real, scaffold-able evals (JSON, not code).
+
+Each packaged entry is a `{eval, commissions}` document. `avp init <key>`
+**installs** its commissions into the portable library (`~/.avp/commissions/`,
+skipping any id you already have) and writes the eval file *in place* as
+`<key>.eval.json` so you can edit and commit it. The eval references the
+commissions by id. `capitals` is the smallest entry (the default off a
+non-interactive terminal): inline data, runs in seconds for pennies, no extra deps.
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+from importlib.resources import files
+from pathlib import Path
+from typing import Any
+
+from avp.commission import Commission
+from avp_cli import library
+
+
+@dataclass(frozen=True)
+class CatalogEntry:
+    key: str
+    title: str
+    description: str
+    file: str
+    needs: list[str] = field(default_factory=list)  # optional extras the config requires
+
+
+@dataclass(frozen=True)
+class ScaffoldResult:
+    eval_path: Path
+    installed: list[str]  # commission ids written into the library
+    skipped: list[str]  # ids already present, left untouched
+
+
+# Ordered for the picker. Each `file` is a `{eval, commissions}` JSON in this package.
+ENTRIES: list[CatalogEntry] = [
+    CatalogEntry(
+        key="capitals",
+        title="Capitals (structured extraction)",
+        description="Tiny structured-extraction sample. Inline data, runs for pennies, no extra deps.",
+        file="capitals.json",
+    ),
+    CatalogEntry(
+        key="parsebench",
+        title="ParseBench (tables)",
+        description="PDF pages to HTML, scored on structural fidelity. Real LlamaIndex benchmark.",
+        file="parsebench.json",
+        needs=["parsebench"],
+    ),
+    CatalogEntry(
+        key="custom",
+        title="Custom (start from scratch)",
+        description="A minimal real eval you fill in with your own task and commissions.",
+        file="custom.json",
+    ),
+]
+
+
+def get(key: str) -> CatalogEntry | None:
+    return next((e for e in ENTRIES if e.key == key), None)
+
+
+def load(entry: CatalogEntry) -> dict[str, Any]:
+    """Parse the entry's packaged `{eval, commissions}` document."""
+    return json.loads((files("avp_cli.catalog") / entry.file).read_text())
+
+
+def _unique_eval_path(dest_dir: Path, key: str) -> Path:
+    """`<dest>/<key>.eval.json`, or `<key>-2.eval.json`, `-3`, ... if taken.
+
+    `avp init` is non-destructive: scaffolding a benchmark you already have writes
+    a fresh, editable copy rather than clobbering or refusing.
+    """
+    target = dest_dir / f"{key}.eval.json"
+    i = 2
+    while target.exists():
+        target = dest_dir / f"{key}-{i}.eval.json"
+        i += 1
+    return target
+
+
+def scaffold(
+    entry: CatalogEntry,
+    dest_dir: Path,
+    agents: list[str] | None = None,
+    *,
+    commissions_dir: Path | None = None,
+) -> ScaffoldResult:
+    """Install the entry's commissions into the library and write its eval file.
+
+    The eval file goes to `<dest>/<key>.eval.json` (or a `-2`, `-3`, ... suffix if
+    that name is taken) and references the commissions by id. Commissions go to the
+    library; an id you already have is left untouched (reported in `skipped`).
+
+    The eval's `commissions` block is taken verbatim from the entry when present
+    (so an entry can bind commissions to agents via the `{agent: [ids]}` map);
+    otherwise it defaults to a flat list of every installed id. When `agents` is
+    given and the entry doesn't already bind agents via a map, they're pinned in
+    an `"agents"` key so `avp eval run` targets them without `--agent`.
+    """
+    doc = load(entry)
+    commissions: dict[str, Any] = doc["commissions"]  # {id: wire Commission}
+    installed: list[str] = []
+    skipped: list[str] = []
+    for cid, spec in commissions.items():
+        if library.exists(cid, commissions_dir=commissions_dir):
+            skipped.append(cid)
+        else:
+            library.save(cid, Commission.model_validate(spec), commissions_dir=commissions_dir)
+            installed.append(cid)
+
+    target = _unique_eval_path(dest_dir, entry.key)
+    eval_spec = doc["eval"]
+    eval_doc: dict[str, Any] = {"name": eval_spec.get("name", entry.key)}
+    binding = eval_spec.get("commissions", list(commissions.keys()))
+    # A map binding names its agents in the keys; only pin a separate "agents"
+    # key for the flat-list form.
+    if agents and not isinstance(binding, dict):
+        eval_doc["agents"] = agents
+    eval_doc["dataset"] = eval_spec["dataset"]
+    eval_doc["scorer"] = eval_spec["scorer"]
+    eval_doc["commissions"] = binding
+    target.write_text(json.dumps(eval_doc, indent=2) + "\n")
+    return ScaffoldResult(eval_path=target, installed=installed, skipped=skipped)

avp_cli/catalog/capitals.json

@@ -0,0 +1,67 @@
+{
+  "eval": {
+    "name": "capitals-extraction",
+    "dataset": {
+      "source": "inline",
+      "items": [
+        {
+          "id": "paris",
+          "prompt": "Paris is the capital of France; about 2 million people live there.",
+          "expected": {
+            "city": "Paris",
+            "country": "France",
+            "population_millions": 2
+          }
+        },
+        {
+          "id": "tokyo",
+          "prompt": "Tokyo, the capital of Japan, is home to around 14 million.",
+          "expected": {
+            "city": "Tokyo",
+            "country": "Japan",
+            "population_millions": 14
+          }
+        }
+      ]
+    },
+    "scorer": {
+      "name": "structural-match",
+      "threshold": 1.0
+    }
+  },
+  "commissions": {
+    "capitals-baseline": {
+      "schema_version": "0.1",
+      "run_id": "capitals-baseline",
+      "model": "anthropic/claude-haiku-4-5",
+      "prompt": "Extract the structured facts from this sentence: {input}",
+      "output_schema": {
+        "type": "object",
+        "properties": {
+          "city": { "type": "string" },
+          "country": { "type": "string" },
+          "population_millions": { "type": "number" }
+        },
+        "required": ["city", "country", "population_millions"],
+        "additionalProperties": false
+      }
+    },
+    "capitals-few-shot": {
+      "schema_version": "0.1",
+      "run_id": "capitals-few-shot",
+      "model": "anthropic/claude-haiku-4-5",
+      "system_prompt": "You extract facts as JSON. Example:\nInput: \"Berlin is the capital of Germany, about 3.7 million people.\"\nOutput: {\"city\": \"Berlin\", \"country\": \"Germany\", \"population_millions\": 3.7}",
+      "prompt": "Now do the same for: {input}",
+      "output_schema": {
+        "type": "object",
+        "properties": {
+          "city": { "type": "string" },
+          "country": { "type": "string" },
+          "population_millions": { "type": "number" }
+        },
+        "required": ["city", "country", "population_millions"],
+        "additionalProperties": false
+      }
+    }
+  }
+}

avp_cli/catalog/custom.json

@@ -0,0 +1,35 @@
+{
+  "eval": {
+    "name": "my-eval",
+    "dataset": {
+      "source": "inline",
+      "items": [
+        {
+          "id": "example-1",
+          "prompt": "Replace this with your task input.",
+          "expected": {
+            "answer": "what a correct result looks like"
+          }
+        }
+      ]
+    },
+    "scorer": {
+      "name": "structural-match",
+      "threshold": 1.0
+    }
+  },
+  "commissions": {
+    "baseline": {
+      "schema_version": "0.1",
+      "run_id": "baseline",
+      "model": "anthropic/claude-haiku-4-5",
+      "prompt": "{input}"
+    },
+    "variant-a": {
+      "schema_version": "0.1",
+      "run_id": "variant-a",
+      "model": "anthropic/claude-haiku-4-5",
+      "prompt": "{input}"
+    }
+  }
+}

avp_cli/catalog/parsebench.json

@@ -0,0 +1,44 @@
+{
+  "eval": {
+    "name": "parsebench-table",
+    "dataset": {
+      "source": "huggingface",
+      "id": "llamaindex/ParseBench",
+      "split": "table[:2]",
+      "input": "https://huggingface.co/datasets/llamaindex/ParseBench/resolve/main/{pdf}",
+      "expected_field": "expected_markdown",
+      "id_field": "id"
+    },
+    "scorer": {
+      "name": "structural-fidelity",
+      "threshold": 0.8
+    },
+    "commissions": {
+      "goose": ["parsebench-goose"],
+      "claude-code": ["parsebench-claude"]
+    }
+  },
+  "commissions": {
+    "parsebench-goose": {
+      "schema_version": "0.1",
+      "run_id": "parsebench-goose",
+      "model": "anthropic/claude-opus-4-8",
+      "prompt": "Reproduce the table from a PDF page as a single HTML <table>.\n1. Download it: curl -sL '{input}' -o page.pdf\n2. Render the page to an image with the pdf-vision get_page_image tool and LOOK at it. The image is the ground truth for the 2D layout: how many columns there are, which cells are merged or span rows, section-header rows, and which values visually share one cell (e.g. multiple holders inside a single cell). Use get_page_text only to copy exact text; trust the image for structure.\n3. Rebuild as ONE HTML <table> that matches what you see: one <tr> per visual row, <th> for header cells, <td> for data, colspan/rowspan for merged cells. Keep column order and exact cell text. Do NOT split a value into extra columns or merge rows that are visually separate.\n4. Verify against the image: the header column count and each row's cell count must match the page. Fix and redo if not. Do not submit a table you can see is wrong.\n5. Output only the final HTML <table>.",
+      "enabled_builtin_tools": ["shell", "write", "edit"],
+      "mcp_servers": [
+        {
+          "type": "stdio",
+          "id": "pdf-vision",
+          "command": ["uvx", "--from", "git+https://github.com/I-CAN-hack/pdf-mcp.git", "pdf-mcp"]
+        }
+      ]
+    },
+    "parsebench-claude": {
+      "schema_version": "0.1",
+      "run_id": "parsebench-claude",
+      "model": "anthropic/claude-opus-4-8",
+      "prompt": "Download the PDF page at {input}, read it, rebuild it as a single HTML <table>, then re-read the original and verify your table matches before returning. Output only the HTML.",
+      "enabled_builtin_tools": ["Bash", "Read", "Write", "WebFetch"]
+    }
+  }
+}