quickstarted 0.2.0__py3-none-any.whl

quickstarted/exec/local.py

@@ -0,0 +1,16 @@
+"""Unenforced local execution: a throwaway directory and a scrubbed environment.
+
+Commands run as your user, on your filesystem, with your network. Absolute
+paths reach your home directory and a command that ignores the proxy variables
+reaches any host. Useful for developing journeys against your own code; wrong
+for anything you did not write.
+"""
+
+from __future__ import annotations
+
+from .base import ProcessExecutor
+
+
+class LocalExecutor(ProcessExecutor):
+    name = "local"
+    enforced = False

quickstarted/exec/seatbelt.py

@@ -0,0 +1,83 @@
+"""Enforced local execution on macOS via sandbox-exec (Seatbelt).
+
+This is the backend that makes the harness's central claim true rather than
+merely intended. The kernel, not the agent's good manners, is what stops a
+command from reaching a documentation host directly: all outbound network is
+denied except the loopback port the harness proxy listens on, so every page
+the agent reads is either a `read_docs` call or a recorded proxy request.
+
+It also confines the blast radius of running commands that came out of a
+stranger's quickstart: reads of the real home directory are denied and writes
+are confined to the workspace.
+
+Seatbelt is deprecated by Apple but present and functional; Docker is the
+portable path, and CI should use it.
+"""
+
+from __future__ import annotations
+
+import os
+import platform
+from pathlib import Path
+
+from .base import ExecutorError, ProcessExecutor
+
+SANDBOX_EXEC = "/usr/bin/sandbox-exec"
+
+
+def available() -> bool:
+    return platform.system() == "Darwin" and Path(SANDBOX_EXEC).is_file()
+
+
+def build_profile(workspace: Path, proxy_port: int | None, real_home: Path) -> str:
+    """Seatbelt profile source. Later rules win, so order is load-bearing."""
+    lines = [
+        "(version 1)",
+        "(deny default)",
+        # Running programs at all.
+        "(allow process-exec)",
+        "(allow process-fork)",
+        "(allow sysctl-read)",
+        "(allow mach-lookup)",
+        "(allow ipc-posix-shm)",
+        "(allow signal (target same-sandbox))",
+        # Interpreters and compilers read all over the system prefix.
+        "(allow file-read*)",
+        # ... but not the user's own files. This is the point.
+        f'(deny file-read* (subpath "{real_home}"))',
+        f'(allow file-read* (subpath "{workspace}"))',
+        # Writes stay in the workspace, plus the device files tools expect.
+        f'(allow file-write* (subpath "{workspace}"))',
+        '(allow file-write* (regex #"^/dev/"))',
+        "(allow file-ioctl)",
+    ]
+    if proxy_port:
+        # The only route off the machine is the harness proxy.
+        lines.append(f'(allow network-outbound (remote ip "localhost:{proxy_port}"))')
+        # Loopback DNS and similar helpers travel over unix sockets; the proxy
+        # resolves real hostnames on the agent's behalf.
+        lines.append("(allow network-outbound (remote unix-socket))")
+        lines.append("(allow network-bind (local ip \"localhost:*\"))")
+    return "\n".join(lines) + "\n"
+
+
+class SeatbeltExecutor(ProcessExecutor):
+    name = "seatbelt"
+    enforced = True
+
+    def __init__(self, keep: bool = False, proxy_url: str | None = None):
+        if not available():
+            raise ExecutorError(
+                "seatbelt backend requires macOS with /usr/bin/sandbox-exec"
+            )
+        super().__init__(keep=keep, proxy_url=proxy_url)
+        port = None
+        if proxy_url:
+            port = int(proxy_url.rsplit(":", 1)[-1].strip("/"))
+        real_home = Path(os.path.expanduser("~")).resolve()
+        self.profile = build_profile(self.root.resolve(), port, real_home)
+        self._profile_path = self.root / "tmp" / ".quickstarted-sandbox.sb"
+        self._profile_path.write_text(self.profile, encoding="utf-8")
+
+    def argv(self, command: str) -> list[str]:
+        return [SANDBOX_EXEC, "-f", str(self._profile_path), "bash", "-c", command]

quickstarted/journey.py

@@ -0,0 +1,182 @@
+"""Journey definitions: the YAML unit of testing.
+
+A journey states a goal an agent should reach using only the target project's
+documentation, plus a machine-checkable success assertion. Pass/fail is always
+decided by the assertion script's exit code, never by a model's opinion.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from urllib.parse import urlparse
+
+import yaml
+
+from .net.proxy import DEFAULT_NETWORK_ALLOW, host_matches
+
+
+class JourneyError(ValueError):
+    """Raised when a journey file is missing or malformed."""
+
+
+@dataclass(frozen=True)
+class Budgets:
+    max_turns: int = 20
+    max_seconds: int = 900
+    max_command_seconds: int = 300
+    max_output_chars: int = 20_000
+    #: Hard ceiling on billable tokens for one run, cache traffic included.
+    #: 0 means unlimited. Prefer this to a dollar cap: it needs no price list
+    #: and cannot drift when vendors change their rates.
+    max_tokens: int = 0
+
+
+@dataclass(frozen=True)
+class Journey:
+    name: str
+    goal: str
+    docs_entrypoint: str
+    docs_allow: tuple[str, ...]
+    success_script: str
+    setup: tuple[str, ...] = ()
+    replay: tuple[str, ...] = ()
+    budgets: Budgets = field(default_factory=Budgets)
+    #: Hosts a shell may reach (package registries and the like). Documentation
+    #: hosts are deliberately excluded: they are readable only through the
+    #: recorded `read_docs` tool, which is what keeps attribution complete.
+    network_allow: tuple[str, ...] = DEFAULT_NETWORK_ALLOW
+    #: Hosts named by hand under `network.allow`/`network.only`. These beat the
+    #: docs-host rule, so a registry that also serves documentation stays
+    #: installable when the author says so.
+    network_explicit: tuple[str, ...] = ()
+    source: str = ""
+
+    @property
+    def network_conflicts(self) -> tuple[str, ...]:
+        """Docs hosts that installs usually need, which will now be refused.
+
+        Declaring a package registry as a documentation host is the easy
+        mistake: the proxy then refuses `pip install`, and the run fails for a
+        reason that has nothing to do with the documentation.
+        """
+        return tuple(
+            host
+            for host in self.docs_allow
+            if host_matches(host, DEFAULT_NETWORK_ALLOW)
+            and not host_matches(host, self.network_explicit)
+        )
+
+    @property
+    def attribution_gaps(self) -> tuple[str, ...]:
+        """Docs hosts a shell may also reach, so their reads are not all logged."""
+        return tuple(
+            host for host in self.docs_allow if host_matches(host, self.network_explicit)
+        )
+
+    def host_allowed(self, url: str) -> bool:
+        host = (urlparse(url).hostname or "").lower()
+        if not host:
+            return False
+        return host_matches(host, self.docs_allow)
+
+
+def _require(data: dict, key: str, source: str):
+    if key not in data or data[key] in (None, "", []):
+        raise JourneyError(f"{source}: missing required field '{key}'")
+    return data[key]
+
+
+def _str_list(value, key: str, source: str) -> tuple[str, ...]:
+    if value is None:
+        return ()
+    if not isinstance(value, list) or not all(isinstance(x, str) for x in value):
+        raise JourneyError(f"{source}: '{key}' must be a list of strings")
+    return tuple(x.strip() for x in value if x.strip())
+
+
+def _normalize_host(entry: str, source: str) -> str:
+    entry = entry.strip().lower()
+    if "://" in entry:
+        entry = urlparse(entry).hostname or ""
+    entry = entry.strip("/")
+    if not entry or "/" in entry:
+        raise JourneyError(
+            f"{source}: docs.allow entries must be bare hostnames, got {entry!r}"
+        )
+    return entry
+
+
+def load_journey(path: str | Path) -> Journey:
+    path = Path(path)
+    source = str(path)
+    if not path.is_file():
+        raise JourneyError(f"{source}: no such file")
+    try:
+        data = yaml.safe_load(path.read_text(encoding="utf-8"))
+    except yaml.YAMLError as exc:
+        raise JourneyError(f"{source}: invalid YAML: {exc}") from exc
+    if not isinstance(data, dict):
+        raise JourneyError(f"{source}: top level must be a mapping")
+
+    name = str(_require(data, "name", source))
+    goal = str(_require(data, "goal", source)).strip()
+
+    docs = _require(data, "docs", source)
+    if not isinstance(docs, dict):
+        raise JourneyError(f"{source}: 'docs' must be a mapping")
+    entrypoint = str(_require(docs, "entrypoint", source)).strip()
+    parsed = urlparse(entrypoint)
+    if parsed.scheme not in ("http", "https") or not parsed.hostname:
+        raise JourneyError(f"{source}: docs.entrypoint must be an http(s) URL")
+    allow = [
+        _normalize_host(h, source)
+        for h in _str_list(docs.get("allow"), "docs.allow", source)
+    ]
+    entry_host = parsed.hostname.lower()
+    if entry_host not in allow:
+        allow.insert(0, entry_host)
+
+    network = data.get("network") or {}
+    if not isinstance(network, dict):
+        raise JourneyError(f"{source}: 'network' must be a mapping")
+    unknown_net = set(network) - {"allow", "only"}
+    if unknown_net:
+        raise JourneyError(f"{source}: unknown network keys: {sorted(unknown_net)}")
+    extra_net = [
+        _normalize_host(h, source)
+        for h in _str_list(network.get("allow"), "network.allow", source)
+    ]
+    only_net = [
+        _normalize_host(h, source)
+        for h in _str_list(network.get("only"), "network.only", source)
+    ]
+    network_allow = tuple(only_net) if only_net else DEFAULT_NETWORK_ALLOW + tuple(extra_net)
+
+    success = _require(data, "success", source)
+    if not isinstance(success, dict):
+        raise JourneyError(f"{source}: 'success' must be a mapping")
+    success_script = str(_require(success, "script", source))
+
+    budgets_data = data.get("budgets") or {}
+    if not isinstance(budgets_data, dict):
+        raise JourneyError(f"{source}: 'budgets' must be a mapping")
+    known = set(Budgets.__dataclass_fields__)
+    unknown = set(budgets_data) - known
+    if unknown:
+        raise JourneyError(f"{source}: unknown budget keys: {sorted(unknown)}")
+    budgets = Budgets(**{k: int(v) for k, v in budgets_data.items()})
+
+    return Journey(
+        name=name,
+        goal=goal,
+        docs_entrypoint=entrypoint,
+        docs_allow=tuple(allow),
+        success_script=success_script,
+        setup=_str_list(data.get("setup"), "setup", source),
+        replay=_str_list(data.get("replay"), "replay", source),
+        budgets=budgets,
+        network_allow=network_allow,
+        network_explicit=tuple(only_net or extra_net),
+        source=source,
+    )

quickstarted/net/__init__.py

@@ -0,0 +1,5 @@
+"""Network policy: the harness-owned egress proxy."""
+
+from .proxy import DEFAULT_NETWORK_ALLOW, EgressProxy, host_matches
+
+__all__ = ["DEFAULT_NETWORK_ALLOW", "EgressProxy", "host_matches"]

quickstarted/net/proxy.py

@@ -0,0 +1,274 @@
+"""The harness-owned egress proxy.
+
+Every packet an agent's shell sends leaves through here, which buys three
+things the v0 design only hoped for:
+
+1. **Enforcement.** The journey's network allowlist is applied by the proxy,
+   not by asking the agent nicely to use the `read_docs` tool.
+2. **Attribution.** Documentation hosts are deliberately *not* reachable from
+   the shell. An agent that tries `curl https://docs.example.com/...` is
+   refused and the attempt is recorded, so the set of pages the agent read is
+   the set the trace knows about. Failure attribution stops being a guess.
+3. **Honest infrastructure signals.** A connection refused by us and a
+   connection that failed upstream are different events, which is what lets a
+   run be classified as a docs failure rather than a flaky network.
+
+The split matters: `docs.allow` hosts are readable only through `read_docs`,
+while `network.allow` hosts (package registries and the like) are what a
+shell may talk to in order to install things.
+"""
+
+from __future__ import annotations
+
+import select
+import socket
+import threading
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from urllib.parse import urlparse
+
+# Package registries and source hosts a quickstart legitimately needs.
+DEFAULT_NETWORK_ALLOW = (
+    "pypi.org",
+    "files.pythonhosted.org",
+    "registry.npmjs.org",
+    "github.com",
+    "codeload.github.com",
+    "objects.githubusercontent.com",
+    "crates.io",
+    "static.crates.io",
+    "proxy.golang.org",
+)
+
+_RELAY_CHUNK = 65536
+
+
+def host_matches(host: str, entries) -> bool:
+    host = (host or "").lower().strip(".")
+    for entry in entries:
+        entry = entry.lower().strip(".")
+        if host == entry or host.endswith("." + entry):
+            return True
+    return False
+
+
+class EgressProxy:
+    """A minimal forward proxy: CONNECT tunnels plus absolute-URI HTTP."""
+
+    #: Overridden to "0.0.0.0" when running as a sidecar for a container.
+    bind_host = "127.0.0.1"
+
+    def __init__(self, network_allow, docs_hosts=(), trace=None, explicit_allow=()):
+        self.network_allow = tuple(network_allow)
+        self.docs_hosts = tuple(docs_hosts)
+        #: Hosts the journey named under `network.allow` by hand. A host can be
+        #: both documentation and a package registry (PyPI is the obvious
+        #: case), and when an author says so explicitly, installs win over
+        #: attribution for that host. The override is recorded, not silent.
+        self.explicit_allow = tuple(explicit_allow)
+        self.trace = trace
+        self._server: ThreadingHTTPServer | None = None
+        self._thread: threading.Thread | None = None
+        self.blocked_docs_attempts = 0
+
+    # -- policy ---------------------------------------------------------
+    def decide(self, host: str) -> tuple[bool, str]:
+        """(allowed, reason). Docs hosts lose to keep attribution complete."""
+        if host_matches(host, self.explicit_allow):
+            return True, "explicit_network_override"
+        if host_matches(host, self.docs_hosts):
+            return False, "docs_host_requires_read_docs"
+        if host_matches(host, self.network_allow):
+            return True, "allowlisted"
+        return False, "not_allowlisted"
+
+    def record(self, event: str, **data) -> None:
+        if self.trace is not None:
+            self.trace.add(event, **data)
+
+    # -- lifecycle ------------------------------------------------------
+    @property
+    def url(self) -> str:
+        if not self._server:
+            raise RuntimeError("proxy not started")
+        _host, port = self._server.server_address[:2]
+        return f"http://127.0.0.1:{port}"
+
+    @property
+    def port(self) -> int:
+        if not self._server:
+            raise RuntimeError("proxy not started")
+        return int(self._server.server_address[1])
+
+    def start(self, port: int = 0) -> str:
+        proxy = self
+
+        class Handler(_ProxyHandler):
+            pass
+
+        Handler.proxy = proxy
+        self._server = ThreadingHTTPServer((self.bind_host, port), Handler)
+        self._server.daemon_threads = True
+        self._thread = threading.Thread(
+            target=self._server.serve_forever, kwargs={"poll_interval": 0.2}, daemon=True
+        )
+        self._thread.start()
+        return self.url
+
+    def stop(self) -> None:
+        if self._server:
+            self._server.shutdown()
+            self._server.server_close()
+            self._server = None
+        if self._thread:
+            self._thread.join(timeout=5)
+            self._thread = None
+
+    def __enter__(self) -> EgressProxy:
+        self.start()
+        return self
+
+    def __exit__(self, *exc) -> None:
+        self.stop()
+
+
+class _ProxyHandler(BaseHTTPRequestHandler):
+    proxy: EgressProxy
+    protocol_version = "HTTP/1.1"
+
+    def log_message(self, fmt, *args):
+        pass
+
+    # -- helpers --------------------------------------------------------
+    def _refuse(self, host: str, reason: str, method: str) -> None:
+        self.proxy.record(
+            "egress_blocked", host=host, reason=reason, method=method
+        )
+        if reason == "docs_host_requires_read_docs":
+            self.proxy.blocked_docs_attempts += 1
+            body = (
+                f"BLOCKED by quickstarted: {host} is a documentation host. "
+                "Read documentation with the read_docs tool, not the shell, so "
+                "that every page you read is recorded.\n"
+            ).encode()
+        else:
+            allowed = ", ".join(self.proxy.network_allow) or "(none)"
+            body = (
+                f"BLOCKED by quickstarted: {host} is not on this journey's network "
+                f"allowlist ({allowed}).\n"
+            ).encode()
+        self.send_response(403)
+        self.send_header("Content-Type", "text/plain")
+        self.send_header("Content-Length", str(len(body)))
+        self.send_header("Connection", "close")
+        self.end_headers()
+        self.wfile.write(body)
+
+    def _upstream_failed(self, host: str, exc: Exception, method: str) -> None:
+        self.proxy.record(
+            "egress_error", host=host, error=str(exc), method=method
+        )
+        body = f"quickstarted proxy: upstream connection to {host} failed: {exc}\n".encode()
+        self.send_response(502)
+        self.send_header("Content-Type", "text/plain")
+        self.send_header("Content-Length", str(len(body)))
+        self.send_header("Connection", "close")
+        self.end_headers()
+        self.wfile.write(body)
+
+    # -- CONNECT (https) -------------------------------------------------
+    def do_CONNECT(self) -> None:
+        target = self.path
+        host, _, port_s = target.partition(":")
+        port = int(port_s or 443)
+        allowed, reason = self.proxy.decide(host)
+        if not allowed:
+            self._refuse(host, reason, "CONNECT")
+            return
+        try:
+            upstream = socket.create_connection((host, port), timeout=30)
+        except OSError as exc:
+            self._upstream_failed(host, exc, "CONNECT")
+            return
+        self.proxy.record("egress_allowed", host=host, port=port, method="CONNECT")
+        self.send_response(200, "Connection established")
+        self.end_headers()
+        try:
+            self.wfile.flush()
+        except OSError:
+            upstream.close()
+            return
+        self._relay(self.connection, upstream)
+
+    def _relay(self, client: socket.socket, upstream: socket.socket) -> None:
+        sockets = [client, upstream]
+        try:
+            while True:
+                readable, _, errored = select.select(sockets, [], sockets, 60)
+                if errored or not readable:
+                    break
+                for sock in readable:
+                    other = upstream if sock is client else client
+                    try:
+                        data = sock.recv(_RELAY_CHUNK)
+                    except OSError:
+                        return
+                    if not data:
+                        return
+                    try:
+                        other.sendall(data)
+                    except OSError:
+                        return
+        finally:
+            upstream.close()
+
+    # -- plain HTTP ------------------------------------------------------
+    def _forward(self) -> None:
+        parsed = urlparse(self.path)
+        host = parsed.hostname or ""
+        if not host:
+            self._refuse(host or "(relative-url)", "not_allowlisted", self.command)
+            return
+        allowed, reason = self.proxy.decide(host)
+        if not allowed:
+            self._refuse(host, reason, self.command)
+            return
+        port = parsed.port or 80
+        path = parsed.path or "/"
+        if parsed.query:
+            path += "?" + parsed.query
+        length = int(self.headers.get("Content-Length") or 0)
+        body = self.rfile.read(length) if length else b""
+        try:
+            upstream = socket.create_connection((host, port), timeout=30)
+        except OSError as exc:
+            self._upstream_failed(host, exc, self.command)
+            return
+        self.proxy.record(
+            "egress_allowed", host=host, port=port, method=self.command, path=path
+        )
+        try:
+            request = [f"{self.command} {path} HTTP/1.1", f"Host: {parsed.netloc}"]
+            for key, value in self.headers.items():
+                if key.lower() in ("proxy-connection", "connection", "host"):
+                    continue
+                request.append(f"{key}: {value}")
+            request.append("Connection: close")
+            blob = ("\r\n".join(request) + "\r\n\r\n").encode() + body
+            upstream.sendall(blob)
+            while True:
+                chunk = upstream.recv(_RELAY_CHUNK)
+                if not chunk:
+                    break
+                self.wfile.write(chunk)
+        except OSError:
+            pass
+        finally:
+            upstream.close()
+        self.close_connection = True
+
+    do_GET = _forward
+    do_POST = _forward
+    do_HEAD = _forward
+    do_PUT = _forward
+    do_DELETE = _forward
+    do_PATCH = _forward

quickstarted/net/proxy_main.py

@@ -0,0 +1,53 @@
+"""Entrypoint for running the egress proxy as a sidecar container.
+
+Kept dependency-free and importable as a bare script so it can run in a stock
+`python:*-alpine` image with only this directory mounted: the sidecar must not
+need quickstarted's own dependencies installed.
+
+Events are written to stdout as JSONL; the harness collects them with
+`docker logs` and merges them into the run trace.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+import threading
+import time
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+from proxy import EgressProxy
+
+
+class StdoutTrace:
+    """Duck-types Trace.add, emitting one JSON object per event."""
+
+    def add(self, type: str, **data) -> None:
+        sys.stdout.write(json.dumps({"ts": time.time(), "type": type, **data}) + "\n")
+        sys.stdout.flush()
+
+
+def _hosts(name: str) -> tuple[str, ...]:
+    raw = os.environ.get(name, "")
+    return tuple(h for h in (x.strip() for x in raw.split(",")) if h)
+
+
+def main() -> int:
+    port = int(os.environ.get("QUICKSTARTED_PROXY_PORT", "8080"))
+    proxy = EgressProxy(
+        network_allow=_hosts("QUICKSTARTED_NETWORK_ALLOW"),
+        docs_hosts=_hosts("QUICKSTARTED_DOCS_HOSTS"),
+        trace=StdoutTrace(),
+    )
+    # Bind on all interfaces inside the sidecar so the sandbox container can
+    # reach it; the container is on an internal network with no route out.
+    proxy.bind_host = "0.0.0.0"
+    proxy.start(port=port)
+    threading.Event().wait()
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())