quickstarted 0.2.0__py3-none-any.whl

quickstarted/docs.py

@@ -0,0 +1,263 @@
+"""The documentation client: caching, politeness, and affordance policy.
+
+Three things live here that a benchmark cannot do without.
+
+**Reproducibility.** Responses are cached by content hash, so a rerun reads
+the same bytes the first run did. When a refresh sees different bytes, that is
+recorded: "the docs changed under us" is a finding, not an inconvenience.
+
+**Politeness.** Benchmarking fifty projects means fetching from fifty
+companies who did not ask to be measured. A truthful User-Agent, one request
+per host at a time, and robots.txt honoured by default are the minimum for
+doing this at scale without being a nuisance.
+
+**Affordance policy.** Whether a project ships `llms.txt` is a checklist item
+anybody can curl, and scoring it would be exactly the proxy metric this tool
+exists to replace. What nobody can currently answer is whether the file
+*helps*. So affordances are never scored, only recorded, and they can be
+withheld from the agent: run the same journey with `all` and with `none`, and
+the difference in pass rate is a measurement of the affordance itself.
+
+The prompt is identical under both conditions. Only availability changes,
+which is what keeps the comparison honest.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import threading
+import time
+import urllib.error
+import urllib.robotparser
+from dataclasses import dataclass
+from pathlib import Path
+from urllib.parse import urljoin, urlparse
+
+from . import transport
+
+#: URL shapes that exist for the benefit of language models rather than people.
+AFFORDANCE_FILES = ("llms.txt", "llms-full.txt")
+AFFORDANCE_POLICIES = ("all", "none")
+
+
+@dataclass(frozen=True)
+class FetchResult:
+    url: str
+    status: int
+    content_type: str
+    text: str
+    from_cache: bool = False
+    content_hash: str = ""
+    changed: bool = False
+    blocked_reason: str = ""
+
+    @property
+    def ok(self) -> bool:
+        return not self.blocked_reason and 200 <= self.status < 300
+
+
+@dataclass
+class Affordance:
+    url: str
+    present: bool
+    status: int = 0
+    bytes: int = 0
+    note: str = ""
+
+
+def is_affordance_url(url: str) -> bool:
+    path = urlparse(url).path.lower()
+    name = path.rsplit("/", 1)[-1]
+    return name in AFFORDANCE_FILES or path.endswith(".md")
+
+
+class DocsClient:
+    def __init__(
+        self,
+        cache_dir: str | None = None,
+        rate_limit_seconds: float = 1.0,
+        respect_robots: bool = True,
+        affordances: str = "all",
+        refresh: bool = False,
+        offline: bool = False,
+        timeout: int = 30,
+    ):
+        if affordances not in AFFORDANCE_POLICIES:
+            raise ValueError(
+                f"affordances must be one of {AFFORDANCE_POLICIES}, got {affordances!r}"
+            )
+        self.cache_dir = Path(cache_dir) if cache_dir else None
+        self.rate_limit_seconds = rate_limit_seconds
+        self.respect_robots = respect_robots
+        self.affordances = affordances
+        self.refresh = refresh
+        self.offline = offline
+        self.timeout = timeout
+        self._last_request: dict[str, float] = {}
+        self._robots: dict[str, urllib.robotparser.RobotFileParser | None] = {}
+        self._lock = threading.Lock()
+
+    # -- cache ----------------------------------------------------------
+    def _cache_path(self, url: str) -> Path | None:
+        if not self.cache_dir:
+            return None
+        digest = hashlib.sha256(url.encode("utf-8")).hexdigest()
+        return self.cache_dir / digest[:2] / f"{digest}.json"
+
+    def _read_cache(self, url: str) -> dict | None:
+        path = self._cache_path(url)
+        if not path or not path.is_file():
+            return None
+        try:
+            return json.loads(path.read_text(encoding="utf-8"))
+        except ValueError:
+            return None
+
+    def _write_cache(self, url: str, payload: dict) -> None:
+        path = self._cache_path(url)
+        if not path:
+            return
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(json.dumps(payload), encoding="utf-8")
+
+    # -- politeness -----------------------------------------------------
+    def _throttle(self, host: str) -> None:
+        if self.rate_limit_seconds <= 0:
+            return
+        with self._lock:
+            last = self._last_request.get(host, 0.0)
+            wait = self.rate_limit_seconds - (time.monotonic() - last)
+            if wait > 0:
+                time.sleep(wait)
+            self._last_request[host] = time.monotonic()
+
+    def robots_allows(self, url: str) -> bool:
+        if not self.respect_robots:
+            return True
+        parsed = urlparse(url)
+        origin = f"{parsed.scheme}://{parsed.netloc}"
+        with self._lock:
+            known = origin in self._robots
+            parser = self._robots.get(origin)
+        if not known:
+            parser = urllib.robotparser.RobotFileParser()
+            try:
+                response = transport.http_get(
+                    urljoin(origin, "/robots.txt"), timeout=self.timeout
+                )
+                if 200 <= response.status < 300:
+                    parser.parse(response.text.splitlines())
+                else:
+                    parser = None
+            except Exception:
+                # No robots.txt, or unreachable: nothing to disallow.
+                parser = None
+            with self._lock:
+                self._robots[origin] = parser
+        if parser is None:
+            return True
+        return parser.can_fetch(transport.USER_AGENT, url)
+
+    # -- fetching -------------------------------------------------------
+    def get(self, url: str) -> FetchResult:
+        if self.affordances == "none" and is_affordance_url(url):
+            return FetchResult(
+                url, 0, "", "", blocked_reason="affordance_withheld"
+            )
+
+        cached = self._read_cache(url)
+        if cached and not self.refresh:
+            return FetchResult(
+                url=url,
+                status=cached["status"],
+                content_type=cached["content_type"],
+                text=cached["text"],
+                from_cache=True,
+                content_hash=cached["content_hash"],
+            )
+
+        if self.offline:
+            return FetchResult(url, 0, "", "", blocked_reason="offline_cache_miss")
+
+        if not self.robots_allows(url):
+            return FetchResult(url, 0, "", "", blocked_reason="robots_disallowed")
+
+        host = urlparse(url).hostname or ""
+        self._throttle(host)
+        response = transport.http_get(url, timeout=self.timeout)
+        text = response.text
+        if "html" in (response.content_type or "").lower():
+            text = transport.html_to_text(text)
+        content_hash = hashlib.sha256(text.encode("utf-8")).hexdigest()
+        changed = bool(cached and cached.get("content_hash") != content_hash)
+        self._write_cache(
+            url,
+            {
+                "url": url,
+                "status": response.status,
+                "content_type": response.content_type,
+                "text": text,
+                "content_hash": content_hash,
+                "fetched_at": time.time(),
+            },
+        )
+        return FetchResult(
+            url=url,
+            status=response.status,
+            content_type=response.content_type,
+            text=text,
+            content_hash=content_hash,
+            changed=changed,
+        )
+
+    # -- affordance probing ---------------------------------------------
+    def probe(self, entrypoint: str) -> dict[str, Affordance]:
+        """Record which machine-facing affordances exist. Never scored.
+
+        Presence is context for a human reading a failure, and a variable for
+        the ablation. It is not a grade: a 1.8 MB llms-full.txt is 'present'
+        and may still be useless to an agent with a context window.
+        """
+        parsed = urlparse(entrypoint)
+        origin = f"{parsed.scheme}://{parsed.netloc}"
+        found: dict[str, Affordance] = {}
+        candidates = {name: urljoin(origin + "/", name) for name in AFFORDANCE_FILES}
+        page = entrypoint.split("?")[0].rstrip("/")
+        if page and not page.endswith(".md"):
+            candidates["page.md"] = page + ".md"
+        for label, url in candidates.items():
+            try:
+                self._throttle(urlparse(url).hostname or "")
+                response = transport.http_get(url, timeout=self.timeout)
+            except urllib.error.HTTPError as exc:
+                found[label] = Affordance(url, False, status=exc.code)
+                continue
+            except Exception as exc:
+                found[label] = Affordance(url, False, note=str(exc)[:120])
+                continue
+            body = response.text or ""
+            # A soft 200 that serves the site's HTML shell is not an affordance.
+            looks_like_html = "html" in (response.content_type or "").lower()
+            present = 200 <= response.status < 300 and bool(body) and not looks_like_html
+            found[label] = Affordance(
+                url=url,
+                present=present,
+                status=response.status,
+                bytes=len(body.encode("utf-8")),
+                note="served HTML, not a machine-readable file" if looks_like_html else "",
+            )
+        return found
+
+
+def affordance_summary(found: dict[str, Affordance]) -> dict[str, dict]:
+    return {
+        label: {
+            "url": a.url,
+            "present": a.present,
+            "status": a.status,
+            "bytes": a.bytes,
+            "note": a.note,
+        }
+        for label, a in found.items()
+    }

quickstarted/exec/__init__.py

@@ -0,0 +1,89 @@
+"""Execution backends and the policy for choosing one.
+
+`auto` prefers enforcement: Docker when a daemon is reachable, Seatbelt on
+macOS, and local only as a last resort. Choosing an unenforced backend is
+allowed but never silent.
+"""
+
+from __future__ import annotations
+
+import platform
+
+from . import docker as _docker
+from . import seatbelt as _seatbelt
+from .base import CommandResult, Executor, ExecutorError, ProcessExecutor, truncate
+from .docker import DockerExecutor
+from .local import LocalExecutor
+from .seatbelt import SeatbeltExecutor
+
+BACKENDS = ("auto", "docker", "seatbelt", "local")
+
+
+def available_backends() -> list[str]:
+    found = []
+    if _docker.available():
+        found.append("docker")
+    if _seatbelt.available():
+        found.append("seatbelt")
+    found.append("local")
+    return found
+
+
+def resolve_backend(requested: str = "auto") -> str:
+    if requested not in BACKENDS:
+        raise ExecutorError(
+            f"unknown backend {requested!r}; choose from {', '.join(BACKENDS)}"
+        )
+    if requested != "auto":
+        return requested
+    for candidate in ("docker", "seatbelt"):
+        if candidate in available_backends():
+            return candidate
+    return "local"
+
+
+def make_executor(
+    backend: str,
+    keep: bool = False,
+    proxy_url: str | None = None,
+    network_allow=(),
+    docs_hosts=(),
+    image: str | None = None,
+    trace=None,
+) -> Executor:
+    if backend == "docker":
+        return DockerExecutor(
+            keep=keep,
+            network_allow=network_allow,
+            docs_hosts=docs_hosts,
+            image=image or _docker.DEFAULT_IMAGE,
+            trace=trace,
+        )
+    if backend == "seatbelt":
+        return SeatbeltExecutor(keep=keep, proxy_url=proxy_url)
+    if backend == "local":
+        return LocalExecutor(keep=keep, proxy_url=proxy_url)
+    raise ExecutorError(f"unknown backend {backend!r}")
+
+
+def needs_host_proxy(backend: str) -> bool:
+    """Docker runs its own proxy sidecar; process backends need one on the host."""
+    return backend in ("seatbelt", "local")
+
+
+__all__ = [
+    "BACKENDS",
+    "CommandResult",
+    "DockerExecutor",
+    "Executor",
+    "ExecutorError",
+    "LocalExecutor",
+    "ProcessExecutor",
+    "SeatbeltExecutor",
+    "available_backends",
+    "make_executor",
+    "needs_host_proxy",
+    "platform",
+    "resolve_backend",
+    "truncate",
+]

quickstarted/exec/base.py

@@ -0,0 +1,131 @@
+"""Execution backends for agent-issued commands.
+
+An executor owns the workspace a journey runs in and decides what a command
+can reach. Backends differ in one respect that matters: whether the journey's
+network policy is *enforced* by the operating system or merely *requested*.
+
+`enforced = False` means an agent that ignores the proxy environment can talk
+to any host it likes, and every docs page it reads that way is invisible to
+the trace. Since attribution of a failure to a documentation page is the whole
+product, unenforced backends are for developing journeys against code you
+trust, never for benchmarking third-party projects.
+"""
+
+from __future__ import annotations
+
+import os
+import shutil
+import subprocess
+import tempfile
+import time
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Protocol
+
+_TRUNCATION_NOTE = "\n[... output truncated by quickstarted ...]\n"
+
+
+@dataclass(frozen=True)
+class CommandResult:
+    exit_code: int
+    output: str
+    duration: float
+    timed_out: bool = False
+
+
+def truncate(text: str, limit: int) -> str:
+    if len(text) <= limit:
+        return text
+    head = text[: limit // 2]
+    tail = text[-(limit // 2) :]
+    return head + _TRUNCATION_NOTE + tail
+
+
+class ExecutorError(RuntimeError):
+    """Raised when a backend cannot be started on this machine."""
+
+
+class Executor(Protocol):
+    name: str
+    enforced: bool
+    root: Path
+
+    def run(
+        self, command: str, timeout: int, max_output_chars: int = 20_000
+    ) -> CommandResult:
+        ...
+
+    def cleanup(self) -> None:
+        ...
+
+
+class ProcessExecutor:
+    """Shared plumbing for backends that run commands as local processes."""
+
+    name = "process"
+    enforced = False
+
+    def __init__(self, keep: bool = False, proxy_url: str | None = None):
+        self.root = Path(tempfile.mkdtemp(prefix="quickstarted-"))
+        (self.root / "tmp").mkdir()
+        self.keep = keep
+        self.proxy_url = proxy_url
+
+    def env(self) -> dict[str, str]:
+        env = {
+            "PATH": os.environ.get("PATH", "/usr/bin:/bin"),
+            "HOME": str(self.root),
+            "TMPDIR": str(self.root / "tmp"),
+            "LANG": os.environ.get("LANG", "en_US.UTF-8"),
+            "LC_ALL": os.environ.get("LC_ALL", os.environ.get("LANG", "en_US.UTF-8")),
+            "TERM": "dumb",
+            "NO_COLOR": "1",
+        }
+        if self.proxy_url:
+            # Both cases: tools are inconsistent about which they read.
+            for key in ("HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY"):
+                env[key] = self.proxy_url
+                env[key.lower()] = self.proxy_url
+            # Loopback must not go through the proxy. Journeys routinely start a
+            # server and then ask it a question, and without this the request is
+            # sent to the proxy, which refuses it as an unlisted host.
+            for key in ("NO_PROXY", "no_proxy"):
+                env[key] = "localhost,127.0.0.1,::1"
+        return env
+
+    def argv(self, command: str) -> list[str]:
+        return ["bash", "-c", command]
+
+    def run(
+        self, command: str, timeout: int, max_output_chars: int = 20_000
+    ) -> CommandResult:
+        start = time.monotonic()
+        try:
+            proc = subprocess.run(
+                self.argv(command),
+                cwd=self.root,
+                env=self.env(),
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+                timeout=timeout,
+                text=True,
+                errors="replace",
+            )
+            output, exit_code, timed_out = proc.stdout or "", proc.returncode, False
+        except subprocess.TimeoutExpired as exc:
+            raw = exc.output or b""
+            if isinstance(raw, bytes):
+                raw = raw.decode("utf-8", errors="replace")
+            output = raw + f"\n[command timed out after {timeout}s]"
+            exit_code, timed_out = 124, True
+        duration = time.monotonic() - start
+        return CommandResult(
+            exit_code=exit_code,
+            output=truncate(output, max_output_chars),
+            duration=duration,
+            timed_out=timed_out,
+        )
+
+    def cleanup(self) -> None:
+        if not self.keep:
+            shutil.rmtree(self.root, ignore_errors=True)

quickstarted/exec/docker.py

@@ -0,0 +1,216 @@
+"""Enforced execution in containers: the portable backend, and what CI uses.
+
+Topology, which is the whole reason this is enforced rather than requested:
+
+    [sandbox container]  --internal network-->  [proxy sidecar] --bridge--> internet
+
+The sandbox container is attached *only* to a Docker network created with
+`--internal`, which has no route off the host. The single reachable address is
+the sidecar, which is attached to both that network and a normal bridge. So a
+command cannot leave the machine except through the harness's policy, whether
+or not it honours the proxy environment variables.
+
+The sidecar writes its events to stdout; they are collected at teardown and
+merged into the run trace.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import subprocess
+import tempfile
+import time
+import uuid
+from pathlib import Path
+
+from ..net.proxy import DEFAULT_NETWORK_ALLOW
+from .base import CommandResult, ExecutorError, truncate
+
+DEFAULT_IMAGE = "python:3.12-slim"
+PROXY_IMAGE = "python:3.12-alpine"
+PROXY_PORT = 8080
+_NET_DIR = Path(__file__).resolve().parent.parent / "net"
+
+
+def available() -> bool:
+    if not shutil.which("docker"):
+        return False
+    try:
+        return (
+            subprocess.run(
+                ["docker", "info"],
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+                timeout=20,
+            ).returncode
+            == 0
+        )
+    except (OSError, subprocess.SubprocessError):
+        return False
+
+
+def _docker(args: list[str], timeout: int = 120) -> subprocess.CompletedProcess:
+    return subprocess.run(
+        ["docker", *args],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        text=True,
+        errors="replace",
+        timeout=timeout,
+    )
+
+
+class DockerExecutor:
+    name = "docker"
+    enforced = True
+
+    def __init__(
+        self,
+        keep: bool = False,
+        network_allow=DEFAULT_NETWORK_ALLOW,
+        docs_hosts=(),
+        image: str = DEFAULT_IMAGE,
+        trace=None,
+    ):
+        if not available():
+            raise ExecutorError(
+                "docker backend requires a running Docker daemon "
+                "(`docker info` must succeed)"
+            )
+        self.keep = keep
+        self.image = image
+        self.trace = trace
+        self.root = Path(tempfile.mkdtemp(prefix="quickstarted-"))
+        (self.root / "tmp").mkdir()
+        # mkdtemp gives 0700 owned by the invoking user. Where the daemon remaps
+        # container root to another uid (rootless Docker, userns-remap, some CI
+        # runners), that uid cannot write the bind mount and every journey dies
+        # in setup. Widen the throwaway workspace instead of forcing the
+        # container to a non-root user, because a quickstart is allowed to say
+        # `apt-get install`.
+        os.chmod(self.root, 0o777)
+        os.chmod(self.root / "tmp", 0o777)
+        token = uuid.uuid4().hex[:10]
+        self.network = f"quickstarted-net-{token}"
+        self.proxy_name = f"quickstarted-proxy-{token}"
+        self.container = f"quickstarted-sbx-{token}"
+        self._started = False
+        self._start(tuple(network_allow), tuple(docs_hosts))
+
+    # -- lifecycle ------------------------------------------------------
+    def _start(self, network_allow, docs_hosts) -> None:
+        created = _docker(["network", "create", "--internal", self.network])
+        if created.returncode != 0:
+            raise ExecutorError(f"could not create docker network: {created.stdout}")
+        self._started = True
+
+        proxy = _docker(
+            [
+                "run", "-d", "--name", self.proxy_name,
+                "--network", self.network,
+                "-v", f"{_NET_DIR}:/opt/quickstarted:ro",
+                "-e", f"QUICKSTARTED_PROXY_PORT={PROXY_PORT}",
+                "-e", "QUICKSTARTED_NETWORK_ALLOW=" + ",".join(network_allow),
+                "-e", "QUICKSTARTED_DOCS_HOSTS=" + ",".join(docs_hosts),
+                PROXY_IMAGE,
+                "python", "/opt/quickstarted/proxy_main.py",
+            ]
+        )
+        if proxy.returncode != 0:
+            self.cleanup()
+            raise ExecutorError(f"could not start proxy sidecar: {proxy.stdout}")
+
+        # Give the sidecar, and only the sidecar, a route to the internet.
+        bridged = _docker(["network", "connect", "bridge", self.proxy_name])
+        if bridged.returncode != 0:
+            self.cleanup()
+            raise ExecutorError(f"could not bridge proxy sidecar: {bridged.stdout}")
+
+        proxy_url = f"http://{self.proxy_name}:{PROXY_PORT}"
+        env_args: list[str] = []
+        for key in ("HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY"):
+            env_args += ["-e", f"{key}={proxy_url}", "-e", f"{key.lower()}={proxy_url}"]
+        # Loopback stays off the proxy; see ProcessExecutor.env for why.
+        for key in ("NO_PROXY", "no_proxy"):
+            env_args += ["-e", f"{key}=localhost,127.0.0.1,::1"]
+
+        sandbox = _docker(
+            [
+                "run", "-d", "--name", self.container,
+                "--network", self.network,
+                "--cap-drop", "ALL",
+                "--security-opt", "no-new-privileges",
+                "--pids-limit", "512",
+                "-v", f"{self.root}:/workspace",
+                "-w", "/workspace",
+                "-e", "HOME=/workspace",
+                "-e", "TMPDIR=/workspace/tmp",
+                "-e", "NO_COLOR=1",
+                "-e", "TERM=dumb",
+                *env_args,
+                self.image, "sleep", "infinity",
+            ]
+        )
+        if sandbox.returncode != 0:
+            self.cleanup()
+            raise ExecutorError(f"could not start sandbox container: {sandbox.stdout}")
+
+    def run(
+        self, command: str, timeout: int, max_output_chars: int = 20_000
+    ) -> CommandResult:
+        start = time.monotonic()
+        try:
+            proc = _docker(
+                ["exec", "-w", "/workspace", self.container, "bash", "-c", command],
+                timeout=timeout,
+            )
+            output, exit_code, timed_out = proc.stdout or "", proc.returncode, False
+        except subprocess.TimeoutExpired as exc:
+            raw = exc.output or ""
+            if isinstance(raw, bytes):
+                raw = raw.decode("utf-8", errors="replace")
+            # The exec is detached from our process; stop it inside the container.
+            _docker(["exec", self.container, "pkill", "-9", "-f", "bash -c"], timeout=30)
+            output = raw + f"\n[command timed out after {timeout}s]"
+            exit_code, timed_out = 124, True
+        return CommandResult(
+            exit_code=exit_code,
+            output=truncate(output, max_output_chars),
+            duration=time.monotonic() - start,
+            timed_out=timed_out,
+        )
+
+    def collect_proxy_events(self) -> int:
+        """Merge sidecar events into the trace. Returns how many were merged."""
+        if self.trace is None or not self._started:
+            return 0
+        logs = _docker(["logs", self.proxy_name], timeout=60)
+        if logs.returncode != 0:
+            return 0
+        merged = 0
+        for line in logs.stdout.splitlines():
+            line = line.strip()
+            if not line.startswith("{"):
+                continue
+            try:
+                event = json.loads(line)
+            except ValueError:
+                continue
+            kind = event.pop("type", "egress")
+            event.pop("ts", None)
+            self.trace.add(kind, **event)
+            merged += 1
+        return merged
+
+    def cleanup(self) -> None:
+        if not self._started:
+            return
+        self.collect_proxy_events()
+        for name in (self.container, self.proxy_name):
+            _docker(["rm", "-f", name], timeout=60)
+        _docker(["network", "rm", self.network], timeout=60)
+        self._started = False
+        if not self.keep:
+            shutil.rmtree(self.root, ignore_errors=True)