zjcode 0.0.1__py3-none-any.whl

deepagents_code/managed_tools.py

@@ -0,0 +1,642 @@
+"""Auto-install pinned upstream binaries for optional tools.
+
+Today this only manages `ripgrep`. The SDK shells out to `rg` via `PATH`,
+so installing into `~/.deepagents/bin/` and prepending that directory to
+`os.environ["PATH"]` is sufficient — no SDK change required.
+
+The pinned `RIPGREP_VERSION` and `RIPGREP_ASSETS` table is the single
+source of truth for what gets downloaded and verified. When bumping the
+version, refresh both the version and the SHA-256 entries together.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+import sys
+from pathlib import Path
+from typing import TYPE_CHECKING, Literal
+
+from deepagents_code._env_vars import OFFLINE, RIPGREP_INSTALLER, is_env_truthy
+
+if TYPE_CHECKING:
+    import tarfile
+    import zipfile
+
+logger = logging.getLogger(__name__)
+
+RIPGREP_VERSION = "14.1.1"
+"""Pinned upstream ripgrep release. Bump alongside `RIPGREP_ASSETS`."""
+
+_RELEASE_URL_PREFIX = (
+    "https://github.com/BurntSushi/ripgrep/releases/download/" + RIPGREP_VERSION
+)
+
+RIPGREP_ASSETS: dict[tuple[str, str], tuple[str, str]] = {
+    ("darwin", "arm64"): (
+        f"ripgrep-{RIPGREP_VERSION}-aarch64-apple-darwin.tar.gz",
+        "24ad76777745fbff131c8fbc466742b011f925bfa4fffa2ded6def23b5b937be",
+    ),
+    ("darwin", "x86_64"): (
+        f"ripgrep-{RIPGREP_VERSION}-x86_64-apple-darwin.tar.gz",
+        "fc87e78f7cb3fea12d69072e7ef3b21509754717b746368fd40d88963630e2b3",
+    ),
+    ("linux", "arm64"): (
+        f"ripgrep-{RIPGREP_VERSION}-aarch64-unknown-linux-gnu.tar.gz",
+        "c827481c4ff4ea10c9dc7a4022c8de5db34a5737cb74484d62eb94a95841ab2f",
+    ),
+    ("linux", "x86_64"): (
+        f"ripgrep-{RIPGREP_VERSION}-x86_64-unknown-linux-musl.tar.gz",
+        "4cf9f2741e6c465ffdb7c26f38056a59e2a2544b51f7cc128ef28337eeae4d8e",
+    ),
+    # Windows on ARM runs x64 binaries via emulation; upstream does not
+    # ship an arm64-windows build for ripgrep, so both Windows entries
+    # point at the same x86_64 MSVC asset.
+    ("win32", "arm64"): (
+        f"ripgrep-{RIPGREP_VERSION}-x86_64-pc-windows-msvc.zip",
+        "d0f534024c42afd6cb4d38907c25cd2b249b79bbe6cc1dbee8e3e37c2b6e25a1",
+    ),
+    ("win32", "x86_64"): (
+        f"ripgrep-{RIPGREP_VERSION}-x86_64-pc-windows-msvc.zip",
+        "d0f534024c42afd6cb4d38907c25cd2b249b79bbe6cc1dbee8e3e37c2b6e25a1",
+    ),
+}
+"""`(sys.platform, normalized arch) -> (asset filename, sha256 hex)`."""
+
+BIN_DIR: Path = Path.home() / ".zjcode" / "bin"
+"""Directory holding managed binaries. Prepended to `PATH` on startup."""
+
+_DOWNLOAD_TIMEOUT_SECONDS = 120
+_VERSION_CHECK_TIMEOUT_SECONDS = 5
+_DOWNLOAD_CHUNK_BYTES = 1 << 16
+_ARCH_ALIASES = {
+    "aarch64": "arm64",
+    "arm64": "arm64",
+    "amd64": "x86_64",
+    "x86_64": "x86_64",
+    "x64": "x86_64",
+}
+
+
+class ChecksumMismatchError(Exception):
+    """Raised when a downloaded archive fails SHA-256 verification.
+
+    Distinct from generic install failure so callers can surface a loud,
+    user-visible notice — a checksum mismatch is a supply-chain anomaly
+    (CDN poisoning, MITM, tampered mirror) and must not be silently
+    treated like "you're offline".
+    """
+
+
+UnavailableReason = Literal["unsupported", "artifact_not_found"]
+"""Stable reason token for logging and telemetry."""
+
+
+class ManagedToolUnavailableError(Exception):
+    """Raised when no managed helper binary is available for this system.
+
+    Distinct from transient network/download failures so callers can tell users
+    whether retrying can help or whether they need a different install path.
+    """
+
+    def __init__(
+        self, *, tool: Literal["ripgrep"], reason: UnavailableReason, message: str
+    ) -> None:
+        """Initialize the unavailable-tool error.
+
+        Args:
+            tool: Managed helper name.
+            reason: Stable reason token for logging and telemetry.
+            message: User-facing remediation message.
+        """
+        super().__init__(message)
+        self.tool = tool
+        self.reason = reason
+        self.message = message
+
+
+def _normalized_arch() -> str | None:
+    """Return a normalized arch key matching `RIPGREP_ASSETS`.
+
+    Returns `None` for unsupported architectures (e.g. 32-bit, ppc, s390x).
+    """
+    import platform
+
+    raw = platform.machine().lower()
+    return _ARCH_ALIASES.get(raw)
+
+
+def _unsupported_ripgrep_error(
+    platform_name: str, arch: str | None
+) -> ManagedToolUnavailableError:
+    """Return a clear unsupported-platform error for managed ripgrep."""
+    target = platform_name if arch is None else f"{platform_name}/{arch}"
+    return ManagedToolUnavailableError(
+        tool="ripgrep",
+        reason="unsupported",
+        message=(
+            f"Managed ripgrep is not available for this system ({target}). "
+            "Install ripgrep manually, or set DEEPAGENTS_CODE_RIPGREP_INSTALLER=system."
+        ),
+    )
+
+
+def _artifact_not_found_error(
+    platform_name: str, arch: str
+) -> ManagedToolUnavailableError:
+    """Return a clear missing-artifact error for managed ripgrep."""
+    return ManagedToolUnavailableError(
+        tool="ripgrep",
+        reason="artifact_not_found",
+        message=(
+            f"Managed ripgrep artifact for {platform_name}/{arch} was not found "
+            f"in pinned ripgrep {RIPGREP_VERSION}. Install ripgrep manually, or "
+            "try a newer dcode version."
+        ),
+    )
+
+
+def managed_rg_path() -> Path:
+    """Return the managed ripgrep binary path (`.exe` on Windows)."""
+    name = "rg.exe" if sys.platform == "win32" else "rg"
+    return BIN_DIR / name
+
+
+def is_offline() -> bool:
+    """Return whether managed-tool downloads are disabled via env var."""
+    return is_env_truthy(OFFLINE)
+
+
+RipgrepInstaller = Literal["managed", "system"]
+"""The two recognized ripgrep installer modes."""
+
+INSTALLER_MANAGED: RipgrepInstaller = "managed"
+"""Default installer mode: fetch the pinned, checksummed upstream binary."""
+
+INSTALLER_SYSTEM: RipgrepInstaller = "system"
+"""Installer mode that defers ripgrep to the system package manager / `PATH`."""
+
+
+def ripgrep_installer() -> RipgrepInstaller:
+    """Return the configured ripgrep installer mode.
+
+    Reads `RIPGREP_INSTALLER` and normalizes it to `INSTALLER_MANAGED` or
+    `INSTALLER_SYSTEM`, falling back to `INSTALLER_MANAGED` for unset or
+    unrecognized values. The `strip().lower()` normalization must stay in
+    sync with the `case` block in `scripts/install.sh` so both layers agree
+    on the parsed mode.
+    """
+    raw = os.environ.get(RIPGREP_INSTALLER, "").strip().lower()
+    if raw == INSTALLER_SYSTEM:
+        return INSTALLER_SYSTEM
+    if raw and raw != INSTALLER_MANAGED:
+        logger.warning(
+            "Unrecognized %s=%r; expected %r or %r. Defaulting to %r.",
+            RIPGREP_INSTALLER,
+            raw,
+            INSTALLER_MANAGED,
+            INSTALLER_SYSTEM,
+            INSTALLER_MANAGED,
+        )
+    return INSTALLER_MANAGED
+
+
+def prefers_system_ripgrep() -> bool:
+    """Return whether the user opted into the `system` ripgrep installer.
+
+    In `system` mode ripgrep is provisioned by the OS package manager or an
+    existing `PATH` entry rather than the managed download.
+    """
+    return ripgrep_installer() == INSTALLER_SYSTEM
+
+
+def prepend_managed_bin_to_path() -> None:
+    """Idempotently prepend `BIN_DIR` to `os.environ["PATH"]`.
+
+    Safe to call on every startup. Callers do not need to check whether
+    the directory exists — adding a non-existent directory to `PATH` is
+    harmless and matches behavior of common version managers.
+    """
+    bin_str = str(BIN_DIR)
+    current = os.environ.get("PATH", "")
+    parts = current.split(os.pathsep) if current else []
+    if parts and parts[0] == bin_str:
+        return
+    parts = [bin_str, *(p for p in parts if p != bin_str)]
+    os.environ["PATH"] = os.pathsep.join(parts)
+
+
+def _path_without_managed_bin() -> str | None:
+    """Return `PATH` with `BIN_DIR` removed."""
+    current = os.environ.get("PATH")
+    if not current:
+        return None
+
+    managed_dir = BIN_DIR.resolve()
+    parts = [
+        part
+        for part in current.split(os.pathsep)
+        if not part or Path(part).resolve() != managed_dir
+    ]
+    return os.pathsep.join(parts)
+
+
+def _managed_binary_is_current(binary: Path) -> bool:
+    """Return whether the on-disk managed `rg` matches `RIPGREP_VERSION`.
+
+    Returns `False` on any concrete failure (`OSError`, non-zero exit,
+    empty stdout, version mismatch) so a corrupted or wrong-arch
+    binary written by a previously crashed install gets re-fetched. Only
+    `TimeoutExpired` "falls open" — that case suggests a sandboxed
+    subprocess rather than a broken binary.
+    """
+    import subprocess  # noqa: S404  # fixed-argv probe of a managed binary
+
+    try:
+        result = subprocess.run(  # noqa: S603  # fixed argv, managed path
+            [str(binary), "--version"],
+            check=False,
+            capture_output=True,
+            text=True,
+            timeout=_VERSION_CHECK_TIMEOUT_SECONDS,
+        )
+    except subprocess.TimeoutExpired:
+        logger.debug("rg --version probe timed out for %s; assuming current", binary)
+        return True
+    except OSError:
+        logger.debug(
+            "rg --version probe failed for %s; treating as stale",
+            binary,
+            exc_info=True,
+        )
+        return False
+    if result.returncode != 0:
+        logger.debug(
+            "rg --version exited %d for %s; treating as stale",
+            result.returncode,
+            binary,
+        )
+        return False
+    first_line = (result.stdout or "").splitlines()[:1]
+    if not first_line:
+        return False
+    return RIPGREP_VERSION in first_line[0]
+
+
+def _download_to(url: str, dest: Path) -> None:
+    """Stream `url` to `dest`, bounded by a wall-clock deadline.
+
+    `urlopen(timeout=...)` only bounds per-operation socket waits, so a
+    slow trickle of bytes from a flaky peer could otherwise stretch the
+    transfer well beyond the configured timeout. The chunked read here
+    enforces an end-to-end deadline, checked between chunk reads.
+
+    A non-200 response is rejected before any bytes are written: a proxy
+    interstitial or an unfollowed redirect returned with a non-200 status
+    must not be streamed to disk and then surface downstream as a
+    misleading SHA-256 failure (which reads as a supply-chain anomaly).
+    `urlopen` already raises `HTTPError` for 4xx/5xx, so this guards the
+    residual 2xx/3xx cases.
+
+    Raises:
+        TimeoutError: When total transfer time exceeds the deadline.
+        urllib.error.URLError: When the response status is not 200.
+    """
+    import time
+    import urllib.error
+    import urllib.request
+
+    deadline = time.monotonic() + _DOWNLOAD_TIMEOUT_SECONDS
+    with (
+        urllib.request.urlopen(url, timeout=_DOWNLOAD_TIMEOUT_SECONDS) as resp,  # noqa: S310  # fixed https GitHub release URL
+        dest.open("wb") as fh,
+    ):
+        status = getattr(resp, "status", None)
+        if status is not None and status != 200:  # noqa: PLR2004  # HTTP 200 OK
+            msg = f"Unexpected HTTP {status} response fetching {url}"
+            raise urllib.error.URLError(msg)
+        while True:
+            if time.monotonic() > deadline:
+                msg = (
+                    f"Download of {url} exceeded {_DOWNLOAD_TIMEOUT_SECONDS}s deadline"
+                )
+                raise TimeoutError(msg)
+            chunk = resp.read(_DOWNLOAD_CHUNK_BYTES)
+            if not chunk:
+                break
+            fh.write(chunk)
+
+
+def _verify_sha256(path: Path, expected_hex: str) -> None:
+    """Verify `path` matches `expected_hex`.
+
+    Raises:
+        ChecksumMismatchError: When the SHA-256 of `path` differs from
+            `expected_hex`.
+    """
+    import hashlib
+
+    digest = hashlib.sha256()
+    with path.open("rb") as fh:
+        for chunk in iter(lambda: fh.read(1 << 20), b""):
+            digest.update(chunk)
+    actual = digest.hexdigest()
+    if actual != expected_hex:
+        msg = (
+            f"Checksum mismatch for {path.name}: expected {expected_hex}, got {actual}"
+        )
+        raise ChecksumMismatchError(msg)
+
+
+def _validate_legacy_tar_member(member: tarfile.TarInfo, extract_root: Path) -> None:
+    """Reject tar members that cannot be safely extracted without filters.
+
+    Raises:
+        tarfile.TarError: If a member would extract outside `extract_root`
+            or uses a tar entry type this fallback does not support.
+    """
+    import tarfile
+
+    target = extract_root / member.name
+    try:
+        target.resolve().relative_to(extract_root.resolve())
+    except ValueError as exc:
+        msg = f"Refusing to extract unsafe tar member {member.name!r}"
+        raise tarfile.TarError(msg) from exc
+
+    if not (member.isfile() or member.isdir()):
+        msg = f"Refusing to extract unsupported tar member {member.name!r}"
+        raise tarfile.TarError(msg)
+
+
+def _extract_tar_data(tf: tarfile.TarFile, extract_root: Path) -> None:
+    """Extract a tar archive with `data` filtering when available.
+
+    Python versions before the PEP 706 backport (3.11.0-3.11.3) lack the
+    `filter` keyword on `extractall`, and this package supports those patch
+    versions. The fallback validates the pinned release archive before
+    using the legacy API.
+
+    Raises:
+        TypeError: Re-raised when `extractall` rejects a non-`filter`
+            keyword (i.e. an unrelated `TypeError` we should not swallow).
+    """
+    try:
+        tf.extractall(extract_root, filter="data")
+    except TypeError as exc:
+        if "filter" not in str(exc):
+            raise
+        members = tf.getmembers()
+        for member in members:
+            _validate_legacy_tar_member(member, extract_root)
+        tf.extractall(extract_root, members=members)  # noqa: S202  # validated above
+
+
+def _extract_rg(archive: Path, extract_root: Path) -> Path:
+    """Extract `archive` and locate the `rg` binary inside.
+
+    Handles both `.tar.gz` and `.zip` archives. Release archives nest the
+    binary under `ripgrep-<ver>-<triple>/`, so we walk the tree to find it
+    rather than hard-coding the prefix. Malformed archives or unsafe
+    members propagate `tarfile.TarError` / `zipfile.BadZipFile`.
+
+    Returns:
+        Absolute path to the extracted `rg` (or `rg.exe`) binary.
+
+    Raises:
+        FileNotFoundError: When the archive does not contain an `rg` binary.
+    """
+    import tarfile
+    import zipfile
+
+    if archive.suffix == ".zip":
+        with zipfile.ZipFile(archive) as zf:
+            _extract_zip_validated(zf, extract_root)
+    else:
+        with tarfile.open(archive, mode="r:*") as tf:
+            _extract_tar_data(tf, extract_root)
+
+    target_name = "rg.exe" if sys.platform == "win32" else "rg"
+    for path in extract_root.rglob(target_name):
+        if path.is_file():
+            return path
+    msg = f"Could not find {target_name} inside {archive.name}"
+    raise FileNotFoundError(msg)
+
+
+def _extract_zip_validated(zf: zipfile.ZipFile, extract_root: Path) -> None:
+    """Extract a zip archive after validating each member's path.
+
+    `ZipFile.extractall` does sanitize absolute paths and parent-relative
+    components on modern Python, but defense-in-depth here keeps the
+    SHA-256-verified archive from being the only line of defense against
+    a zip-slip variant in a future upstream archive.
+
+    Raises:
+        zipfile.BadZipFile: If a member would extract outside `extract_root`.
+    """
+    import zipfile
+
+    extract_root.mkdir(parents=True, exist_ok=True)
+    root = extract_root.resolve()
+    for member in zf.infolist():
+        target = (extract_root / member.filename).resolve()
+        try:
+            target.relative_to(root)
+        except ValueError as exc:
+            msg = f"Refusing to extract unsafe zip member {member.filename!r}"
+            raise zipfile.BadZipFile(msg) from exc
+    zf.extractall(extract_root)  # noqa: S202  # validated above
+
+
+def _install_ripgrep_sync(asset: str, sha256: str) -> Path:
+    """Download, verify, extract, and install ripgrep atomically.
+
+    Staging happens *inside* `BIN_DIR` so the final rename is on the same
+    filesystem and therefore atomic on POSIX. Windows keeps replacing the
+    user-facing `rg.exe` directly because symlink support varies by developer
+    mode and policy. POSIX installs use a versioned real binary plus a relative
+    `rg` symlink so moving or bind-mounting `~/.deepagents` does not bake in
+    the original home directory path. `_verify_sha256` propagates
+    `ChecksumMismatchError` to abort install before any move.
+
+    Returns:
+        Absolute path to the installed `rg` entrypoint.
+    """
+    import os
+    import tempfile
+
+    BIN_DIR.mkdir(parents=True, exist_ok=True)
+    url = f"{_RELEASE_URL_PREFIX}/{asset}"
+    with tempfile.TemporaryDirectory(prefix=".deepagents-rg-", dir=BIN_DIR) as tmp_str:
+        tmp = Path(tmp_str)
+        archive = tmp / asset
+        _download_to(url, archive)
+        _verify_sha256(archive, sha256)
+        extracted = _extract_rg(archive, tmp / "unpacked")
+        if sys.platform != "win32":
+            extracted.chmod(0o755)
+        dest = managed_rg_path()
+        if sys.platform == "win32":
+            extracted.replace(dest)
+            return dest
+
+        real = BIN_DIR / f"rg-{RIPGREP_VERSION}"
+        extracted.replace(real)
+        link = tmp / "rg-link"
+        link.symlink_to(os.path.relpath(real, start=BIN_DIR))
+        link.replace(dest)
+        return dest
+
+
+async def ensure_ripgrep() -> Path | None:
+    """Ensure a usable `rg` binary is available, installing if necessary.
+
+    Resolution order:
+
+    1. If the `system` installer is selected, return a non-managed `rg`
+        found on `PATH`, or `None` when only the managed binary is present.
+    2. If a managed `rg` exists *and* matches `RIPGREP_VERSION`, return it.
+    3. Otherwise, if a system `rg` is on `PATH` and no managed binary
+        exists, return its resolved path. This is gated on the *absence*
+        of a managed binary: once a managed `rg` exists, the pinned
+        version always wins, so a stale managed binary is re-fetched
+        rather than deferring to a system `rg` and the resolved version
+        stays deterministic.
+    4. If offline, return `None` so callers fall back to the existing
+        notification + slow path.
+    5. If no managed asset matches the platform/arch, return a non-managed
+        `rg` on `PATH` when one exists; otherwise raise
+        `ManagedToolUnavailableError` so callers can explain that retrying will
+        not help.
+    6. Otherwise download → SHA-256 verify → extract → install →
+        prepend `BIN_DIR` to `PATH` → return the installed path. On a
+        checksum mismatch, raises `ChecksumMismatchError` so callers can
+        surface a loud notice. On a 404, raises `ManagedToolUnavailableError`;
+        other failures log and return `None`.
+
+    A stale managed binary is never proactively deleted. The atomic
+    replace in `_install_ripgrep_sync` overwrites it on success, and on
+    failure the user is strictly better off keeping the older copy than
+    being left with no `rg` at all.
+
+    Returns:
+        Path to a usable `rg` binary, or `None` when one could not be
+        located or installed.
+    """
+    import asyncio
+    import platform
+    import shutil
+    import tarfile
+    import urllib.error
+    import zipfile
+
+    managed = managed_rg_path()
+    managed_exists = managed.exists() or managed.is_symlink()
+
+    def non_managed_rg() -> Path | None:
+        system_rg = shutil.which("rg", path=_path_without_managed_bin())
+        if system_rg is not None:
+            return Path(system_rg)
+        return None
+
+    if prefers_system_ripgrep():
+        system_rg_path = non_managed_rg()
+        if system_rg_path is not None:
+            return system_rg_path
+        logger.debug(
+            "Skipping managed ripgrep download: %s=%s",
+            RIPGREP_INSTALLER,
+            INSTALLER_SYSTEM,
+        )
+        return None
+
+    if managed_exists and _managed_binary_is_current(managed):
+        return managed
+
+    if not managed_exists:
+        system_rg = shutil.which("rg")
+        if system_rg is not None:
+            return Path(system_rg)
+
+    if is_offline():
+        logger.debug("Skipping ripgrep install: %s is set", OFFLINE)
+        return None
+    if sys.platform == "android":
+        logger.debug("Skipping ripgrep install: unsupported platform 'android'")
+        system_rg_path = non_managed_rg()
+        if system_rg_path is not None:
+            return system_rg_path
+        error = _unsupported_ripgrep_error(sys.platform, None)
+        raise error
+
+    arch = _normalized_arch()
+    if arch is None:
+        logger.debug(
+            "Skipping ripgrep install: unsupported arch %r", platform.machine()
+        )
+        system_rg_path = non_managed_rg()
+        if system_rg_path is not None:
+            return system_rg_path
+        error = _unsupported_ripgrep_error(sys.platform, platform.machine())
+        raise error
+
+    asset_entry = RIPGREP_ASSETS.get((sys.platform, arch))
+    if asset_entry is None:
+        logger.debug(
+            "Skipping ripgrep install: no asset for (%s, %s)", sys.platform, arch
+        )
+        system_rg_path = non_managed_rg()
+        if system_rg_path is not None:
+            return system_rg_path
+        error = _unsupported_ripgrep_error(sys.platform, arch)
+        raise error
+    asset, sha256 = asset_entry
+
+    if managed_exists:
+        logger.info(
+            "Managed ripgrep at %s is stale; replacing with %s",
+            managed,
+            RIPGREP_VERSION,
+        )
+
+    try:
+        # `_install_ripgrep_sync` atomically replaces the destination on
+        # success, so we deliberately leave any stale binary in place
+        # until the verified replacement is ready. A failed download must
+        # not strand the user with no `rg` at all.
+        installed = await asyncio.to_thread(_install_ripgrep_sync, asset, sha256)
+    except urllib.error.HTTPError as exc:
+        if exc.code == 404:  # noqa: PLR2004  # HTTP 404 Not Found
+            logger.warning(
+                "Managed ripgrep artifact was not found: %s/%s", sys.platform, arch
+            )
+            error = _artifact_not_found_error(sys.platform, arch)
+            raise error from exc
+        logger.warning(
+            "Could not download ripgrep from %s", _RELEASE_URL_PREFIX, exc_info=True
+        )
+        return None
+    except (urllib.error.URLError, TimeoutError):
+        logger.warning(
+            "Could not download ripgrep from %s", _RELEASE_URL_PREFIX, exc_info=True
+        )
+        return None
+    except (tarfile.TarError, zipfile.BadZipFile, FileNotFoundError) as exc:
+        logger.exception(
+            "ripgrep install failed: archive error (%s)", type(exc).__name__
+        )
+        return None
+    except PermissionError:
+        logger.exception(
+            "ripgrep install failed: cannot write to %s — check permissions", BIN_DIR
+        )
+        return None
+    except OSError as exc:
+        logger.exception(
+            "ripgrep install failed: %s (errno=%s)", type(exc).__name__, exc.errno
+        )
+        return None
+    else:
+        prepend_managed_bin_to_path()
+        return installed