ohbin 0.2.2__py3-none-any.whl

ohbin/_engine.py

@@ -0,0 +1,256 @@
+"""Download + SHA256-verify + cache a release binary on first use.
+
+Cache layout (`$XDG_CACHE_HOME` or `~/.cache`):
+
+    ohbin/<tool>/<version>/<binary>   ← extracted, chmod +x binary
+    ohbin/<tool>/<version>/.lock      ← flock during install
+
+Concurrent invocations on the same host coordinate via flock — first one
+downloads, the rest wait and reuse the same binary. POSIX-only (fcntl); a
+Windows path isn't shipped until someone needs it.
+"""
+
+from __future__ import annotations
+
+import fcntl
+import hashlib
+import http.client
+import os
+import shutil
+import tarfile
+import tempfile
+import urllib.error
+import urllib.request
+import zipfile
+from contextlib import contextmanager
+from pathlib import Path
+from typing import IO, TYPE_CHECKING
+from urllib.parse import unquote, urlparse
+
+from ohbin._crypto import decrypt_binary
+from ohbin._errors import OhbinError
+from ohbin._retry import retry, retryable_http_status
+
+if TYPE_CHECKING:
+    from collections.abc import Iterator
+
+_DOWNLOAD_TIMEOUT = 60  # seconds per read/connect attempt
+
+
+class ChecksumMismatchError(OhbinError):
+    """Raised when a downloaded asset's SHA256 doesn't match the pinned value."""
+
+
+class BinaryNotFoundError(OhbinError):
+    """Raised when the expected binary isn't present inside a downloaded archive."""
+
+
+class MissingPasswordError(OhbinError):
+    """Raised when an encrypted tool is run without a password from --password or pyproject."""
+
+
+class _TransientDownloadError(RuntimeError):
+    """Internal: a retryable download failure (network / timeout / truncated stream)."""
+
+
+def cache_root(tool: str, version: str) -> Path:
+    xdg = os.environ.get("XDG_CACHE_HOME")
+    base = Path(xdg) if xdg else Path.home() / ".cache"
+    return base / "ohbin" / tool / version
+
+
+@contextmanager
+def _exclusive_lock(lock_path: Path) -> Iterator[None]:
+    lock_path.parent.mkdir(parents=True, exist_ok=True)
+    fd = os.open(str(lock_path), os.O_CREAT | os.O_RDWR, 0o644)
+    try:
+        fcntl.flock(fd, fcntl.LOCK_EX)
+        yield
+    finally:
+        fcntl.flock(fd, fcntl.LOCK_UN)
+        os.close(fd)
+
+
+def _sha256_of(stream: IO[bytes]) -> str:
+    h = hashlib.sha256()
+    for chunk in iter(lambda: stream.read(1 << 20), b""):
+        h.update(chunk)
+    return h.hexdigest()
+
+
+def _download_once(url: str, dest: Path) -> None:
+    # Release URLs redirect to a signed CDN host; urllib follows redirects by
+    # default. Stream to disk so we don't buffer the archive in RAM.
+    request = urllib.request.Request(url, headers={"User-Agent": "ohbin"})
+    try:
+        with (
+            urllib.request.urlopen(request, timeout=_DOWNLOAD_TIMEOUT) as response,
+            dest.open("wb") as out,
+        ):
+            shutil.copyfileobj(response, out)
+    except urllib.error.HTTPError as exc:  # check first — subclass of URLError/OSError
+        if retryable_http_status(exc.code):
+            raise _TransientDownloadError(f"download {url} → HTTP {exc.code}") from exc
+        raise
+    except (OSError, http.client.IncompleteRead) as exc:
+        # URLError, ConnectionError, timeout, or a truncated/reset stream.
+        raise _TransientDownloadError(f"download {url} failed: {exc}") from exc
+
+
+def _download(url: str, dest: Path) -> None:
+    retry(lambda: _download_once(url, dest), op="download", retry_on=(_TransientDownloadError,))
+
+
+def sha256_of_url(url: str) -> str:
+    """Download a URL to a temp file and return its SHA256 (used by `add`)."""
+    fd, name = tempfile.mkstemp(prefix="ohbin-")
+    os.close(fd)
+    tmp = Path(name)
+    try:
+        _download(url, tmp)
+        with tmp.open("rb") as fh:
+            return _sha256_of(fh)
+    finally:
+        tmp.unlink(missing_ok=True)
+
+
+def text_of_url(url: str) -> str:
+    """Download a URL to a temp file and return its decoded text.
+
+    Used to read a gist file (e.g. the `ohbin.json` index) from its `raw_url` when
+    GitHub truncates the inline `content` returned by the gists API.
+    """
+    fd, name = tempfile.mkstemp(prefix="ohbin-")
+    os.close(fd)
+    tmp = Path(name)
+    try:
+        _download(url, tmp)
+        return tmp.read_text()
+    finally:
+        tmp.unlink(missing_ok=True)
+
+
+def _verify(path: Path, expected_sha256: str) -> None:
+    with path.open("rb") as fh:
+        actual = _sha256_of(fh)
+    if actual != expected_sha256:
+        msg = f"checksum mismatch for {path.name}: expected {expected_sha256}, got {actual}"
+        raise ChecksumMismatchError(msg)
+
+
+def _url_filename(url: str) -> str:
+    name = Path(unquote(urlparse(url).path)).name
+    return name or "download"
+
+
+def _finalize(staged: Path, dest: Path) -> None:
+    staged.chmod(0o755)
+    staged.replace(dest)  # atomic within the same filesystem
+
+
+def _extract_tar(archive: Path, binary: str, dest: Path) -> None:
+    with tarfile.open(archive, "r:*") as tf:  # r:* auto-detects gz/xz/bz2/plain
+        member = next(
+            (m for m in tf.getmembers() if m.isfile() and Path(m.name).name == binary),
+            None,
+        )
+        if member is None:
+            msg = f"binary {binary!r} not found in archive {archive.name}"
+            raise BinaryNotFoundError(msg)
+        with tempfile.TemporaryDirectory(dir=str(dest.parent)) as staging:
+            tf.extract(member, path=staging, filter="data")
+            _finalize(Path(staging) / member.name, dest)
+
+
+def _extract_zip(archive: Path, binary: str, dest: Path) -> None:
+    with zipfile.ZipFile(archive) as zf:
+        member = next(
+            (n for n in zf.namelist() if not n.endswith("/") and Path(n).name == binary),
+            None,
+        )
+        if member is None:
+            msg = f"binary {binary!r} not found in archive {archive.name}"
+            raise BinaryNotFoundError(msg)
+        with tempfile.TemporaryDirectory(dir=str(dest.parent)) as staging:
+            zf.extract(member, path=staging)
+            _finalize(Path(staging) / member, dest)
+
+
+def _place_raw(archive: Path, dest: Path) -> None:
+    # Asset is the bare executable (no archive wrapper).
+    with tempfile.TemporaryDirectory(dir=str(dest.parent)) as staging:
+        staged = Path(staging) / dest.name
+        shutil.copyfile(archive, staged)
+        _finalize(staged, dest)
+
+
+def _decrypt_into(archive: Path, dest: Path, *, password: str | None, binary_sha256: str | None) -> None:
+    # The downloaded file is base64 text (gzip+AES blob); decrypt to the raw binary.
+    if not password:
+        msg = "encrypted tool requires a password (pass --password or set 'password' in pyproject)"
+        raise MissingPasswordError(msg)
+    binary = decrypt_binary(archive.read_text(), password)
+    if binary_sha256 is not None:
+        actual = hashlib.sha256(binary).hexdigest()
+        if actual != binary_sha256:
+            msg = (
+                f"decrypted binary checksum mismatch: expected {binary_sha256}, got {actual} "
+                "(wrong password or corrupt blob)"
+            )
+            raise ChecksumMismatchError(msg)
+    with tempfile.TemporaryDirectory(dir=str(dest.parent)) as staging:
+        staged = Path(staging) / dest.name
+        staged.write_bytes(binary)
+        _finalize(staged, dest)
+
+
+def _extract(archive: Path, binary: str, dest: Path) -> None:
+    name = archive.name.lower()
+    if name.endswith((".tar.gz", ".tgz", ".tar.xz", ".tar.bz2", ".tar")):
+        _extract_tar(archive, binary, dest)
+    elif name.endswith(".zip"):
+        _extract_zip(archive, binary, dest)
+    else:
+        _place_raw(archive, dest)
+
+
+def ensure_from(
+    *,
+    tool: str,
+    version: str,
+    binary: str,
+    url: str,
+    sha256: str,
+    encrypted: bool = False,
+    password: str | None = None,
+    binary_sha256: str | None = None,
+) -> Path:
+    """Return the cached binary path, downloading + verifying it on first use.
+
+    For encrypted tools the downloaded file is a base64 gzip+AES blob: `sha256`
+    verifies the ciphertext (tamper-evidence on the gist), then it's decrypted with
+    `password` and the plaintext checked against `binary_sha256`. The decrypted
+    binary is cached like any other — first run decrypts, the rest reuse it.
+    """
+    cache = cache_root(tool, version)
+    target = cache / binary
+    if target.is_file() and os.access(target, os.X_OK):
+        return target
+
+    with _exclusive_lock(cache / ".lock"):
+        # Another process may have finished while we waited on the lock.
+        if target.is_file() and os.access(target, os.X_OK):
+            return target
+
+        archive = cache / _url_filename(url)
+        try:
+            _download(url, archive)
+            _verify(archive, sha256)
+            if encrypted:
+                _decrypt_into(archive, target, password=password, binary_sha256=binary_sha256)
+            else:
+                _extract(archive, binary, target)
+        finally:
+            archive.unlink(missing_ok=True)
+
+    return target

ohbin/_errors.py

@@ -0,0 +1,12 @@
+"""Common base for all expected, user-facing ohbin failures.
+
+The CLI catches `OhbinError` and prints `error: <message>` without a stack trace;
+anything else propagates as a real (unexpected) crash. Library callers can likewise
+`except ohbin.OhbinError` to handle every anticipated failure in one place.
+"""
+
+from __future__ import annotations
+
+
+class OhbinError(RuntimeError):
+    """Base class for anticipated ohbin failures (bad manifest, checksum, decrypt, …)."""

ohbin/_gist.py

@@ -0,0 +1,234 @@
+"""`ohbin publish-gist` / `add-gist`: ship encrypted private binaries through gists.
+
+A gist holds one base64'd `gzip+AES` blob per platform (`<name>-<platform>.enc.b64`)
+plus an `ohbin.json` index that maps each platform to its file and the *decrypted*
+binary's SHA256. `publish-gist` produces and uploads them; `add-gist` reads the index,
+resolves each file's immutable raw URL, pins the ciphertext SHA, and writes the tool
+into pyproject. Gists are secret (link-gated), the binary is encrypted, and the
+password lives elsewhere — a leaked link alone is useless.
+
+All GitHub access goes through the `gh` CLI (`gh api`), reusing the user's auth.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import shutil
+import subprocess
+from pathlib import Path
+from typing import TypedDict
+from urllib.parse import urlparse
+
+from ohbin._add import local_pyproject, write_tool
+from ohbin._crypto import encrypt_binary
+from ohbin._engine import sha256_of_url, text_of_url
+from ohbin._errors import OhbinError
+from ohbin._platform import current_platform
+from ohbin._types import AssetEntry, ToolConfig
+
+_OHBIN_INDEX = "ohbin.json"
+_INDEX_VERSION = 1
+
+
+class GistAssetInfo(TypedDict):
+    """Per-platform entry in a gist's `ohbin.json` index."""
+
+    file: str
+    binary_sha256: str
+
+
+class GistMeta(TypedDict):
+    """The `ohbin.json` index stored alongside the encrypted blobs in a gist."""
+
+    ohbin_gist: int
+    name: str
+    binary: str
+    encrypted: bool
+    assets: dict[str, GistAssetInfo]
+
+
+class GistError(OhbinError):
+    """Raised when a gist can't be published or resolved (gh failure, malformed index)."""
+
+
+def _require_gh() -> str:
+    path = shutil.which("gh")
+    if path is None:
+        msg = "gh CLI not found on PATH — required to publish/resolve gists"
+        raise GistError(msg)
+    return path
+
+
+def _gh_api(method: str, path: str, body: dict[str, object] | None = None) -> dict[str, object]:
+    cmd = [_require_gh(), "api", "-X", method, path]
+    payload: bytes | None = None
+    if body is not None:
+        cmd += ["--input", "-"]
+        payload = json.dumps(body).encode()
+    proc = subprocess.run(cmd, input=payload, capture_output=True, check=False)
+    if proc.returncode != 0:
+        detail = proc.stderr.decode(errors="replace").strip()
+        msg = f"gh api {method} {path} failed: {detail or 'unknown error'}"
+        raise GistError(msg)
+    parsed: dict[str, object] = json.loads(proc.stdout)
+    return parsed
+
+
+def _parse_gist_id(ref: str) -> str:
+    ref = ref.strip()
+    if "/" not in ref and ":" not in ref:
+        return ref  # already a bare id
+    parts = [p for p in urlparse(ref).path.split("/") if p]
+    if "raw" in parts:  # gist raw URL: .../<user>/<id>/raw/<sha>/<file>
+        parts = parts[: parts.index("raw")]
+    if not parts:
+        msg = f"cannot parse a gist id from {ref!r}"
+        raise GistError(msg)
+    return parts[-1]
+
+
+def _file_entry(files: dict[str, object], name: str) -> dict[str, object]:
+    entry = files.get(name)
+    if not isinstance(entry, dict):
+        msg = f"gist is missing file {name!r}"
+        raise GistError(msg)
+    return entry
+
+
+def _load_index(files: dict[str, object]) -> GistMeta:
+    entry = _file_entry(files, _OHBIN_INDEX)
+    raw = entry.get("content")
+    # GitHub truncates the inline `content` of *every* file in the gists API once
+    # any single file exceeds ~1MB — so a tiny index sitting next to a multi-MB
+    # encrypted blob comes back empty (`truncated: true`, `content: ""`). Fall back
+    # to the file's raw_url, which always serves the complete bytes.
+    if entry.get("truncated") or not isinstance(raw, str) or not raw:
+        raw_url = entry.get("raw_url")
+        if not isinstance(raw_url, str):
+            msg = f"{_OHBIN_INDEX} content is truncated and has no raw_url to fall back to"
+            raise GistError(msg)
+        raw = text_of_url(raw_url)
+    meta: GistMeta = json.loads(raw)
+    return meta
+
+
+def publish_gist(
+    *,
+    name: str,
+    binary: str,
+    binary_path: Path,
+    password: str,
+    platform_key: str | None,
+    gist_ref: str | None,
+    description: str,
+) -> str:
+    """Encrypt `binary_path` for one platform and POST/PATCH it into a secret gist.
+
+    With `gist_ref` the blob is added to an existing gist (another platform for the
+    same tool); otherwise a fresh secret gist is created. Returns the gist's html_url.
+    """
+    plat = platform_key or current_platform().key
+    data = binary_path.read_bytes()
+    blob = encrypt_binary(data, password)
+    fname = f"{name}-{plat}.enc.b64"
+
+    if gist_ref is not None:
+        gid = _parse_gist_id(gist_ref)
+        existing = _gh_api("GET", f"gists/{gid}")
+        files = existing.get("files")
+        meta = _load_index(files) if isinstance(files, dict) else _new_index(name, binary)
+    else:
+        meta = _new_index(name, binary)
+
+    meta["assets"][plat] = {"file": fname, "binary_sha256": hashlib.sha256(data).hexdigest()}
+    body: dict[str, object] = {
+        "public": False,
+        "files": {
+            fname: {"content": blob},
+            _OHBIN_INDEX: {"content": json.dumps(meta, indent=2, sort_keys=True)},
+        },
+    }
+    if gist_ref is not None:
+        result = _gh_api("PATCH", f"gists/{_parse_gist_id(gist_ref)}", body)
+    else:
+        body["description"] = description
+        result = _gh_api("POST", "gists", body)
+
+    html_url = result.get("html_url")
+    if not isinstance(html_url, str):
+        msg = "gist created but GitHub returned no html_url"
+        raise GistError(msg)
+    return html_url
+
+
+def _new_index(name: str, binary: str) -> GistMeta:
+    return {
+        "ohbin_gist": _INDEX_VERSION,
+        "name": name,
+        "binary": binary,
+        "encrypted": True,
+        "assets": {},
+    }
+
+
+def _snapshot(gist: dict[str, object], gid: str) -> str:
+    """Short id for the gist's current revision — used as the cache-busting version."""
+    history = gist.get("history")
+    if isinstance(history, list) and history:
+        first = history[0]
+        if isinstance(first, dict):
+            version = first.get("version")
+            if isinstance(version, str):
+                return version[:12]
+    return gid[:12]
+
+
+def resolve_gist(gist_ref: str, *, password: str | None = None) -> tuple[str, ToolConfig]:
+    """Read a gist's `ohbin.json` index and build a ToolConfig (pins ciphertext SHAs)."""
+    gid = _parse_gist_id(gist_ref)
+    gist = _gh_api("GET", f"gists/{gid}")
+    files = gist.get("files")
+    if not isinstance(files, dict):
+        msg = f"gist {gid} has no files"
+        raise GistError(msg)
+    meta = _load_index(files)
+
+    assets: dict[str, AssetEntry] = {}
+    for plat, info in meta["assets"].items():
+        raw_url = _file_entry(files, info["file"]).get("raw_url")
+        if not isinstance(raw_url, str):
+            msg = f"gist file {info['file']!r} has no raw_url"
+            raise GistError(msg)
+        # Hash the bytes GitHub actually serves (immune to any newline normalisation).
+        assets[plat] = AssetEntry(
+            url=raw_url,
+            sha256=sha256_of_url(raw_url),
+            binary_sha256=info["binary_sha256"],
+        )
+
+    cfg = ToolConfig(
+        repo=f"gist:{gid}",
+        version=f"gist-{_snapshot(gist, gid)}",
+        binary=meta["binary"],
+        assets=assets,
+        encrypted=True,
+    )
+    if password is not None:
+        cfg["password"] = password
+    return meta["name"], cfg
+
+
+def add_gist_tool(
+    *,
+    gist_ref: str,
+    name: str | None,
+    password: str | None,
+    pyproject: Path | None,
+) -> tuple[str, Path]:
+    """Resolve a gist and write its (encrypted) tool entry into pyproject."""
+    resolved_name, cfg = resolve_gist(gist_ref, password=password)
+    cmd_name = name or resolved_name
+    target = local_pyproject(pyproject)
+    write_tool(target, cmd_name, cfg)
+    return cmd_name, target

ohbin/_github.py

@@ -0,0 +1,125 @@
+"""Resolve a GitHub release + its assets — via the `gh` CLI when present, else the REST API.
+
+Failures are split two ways: a clean **404** means "this tag doesn't exist"
+(we try `v`-prefixed then bare, then give up — never retried), while anything
+else — network error, timeout, rate-limit, 5xx, a non-404 `gh` failure — is
+**transient** and retried with backoff. Conflating the two is exactly what made
+a flaky network look like "release not found".
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import subprocess
+import urllib.error
+import urllib.request
+from typing import Any, NamedTuple
+
+from ohbin._errors import OhbinError
+from ohbin._retry import retry, retryable_http_status
+
+_API = "https://api.github.com"
+_TIMEOUT = 30  # seconds per attempt
+
+
+class ReleaseNotFoundError(OhbinError):
+    """Raised when no release matches the given repo/version (a real 404)."""
+
+
+class _TransientError(RuntimeError):
+    """Internal: a retryable GitHub fetch failure (network / timeout / 5xx / rate-limit / gh error)."""
+
+
+class Asset(NamedTuple):
+    name: str
+    url: str
+    digest: str | None  # GitHub asset digest like "sha256:..." (may be absent)
+
+
+class Release(NamedTuple):
+    tag: str
+    assets: list[Asset]
+
+
+def _candidate_paths(repo: str, version: str | None) -> list[str]:
+    if version is None:
+        return [f"repos/{repo}/releases/latest"]
+    # Accept both `v1.2.3` and `1.2.3` tag conventions.
+    return [
+        f"repos/{repo}/releases/tags/v{version}",
+        f"repos/{repo}/releases/tags/{version}",
+    ]
+
+
+def _fetch_gh_one(path: str) -> dict[str, Any] | None:
+    """Return parsed JSON, None on a clean 404, or raise `_TransientError` otherwise."""
+    try:
+        proc = subprocess.run(  # fixed argv, `gh` resolved from PATH
+            ["gh", "api", path],
+            capture_output=True,
+            text=True,
+            check=False,
+            timeout=_TIMEOUT,
+        )
+    except subprocess.TimeoutExpired as exc:
+        raise _TransientError(f"gh api {path} timed out after {_TIMEOUT}s") from exc
+    if proc.returncode == 0:
+        parsed: dict[str, Any] = json.loads(proc.stdout)
+        return parsed
+    stderr = proc.stderr.strip()
+    if "http 404" in stderr.lower() or "not found" in stderr.lower():
+        return None
+    raise _TransientError(f"gh api {path} failed: {stderr or 'unknown error'}")
+
+
+def _fetch_http_one(path: str) -> dict[str, Any] | None:
+    headers = {
+        "Accept": "application/vnd.github+json",
+        "User-Agent": "ohbin",
+        "X-GitHub-Api-Version": "2022-11-28",
+    }
+    token = os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN")
+    if token:
+        headers["Authorization"] = f"Bearer {token}"
+    request = urllib.request.Request(f"{_API}/{path}", headers=headers)  # fixed https host
+    try:
+        with urllib.request.urlopen(request, timeout=_TIMEOUT) as response:
+            parsed: dict[str, Any] = json.loads(response.read())
+            return parsed
+    except urllib.error.HTTPError as exc:
+        if exc.code == 404:
+            return None
+        if retryable_http_status(exc.code):
+            raise _TransientError(f"GET {path} → HTTP {exc.code}") from exc
+        raise
+    except urllib.error.URLError as exc:  # network / DNS / timeout
+        raise _TransientError(f"GET {path} failed: {exc.reason}") from exc
+
+
+def _fetch_raw(repo: str, version: str | None) -> dict[str, Any]:
+    fetch_one = _fetch_gh_one if shutil.which("gh") else _fetch_http_one
+    candidates = _candidate_paths(repo, version)
+
+    def attempt() -> dict[str, Any] | None:
+        for path in candidates:
+            data = fetch_one(path)
+            if data is not None:
+                return data
+        return None  # every candidate was a clean 404 → genuinely not found
+
+    raw = retry(attempt, op=f"fetch release for {repo}", retry_on=(_TransientError,))
+    if raw is None:
+        msg = f"could not resolve a release for {repo} (tried: {', '.join(candidates)})"
+        raise ReleaseNotFoundError(msg)
+    return raw
+
+
+def fetch_release(repo: str, version: str | None) -> Release:
+    raw = _fetch_raw(repo, version)
+    assets = [
+        Asset(name=str(a["name"]), url=str(a["browser_download_url"]), digest=a.get("digest"))
+        for a in raw.get("assets", [])
+    ]
+    return Release(tag=str(raw["tag_name"]), assets=assets)