forest-cli 0.1.0__py3-none-any.whl

forest/rclone.py

@@ -0,0 +1,561 @@
+"""Thin active transport wrapper around the external ``rclone`` binary.
+
+This module provides the low-level transport primitives (connection-string
+builder + ``copy``/``copyto``/``lsjson``/``check``) plus the duck-typed
+:class:`RemoteFile` / :class:`TransferResult` data shapes, and the
+:class:`RcloneTransport` adapter + :func:`get_transport` factory used by
+``cli.py``.
+
+forest.yaml is the single source of truth for remotes: every connstring is
+built on the fly and passed as ONE argv element (no shell, no ``rclone.conf``).
+Real-name files at rest on every backend; no md5/content-addressing anywhere.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+import time
+from contextlib import suppress
+from dataclasses import dataclass, field
+from pathlib import Path
+from urllib.parse import urlsplit
+
+from forest import metrics
+from forest.config import RemoteConfig
+from forest.logger import get_logger
+from forest.resilience import CircuitBreaker, CircuitOpenError, backoff_delay, is_transient, transfer_retries
+
+_log = get_logger("forest.rclone")
+
+RCLONE_INSTALL_MSG = "rclone is required. Install with: conda install -c conda-forge rclone"
+
+_LOCAL_METADATA_EXCLUDES = ["--exclude", "._*", "--exclude", ".DS_Store"]
+_APPLEDOUBLE_MAGIC = b"\x00\x05\x16\x07"
+
+
+class RcloneError(Exception):
+    """Raised when an rclone subprocess exits non-zero.
+
+    Carries the captured ``stderr`` text and the process exit ``code`` so
+    callers can surface the failure (e.g. as ``TransferResult.errors``).
+    """
+
+    def __init__(self, stderr: str, code: int) -> None:
+        self.stderr = stderr
+        self.code = code
+        super().__init__(f"rclone failed (exit {code}): {stderr}".rstrip())
+
+
+@dataclass
+class TransferResult:
+    """Outcome of a push or pull operation (duck-typed across cli.py)."""
+
+    files_transferred: int
+    bytes_transferred: int
+    errors: list[str] = field(default_factory=list)
+
+    @property
+    def success(self) -> bool:
+        return len(self.errors) == 0
+
+
+@dataclass
+class RemoteFile:
+    """A file listing entry on a remote.
+
+    ``key`` is PREFIX-RELATIVE and may contain ``/`` (nested dirs preserved).
+    """
+
+    key: str
+    size: int
+    last_modified: str | None = None
+
+
+def is_available() -> bool:
+    """Return True when the ``rclone`` binary is resolvable on ``$PATH``."""
+    return shutil.which("rclone") is not None
+
+
+# --------------------------------------------------------------------------- #
+# Connection-string builder
+# --------------------------------------------------------------------------- #
+
+
+def build_remote(remote_cfg: RemoteConfig) -> str:
+    """Build an on-the-fly rclone connection-string ROOT from a RemoteConfig.
+
+    Dispatches on ``remote_cfg.remote_type`` (``s3`` / ``local`` / ``sftp``).
+    The returned string is a single argv element; never shell-quote it here.
+    """
+    remote_type = remote_cfg.remote_type
+    if remote_type == "local":
+        return _build_local(remote_cfg)
+    if remote_type == "s3":
+        return _build_s3(remote_cfg)
+    if remote_type == "sftp":
+        return _build_sftp(remote_cfg)
+    raise ValueError(f"Unsupported remote type '{remote_type}' for URL: {remote_cfg.url}")
+
+
+def _build_local(remote_cfg: RemoteConfig) -> str:
+    # Plain absolute path, used as-is. Drop trailing slashes so joining
+    # ``{root}/{key}`` never yields ``//``; spaces are preserved verbatim.
+    stripped = remote_cfg.url.rstrip("/")
+    return stripped or "/"
+
+
+def _build_s3(remote_cfg: RemoteConfig) -> str:
+    rest = remote_cfg.url[len("s3://") :].rstrip("/")  # "bucket" or "bucket/prefix"
+    opts = ["provider=AWS", "env_auth=true"]
+    if remote_cfg.profile is not None:
+        opts.append(f"profile={remote_cfg.profile}")
+    if remote_cfg.region is not None:
+        opts.append(f"region={remote_cfg.region}")
+    if remote_cfg.endpoint is not None:
+        opts.append(f"endpoint={remote_cfg.endpoint}")
+    opts.append("no_check_bucket=true")
+    return ":s3," + ",".join(opts) + ":" + rest
+
+
+def _parse_sftp_url(url: str) -> tuple[str | None, str | None, int, str | None]:
+    """Parse an sftp URL into ``(user, host, port, path)``.
+
+    Accepts ``sftp://host[:port]/path`` (credentials rejected) and scp-style
+    ``user@host:[port:]/path`` forms.
+    """
+    if url.startswith("sftp://"):
+        parsed = urlsplit(url)
+        if parsed.username or parsed.password:
+            raise ValueError("Cannot parse sftp url with embedded credentials; use key_file and known_hosts.")
+        return None, parsed.hostname, parsed.port or 22, parsed.path
+
+    match = re.match(r"^(?P<user>[^@]+)@(?P<rest>.+)$", url)
+    if not match:
+        raise ValueError(f"Cannot parse sftp url (expected user@host:/path or sftp://host/path): {url}")
+    user = match.group("user")
+    host_part, _, path_part = match.group("rest").partition(":")
+    if not path_part:
+        raise ValueError(f"Cannot parse sftp url (missing remote path): {url}")
+    port = 22
+    if not path_part.startswith("/") and ":" in path_part:
+        maybe_port, _, rest_path = path_part.partition(":")
+        if maybe_port.isdigit():
+            port = int(maybe_port)
+            path_part = rest_path
+    return user, host_part, port, path_part
+
+
+def _build_sftp(remote_cfg: RemoteConfig) -> str:
+    missing = [
+        name
+        for name, value in (("key_file", remote_cfg.key_file), ("known_hosts", remote_cfg.known_hosts))
+        if not value
+    ]
+    if missing:
+        missing_text = ", ".join(missing)
+        raise ValueError(
+            "sftp remote requires key_file and known_hosts (set them in forest.yaml); "
+            f"missing: {missing_text}; "
+            "refusing to build a connstring without them (no agent/default-key fallback, "
+            "no host-key-check bypass)."
+        )
+
+    user, host_part, port, path_part = _parse_sftp_url(remote_cfg.url)
+
+    if not host_part:
+        raise ValueError(f"Cannot parse sftp url (missing host): {remote_cfg.url}")
+    if not path_part or not path_part.startswith("/"):
+        raise ValueError(f"Cannot parse sftp url (missing remote path): {remote_cfg.url}")
+
+    opts = [
+        f"host={host_part}",
+        f"key_file={remote_cfg.key_file}",
+        f"known_hosts_file={remote_cfg.known_hosts}",
+        f"port={port}",
+    ]
+    if user is not None:
+        opts.insert(1, f"user={user}")
+    return f":sftp,{','.join(opts)}:{path_part}"
+
+
+# --------------------------------------------------------------------------- #
+# Subprocess helpers
+# --------------------------------------------------------------------------- #
+
+
+def _progress_flags(quiet: bool) -> list[str]:
+    if quiet:
+        return ["-q"]
+    # --stats-log-level NOTICE forces rclone to emit the final transfer stats
+    # line to stderr even for fast/sub-second transfers (the default INFO level
+    # is below rclone's console threshold, so small uploads would otherwise show
+    # no stats at all). `-q` in quiet mode suppresses these entirely.
+    return ["--stats", "1s", "--stats-one-line", "--stats-log-level", "NOTICE"]
+
+
+def _run(args: list[str], *, echo_stderr: bool) -> subprocess.CompletedProcess[str]:
+    """Run ``rclone <args>``, raising :class:`RcloneError` on non-zero exit.
+
+    stderr is always captured (so errors carry text); when ``echo_stderr`` it
+    is also forwarded to the real stderr so rclone's stats/errors stay visible.
+    Transient failures (exit code 5 / network-blip stderr) are retried with
+    exponential backoff up to ``FOREST_TRANSFER_RETRIES`` extra attempts; only
+    the final attempt's stderr is echoed so retries do not duplicate output.
+    """
+    op = args[0] if args else "rclone"
+    max_retries = transfer_retries()
+    attempt = 0
+    start = time.monotonic()
+    while True:
+        proc = subprocess.run(["rclone", *args], capture_output=True, text=True)
+        if proc.returncode == 0:
+            break
+        if attempt < max_retries and is_transient(proc.returncode, proc.stderr or ""):
+            attempt += 1
+            delay = backoff_delay(attempt)
+            _log.warning(
+                "transient rclone failure; retrying",
+                extra={"op": op, "exit_code": proc.returncode, "attempt": attempt, "delay_s": delay},
+            )
+            metrics.emit("rclone.retries", 1, tags={"op": op})
+            time.sleep(delay)
+            continue
+        if echo_stderr and proc.stderr:
+            sys.stderr.write(proc.stderr)
+        _log.error("rclone failed", extra={"op": op, "exit_code": proc.returncode, "attempts": attempt + 1})
+        metrics.emit("rclone.failures", 1, tags={"op": op})
+        raise RcloneError(proc.stderr or "", proc.returncode)
+    if echo_stderr and proc.stderr:
+        sys.stderr.write(proc.stderr)
+    duration = round(time.monotonic() - start, 3)
+    _log.debug("rclone ok", extra={"op": op, "attempts": attempt + 1, "duration_s": duration})
+    metrics.emit("rclone.duration_seconds", duration, unit="s", tags={"op": op})
+    return proc
+
+
+def _is_local_dst(dst: str) -> bool:
+    # rclone connstrings for non-local backends start with ':' (e.g. ':s3,', ':sftp,').
+    return not dst.startswith(":")
+
+
+def _remove_local_metadata_sidecars(written_files: list[Path], *, root: Path | None = None) -> None:
+    """Remove local macOS metadata sidecars scoped to written file paths.
+
+    AppleDouble ``._*`` files are deleted only when the 4-byte AppleDouble magic
+    matches, preserving real data files whose names happen to start with ``._``.
+    ``.DS_Store`` has no data payload in forest outputs, so it is deleted
+    unconditionally, but only from directories touched by this transfer.
+    """
+    candidates: set[Path] = set()
+    for path in written_files:
+        path = Path(path)
+        candidates.add(path.parent / f"._{path.name}")
+        candidates.add(path.parent / ".DS_Store")
+        if root is not None:
+            candidates.update(_ancestor_metadata_candidates(path, root))
+
+    for path in candidates:
+        if _is_metadata_sidecar(path):
+            with suppress(OSError):
+                path.unlink()
+
+
+def _ancestor_metadata_candidates(path: Path, root: Path) -> set[Path]:
+    """Return metadata candidates for ancestor dirs from ``root`` to ``path``."""
+    candidates: set[Path] = set()
+    with suppress(ValueError):
+        rel = path.relative_to(root)
+        current = root
+        candidates.add(current / ".DS_Store")
+        for part in rel.parts[:-1]:
+            child = current / part
+            candidates.add(current / f"._{part}")
+            candidates.add(child / ".DS_Store")
+            current = child
+    return candidates
+
+
+def _is_metadata_sidecar(path: Path) -> bool:
+    if not path.is_file():
+        return False
+    if path.name == ".DS_Store":
+        return True
+    if not path.name.startswith("._"):
+        return False
+    with suppress(OSError):
+        return path.read_bytes()[:4] == _APPLEDOUBLE_MAGIC
+    return False
+
+
+# --------------------------------------------------------------------------- #
+# Transfer primitives
+# --------------------------------------------------------------------------- #
+
+
+def copy(files: list[Path], local_base: Path, dst: str, *, quiet: bool) -> None:
+    """Copy ``files`` to ``dst`` preserving paths RELATIVE to ``local_base``.
+
+    Runs ``rclone copy --files-from-raw <tmp list> <local_base> <dst>``. The
+    list entries are each file's path relative to ``local_base``; rclone
+    preserves that relative layout under ``dst`` and CANNOT rename (use
+    :func:`copyto`). ``--files-from-raw`` (not ``--files-from``) is required so
+    real filenames with leading ``#``/``;`` or surrounding whitespace are taken
+    verbatim instead of being parsed as comments/trimmed and silently dropped.
+    """
+    if not files:
+        return
+
+    local_base = Path(local_base)
+    rel_lines = [str(Path(f).relative_to(local_base)) for f in files]
+
+    fd, list_path = tempfile.mkstemp(suffix=".lst")
+    try:
+        with os.fdopen(fd, "w") as fh:
+            fh.write("\n".join(rel_lines) + "\n")
+        args = ["copy", "--files-from-raw", list_path, str(local_base), dst, *_progress_flags(quiet)]
+        _run(args, echo_stderr=not quiet)
+        if _is_local_dst(dst):
+            dst_root = Path(dst)
+            _remove_local_metadata_sidecars([dst_root / rel for rel in rel_lines], root=dst_root)
+    finally:
+        os.unlink(list_path)
+
+
+def copyto(src_file: Path | str, dst: str, *, quiet: bool) -> None:
+    """Copy a single file to an exact destination (rename semantics).
+
+    Runs ``rclone copyto <src> <dst>`` — used for remote_path remaps and
+    ``sync_by=file`` renames where the destination key differs from the source.
+    """
+    args = ["copyto", str(src_file), dst, *_progress_flags(quiet)]
+    _run(args, echo_stderr=not quiet)
+    if _is_local_dst(dst):
+        _remove_local_metadata_sidecars([Path(dst)])
+
+
+# --------------------------------------------------------------------------- #
+# Listing
+# --------------------------------------------------------------------------- #
+
+
+def lsjson(dst: str, prefix: str = "") -> list[RemoteFile]:
+    """List files under ``dst``/``prefix`` recursively as RemoteFile objects.
+
+    Runs ``rclone lsjson --recursive --files-only [--exclude '._*' ...] <target>``
+    and maps each object: ``Path -> key`` (prefix-relative, nested preserved),
+    ``Size -> size``, ``ModTime -> last_modified``. The ModTime parse is tolerant
+    (RFC3339 with variable fractional seconds / tz, or odd/missing values): the
+    raw string is kept and an empty/absent value becomes None. Never crashes.
+    """
+    target = f"{dst.rstrip('/')}/{prefix.strip('/')}" if prefix else dst
+
+    args = ["lsjson", "--recursive", "--files-only"]
+    if _is_local_dst(dst):
+        args += _LOCAL_METADATA_EXCLUDES
+    args.append(target)
+
+    try:
+        proc = _run(args, echo_stderr=False)
+    except RcloneError as exc:
+        # A missing remote dir/prefix is an empty listing, not a failure: the
+        # duck-typed ls contract returns [] so callers (pull --all, status
+        # --remote) skip stages that have nothing on the remote yet.
+        if _is_missing_prefix_error(exc):
+            return []
+        raise
+    stdout = proc.stdout.strip()
+    if not stdout:
+        return []
+
+    files: list[RemoteFile] = []
+    for obj in json.loads(stdout):
+        key = obj.get("Path")
+        if key is None:
+            continue
+        size_raw = obj.get("Size", 0)
+        try:
+            size = int(size_raw)
+        except (TypeError, ValueError):
+            size = 0
+        files.append(RemoteFile(key=key, size=size, last_modified=_coerce_modtime(obj.get("ModTime"))))
+    return files
+
+
+def _coerce_modtime(value: object) -> str | None:
+    if isinstance(value, str):
+        stripped = value.strip()
+        return stripped or None
+    if value is None:
+        return None
+    return str(value)
+
+
+def _is_missing_prefix_error(exc: RcloneError) -> bool:
+    """Return True for rclone's absent remote dir/prefix listing error."""
+    if exc.code == 3:
+        return True
+    stderr = (exc.stderr or "").lower()
+    return "directory not found" in stderr
+
+
+# --------------------------------------------------------------------------- #
+# Verify
+# --------------------------------------------------------------------------- #
+
+
+def check(local_base: Path, dst: str) -> bool:
+    """Return True when ``local_base`` and ``dst`` match (rclone exit 0).
+
+    Runs ``rclone check [--exclude '._*' ...] <local_base> <dst>``. AppleDouble
+    sidecars are always excluded because ``local_base`` is always a local path
+    (exFAT/macOS would otherwise surface spurious ``._*`` differences).
+    """
+    args = ["check", *_LOCAL_METADATA_EXCLUDES, str(local_base), dst]
+    proc = subprocess.run(["rclone", *args], capture_output=True, text=True)
+    return proc.returncode == 0
+
+
+# --------------------------------------------------------------------------- #
+# RcloneTransport adapter + factory
+# --------------------------------------------------------------------------- #
+
+
+def _batchable_base(local_path: Path, remote_key: str) -> Path | None:
+    """Return the directory ``base`` for which ``local_path == base / remote_key``.
+
+    A pair is batchable iff ``remote_key`` is exactly the tail of ``local_path``
+    relative to some real directory boundary; ``rclone copy --files-from-raw``
+    then places the file at ``dst_root/remote_key`` with no rename. Returns
+    None for remote_path remaps and sync_by=file renames (key diverges from the
+    local layout), which must go through :func:`copyto`.
+    """
+    rk = remote_key.strip("/")
+    if not rk:
+        return None
+    lp = local_path.as_posix()
+    suffix = "/" + rk
+    if not lp.endswith(suffix):
+        return None
+    base = Path(lp[: -len(suffix)])
+    try:
+        if local_path.relative_to(base).as_posix() == rk:
+            return base
+    except ValueError:
+        return None
+    return None
+
+
+class RcloneTransport:
+    """Duck-typed transport backed by the rclone binary (local/s3/sftp).
+
+    Implements the interface ``cli.py`` consumes:
+    ``push`` / ``pull`` / ``ls`` / ``is_available`` / ``backend_name``. Each
+    invocation builds its connstring root on the fly from ``remote_cfg`` via
+    :func:`build_remote`; the ``remote_base`` argument (always ``remote_cfg.url``)
+    is accepted for signature compatibility.
+    """
+
+    def __init__(self, remote_cfg: RemoteConfig) -> None:
+        self.remote_cfg = remote_cfg
+        # Trips after consecutive failed operations so multi-unit push/pull
+        # loops fail fast against a dead remote instead of retrying per unit.
+        self._breaker = CircuitBreaker()
+
+    @property
+    def backend_name(self) -> str:
+        return f"rclone:{self.remote_cfg.remote_type}"
+
+    def is_available(self) -> bool:
+        return is_available()
+
+    def push(
+        self,
+        files: list[tuple[Path, str]],
+        remote_base: str,
+        *,
+        quiet: bool = False,
+    ) -> TransferResult:
+        if not files:
+            return TransferResult(files_transferred=0, bytes_transferred=0)
+
+        dst_root = build_remote(self.remote_cfg)
+        groups: dict[Path, list[Path]] = {}
+        renames: list[tuple[Path, str]] = []
+        for local_path, remote_key in files:
+            local_path = Path(local_path)
+            base = _batchable_base(local_path, remote_key)
+            if base is None:
+                renames.append((local_path, remote_key))
+            else:
+                groups.setdefault(base, []).append(local_path)
+
+        try:
+            self._breaker.guard("push")
+            for base, group in groups.items():
+                copy(group, base, dst_root, quiet=quiet)
+            for local_path, remote_key in renames:
+                copyto(str(local_path), f"{dst_root}/{remote_key.strip('/')}", quiet=quiet)
+        except CircuitOpenError as exc:
+            return TransferResult(0, 0, errors=[str(exc)])
+        except RcloneError as exc:
+            self._breaker.record_failure()
+            return TransferResult(0, 0, errors=[exc.stderr or str(exc)])
+        self._breaker.record_success()
+
+        total_bytes = sum(Path(lp).stat().st_size for lp, _ in files)
+        return TransferResult(files_transferred=len(files), bytes_transferred=total_bytes)
+
+    def pull(
+        self,
+        files: list[tuple[str, Path]],
+        remote_base: str,
+        *,
+        quiet: bool = False,
+    ) -> TransferResult:
+        if not files:
+            return TransferResult(files_transferred=0, bytes_transferred=0)
+
+        dst_root = build_remote(self.remote_cfg)
+        try:
+            self._breaker.guard("pull")
+            for remote_key, local_path in files:
+                local_path = Path(local_path)
+                local_path.parent.mkdir(parents=True, exist_ok=True)
+                copyto(f"{dst_root}/{remote_key.strip('/')}", str(local_path), quiet=quiet)
+        except CircuitOpenError as exc:
+            return TransferResult(0, 0, errors=[str(exc)])
+        except RcloneError as exc:
+            self._breaker.record_failure()
+            return TransferResult(0, 0, errors=[exc.stderr or str(exc)])
+        self._breaker.record_success()
+
+        total_bytes = sum(Path(lp).stat().st_size for _, lp in files if Path(lp).exists())
+        return TransferResult(files_transferred=len(files), bytes_transferred=total_bytes)
+
+    def ls(self, remote_base: str, prefix: str = "") -> list[RemoteFile]:
+        self._breaker.guard("ls")
+        try:
+            listing = lsjson(build_remote(self.remote_cfg), prefix=prefix)
+        except RcloneError:
+            self._breaker.record_failure()
+            raise
+        self._breaker.record_success()
+        return listing
+
+
+def get_transport(remote_cfg: RemoteConfig) -> RcloneTransport:
+    """Return an :class:`RcloneTransport` for a local/s3/sftp remote.
+
+    Dispatches on ``remote_cfg.remote_type``; raises for an unsupported type
+    (matching the message the old factory produced so callers stay identical).
+    """
+    remote_type = remote_cfg.remote_type
+    if remote_type in ("local", "s3", "sftp"):
+        return RcloneTransport(remote_cfg)
+    raise ValueError(f"Unsupported remote type '{remote_type}' for URL: {remote_cfg.url}")

forest/resilience.py

@@ -0,0 +1,122 @@
+"""Resilience primitives guarding external rclone transfers.
+
+Two layers, both tunable (and disableable) via environment variables:
+
+- retry with exponential backoff for *transient* failures only
+  (``forest.rclone._run``): rclone exit code 5 ("temporary error") or stderr
+  matching timeout/connection patterns. Assertion-class failures (bad remote,
+  missing path, auth) never retry.
+  Knobs: ``FOREST_TRANSFER_RETRIES`` (default 2 extra attempts, 0 disables),
+  ``FOREST_RETRY_BASE_DELAY`` (seconds, default 0.5, doubles per attempt).
+
+- a per-transport circuit breaker (``forest.rclone.RcloneTransport``): after
+  N *consecutive* failed transfer operations the circuit opens and remaining
+  operations fail fast instead of hammering a dead remote (e.g. a bare
+  ``forest pull`` over hundreds of units against an unreachable host).
+  Knobs: ``FOREST_BREAKER_THRESHOLD`` (default 5, 0 disables),
+  ``FOREST_BREAKER_RESET_SECONDS`` (default 60; after this cool-down one probe
+  operation is allowed through, closing the circuit again on success).
+"""
+
+from __future__ import annotations
+
+import os
+import re
+import time
+from typing import Callable
+
+# rclone exit code 5: "Temporary error (one that more retries might fix)".
+TRANSIENT_EXIT_CODES = frozenset({5})
+
+_TRANSIENT_STDERR = re.compile(
+    r"(timeout|timed out|connection (reset|refused|closed|aborted)"
+    r"|temporar(y|ily)|too many requests|rate ?limit|unexpected eof|broken pipe)",
+    re.IGNORECASE,
+)
+
+
+def is_transient(exit_code: int, stderr: str) -> bool:
+    """Classify an rclone failure as retryable (network blip) or permanent."""
+    if exit_code in TRANSIENT_EXIT_CODES:
+        return True
+    return bool(_TRANSIENT_STDERR.search(stderr or ""))
+
+
+def _env_int(name: str, default: int) -> int:
+    raw = os.environ.get(name)
+    if raw is None:
+        return default
+    try:
+        return max(0, int(raw))
+    except ValueError:
+        return default
+
+
+def _env_float(name: str, default: float) -> float:
+    raw = os.environ.get(name)
+    if raw is None:
+        return default
+    try:
+        return max(0.0, float(raw))
+    except ValueError:
+        return default
+
+
+def transfer_retries() -> int:
+    """Extra attempts allowed after the first failure (0 disables retries)."""
+    return _env_int("FOREST_TRANSFER_RETRIES", 2)
+
+
+def retry_base_delay() -> float:
+    """Initial backoff delay in seconds; doubles on each subsequent retry."""
+    return _env_float("FOREST_RETRY_BASE_DELAY", 0.5)
+
+
+def backoff_delay(attempt: int) -> float:
+    """Delay before retry ``attempt`` (1-based): base * 2**(attempt-1)."""
+    return retry_base_delay() * (2.0 ** max(0, attempt - 1))
+
+
+class CircuitOpenError(Exception):
+    """Raised when the circuit is open and an operation is refused fast."""
+
+
+class CircuitBreaker:
+    """Consecutive-failure circuit breaker with time-based half-open reset."""
+
+    def __init__(
+        self,
+        *,
+        threshold: int = 0,
+        reset_seconds: float = 0.0,
+        clock: Callable[[], float] = time.monotonic,
+    ) -> None:
+        self._threshold = threshold or _env_int("FOREST_BREAKER_THRESHOLD", 5)
+        self._reset_seconds = reset_seconds or _env_float("FOREST_BREAKER_RESET_SECONDS", 60.0)
+        self._clock = clock
+        self._consecutive_failures = 0
+        self._opened_at: float = 0.0
+
+    @property
+    def is_open(self) -> bool:
+        if self._threshold <= 0 or self._consecutive_failures < self._threshold:
+            return False
+        return (self._clock() - self._opened_at) < self._reset_seconds
+
+    def guard(self, operation: str) -> None:
+        """Refuse ``operation`` fast when the circuit is open."""
+        if self.is_open:
+            raise CircuitOpenError(
+                f"circuit breaker open: {self._consecutive_failures} consecutive transfer failures; "
+                f"refusing '{operation}' for {self._reset_seconds:.0f}s to avoid hammering the remote "
+                "(set FOREST_BREAKER_THRESHOLD=0 to disable)"
+            )
+
+    def record_success(self) -> None:
+        self._consecutive_failures = 0
+        self._opened_at = 0.0
+
+    def record_failure(self) -> None:
+        self._consecutive_failures += 1
+        if self._threshold > 0 and self._consecutive_failures >= self._threshold:
+            self._opened_at = self._clock()