usegradient 1.3.1__py3-none-any.whl

gradient_sdk/runtime.py

@@ -0,0 +1,1474 @@
+from __future__ import annotations
+
+import base64
+import copy
+import fnmatch
+import hashlib
+import inspect
+import io
+import json
+import os
+import posixpath
+import sys
+import tarfile
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Callable, Iterable, Mapping
+
+from .client import GradientClient
+from .errors import GradientAPIError
+
+DEFAULT_EXCLUDES = [
+    ".git/**",
+    ".gradient/**",
+    ".venv/**",
+    "venv/**",
+    "__pycache__/**",
+    "*.pyc",
+    "node_modules/**",
+    ".next/**",
+    "dist/**",
+    "build/**",
+    ".env",
+    ".env.*",
+    "**/.env",
+    "**/.env.*",
+]
+
+
+@dataclass(slots=True)
+class TracePolicy:
+    preset: str
+    semantic: bool
+    runtime: bool
+    stdout: bool
+    stderr: bool
+    process: bool
+    fs: bool
+    syscalls: bool
+    network: bool
+    sample_rate: float = 1.0
+    redactions: list[str] = field(default_factory=list)
+    extra: dict[str, Any] = field(default_factory=dict)
+
+    @property
+    def trace_mode(self) -> str:
+        if self.preset == "off":
+            return "proxy"
+        if self.preset == "semantic":
+            return "proxy"
+        if self.network or self.fs or self.syscalls:
+            return "full"
+        if self.process or self.runtime:
+            return "process"
+        return "proxy"
+
+    @property
+    def trace_exec(self) -> bool:
+        return self.preset != "off" and self.runtime
+
+    def capture_policy(self) -> dict[str, Any]:
+        payload = {
+            "preset": self.preset,
+            "semantic": self.semantic,
+            "runtime": self.runtime,
+            "stdout": self.stdout,
+            "stderr": self.stderr,
+            "process": self.process,
+            "fs": self.fs,
+            "syscalls": self.syscalls,
+            "network": self.network,
+            "sample_rate": self.sample_rate,
+            "sampleRate": self.sample_rate,
+            "redactions": list(self.redactions),
+            "redactAttributePatterns": list(self.redactions),
+        }
+        payload.update(self.extra)
+        return payload
+
+    def to_payload(self) -> dict[str, Any]:
+        return {
+            "trace_mode": self.trace_mode,
+            "capture_policy": self.capture_policy(),
+        }
+
+    @classmethod
+    def from_value(
+        cls,
+        value: "TracePolicy | Mapping[str, Any] | str | None",
+        *,
+        fallback: "TracePolicy | None" = None,
+    ) -> "TracePolicy":
+        if isinstance(value, TracePolicy):
+            return value
+        if value is None:
+            return fallback or Trace.full()
+        if isinstance(value, str):
+            if value == "off":
+                return Trace.off()
+            if value == "semantic":
+                return Trace.semantic()
+            if value == "runtime":
+                return Trace.runtime()
+            if value == "full":
+                return Trace.full()
+            return TracePolicy(
+                preset=value,
+                semantic=value in {"proxy", "full"},
+                runtime=value in {"process", "fs", "network", "full"},
+                stdout=value in {"process", "fs", "network", "full"},
+                stderr=value in {"process", "fs", "network", "full"},
+                process=value in {"process", "fs", "network", "full"},
+                fs=value in {"fs", "full"},
+                syscalls=value in {"process", "fs", "network", "full"},
+                network=value in {"network", "full"},
+            )
+        raw = dict(value)
+        preset = str(raw.pop("preset", raw.pop("mode", "custom")))
+        sample_rate = raw.pop("sample_rate", None)
+        if sample_rate is None:
+            sample_rate = raw.pop("sampleRate", None)
+        if sample_rate is None:
+            sample_rate = raw.pop("sampling_rate", 1.0)
+        redactions = raw.pop("redactions", None)
+        if redactions is None:
+            redactions = raw.pop("redactAttributePatterns", [])
+        return TracePolicy(
+            preset=preset,
+            semantic=bool(raw.pop("semantic", preset in {"semantic", "full"})),
+            runtime=bool(raw.pop("runtime", preset in {"runtime", "full"})),
+            stdout=bool(raw.pop("stdout", preset in {"runtime", "full"})),
+            stderr=bool(raw.pop("stderr", preset in {"runtime", "full"})),
+            process=bool(raw.pop("process", preset in {"runtime", "full"})),
+            fs=bool(raw.pop("fs", preset in {"runtime", "full"})),
+            syscalls=bool(raw.pop("syscalls", preset in {"runtime", "full"})),
+            network=bool(raw.pop("network", preset in {"runtime", "full"})),
+            sample_rate=float(sample_rate),
+            redactions=list(redactions),
+            extra=raw,
+        )
+
+
+class Trace:
+    @staticmethod
+    def off(**extra: Any) -> TracePolicy:
+        return TracePolicy(
+            preset="off",
+            semantic=False,
+            runtime=False,
+            stdout=False,
+            stderr=False,
+            process=False,
+            fs=False,
+            syscalls=False,
+            network=False,
+            extra=dict(extra),
+        )
+
+    @staticmethod
+    def semantic(**extra: Any) -> TracePolicy:
+        return TracePolicy(
+            preset="semantic",
+            semantic=True,
+            runtime=False,
+            stdout=False,
+            stderr=False,
+            process=False,
+            fs=False,
+            syscalls=False,
+            network=False,
+            extra=dict(extra),
+        )
+
+    @staticmethod
+    def runtime(**extra: Any) -> TracePolicy:
+        return TracePolicy(
+            preset="runtime",
+            semantic=False,
+            runtime=True,
+            stdout=True,
+            stderr=True,
+            process=True,
+            fs=True,
+            syscalls=True,
+            network=True,
+            extra=dict(extra),
+        )
+
+    @staticmethod
+    def full(**extra: Any) -> TracePolicy:
+        return TracePolicy(
+            preset="full",
+            semantic=True,
+            runtime=True,
+            stdout=True,
+            stderr=True,
+            process=True,
+            fs=True,
+            syscalls=True,
+            network=True,
+            extra=dict(extra),
+        )
+
+
+@dataclass(slots=True)
+class Environment:
+    runtime: str = "python"
+    python_version: str = "3.12"
+    node_version: str = "22"
+    vcpu: int = 2
+    memory_gb: int = 4
+    disk_gb: int = 10
+    region: str = "sjc"
+    packages: list[str] = field(default_factory=list)
+    system_packages: list[str] = field(default_factory=list)
+    env: dict[str, str] = field(default_factory=dict)
+    image: str | None = None
+    install_at_runtime: bool = False
+
+    @classmethod
+    def python(
+        cls,
+        *,
+        version: str = "3.12",
+        python: str | None = None,
+        vcpu: int = 2,
+        memory_gb: int = 4,
+        disk_gb: int = 10,
+        packages: Iterable[str] | None = None,
+        system_packages: Iterable[str] | None = None,
+        env: Mapping[str, str] | None = None,
+        image: str | None = None,
+        install_at_runtime: bool = False,
+        region: str = "sjc",
+    ) -> "Environment":
+        return cls(
+            runtime="python",
+            python_version=python or version,
+            vcpu=vcpu,
+            memory_gb=memory_gb,
+            disk_gb=disk_gb,
+            packages=list(packages or []),
+            system_packages=list(system_packages or []),
+            env=dict(env or {}),
+            image=image,
+            install_at_runtime=install_at_runtime,
+            region=region,
+        )
+
+    @classmethod
+    def node(
+        cls,
+        *,
+        version: str = "22",
+        node: str | None = None,
+        vcpu: int = 2,
+        memory_gb: int = 4,
+        disk_gb: int = 10,
+        packages: Iterable[str] | None = None,
+        system_packages: Iterable[str] | None = None,
+        env: Mapping[str, str] | None = None,
+        image: str | None = None,
+        install_at_runtime: bool = False,
+        region: str = "sjc",
+    ) -> "Environment":
+        return cls(
+            runtime="node",
+            node_version=node or version,
+            vcpu=vcpu,
+            memory_gb=memory_gb,
+            disk_gb=disk_gb,
+            packages=list(packages or []),
+            system_packages=list(system_packages or []),
+            env=dict(env or {}),
+            image=image,
+            install_at_runtime=install_at_runtime,
+            region=region,
+        )
+
+    def to_spec(self) -> dict[str, Any]:
+        spec: dict[str, Any] = {
+            "runtime": self.runtime,
+            "python": self.python_version,
+            "node": self.node_version,
+            "vcpu": self.vcpu,
+            "memory_gb": self.memory_gb,
+            "disk_gb": self.disk_gb,
+            "region": self.region,
+            "packages": self.packages,
+            "system_packages": self.system_packages,
+            "env": self.env,
+            "install_at_runtime": self.install_at_runtime,
+        }
+        if self.image:
+            spec["image"] = self.image
+        return spec
+
+
+@dataclass(slots=True)
+class ProgressEvent:
+    stage: str
+    status: str
+    message: str
+    duration_ms: int = 0
+    cached: bool = False
+
+
+@dataclass(slots=True)
+class BuildHandle:
+    id: str
+    environment_hash: str
+    source_hash: str
+    image_ref: str
+    cache: dict[str, Any]
+    progress: list[ProgressEvent]
+    timings: dict[str, int]
+    environment_version_id: str = ""
+    source_manifest: dict[str, Any] = field(default_factory=dict)
+    environment_spec: dict[str, Any] = field(default_factory=dict)
+    client: GradientClient | None = field(default=None, repr=False)
+
+    def logs(self) -> list[str]:
+        return [event.message for event in self.progress]
+
+
+@dataclass(slots=True)
+class RunHandle:
+    run_id: str
+    machine_id: str
+    environment_version_id: str
+    project: str
+    stdout: str
+    stderr: str
+    exit_code: int
+    proxy_url: str
+    progress: list[ProgressEvent]
+    timings: dict[str, int]
+    status: str = "completed"
+    client: GradientClient | None = field(default=None, repr=False)
+    input: Any | None = field(default=None, repr=False)
+
+    @property
+    def id(self) -> str:
+        return self.run_id
+
+    @property
+    def build_id(self) -> str:
+        return self.environment_version_id
+
+    @property
+    def trace_id(self) -> str:
+        return self.run_id
+
+    @property
+    def trace_url(self) -> str:
+        if self.client is None:
+            return ""
+        return f"{self.client.api_url.rstrip('/')}/traces/{self.run_id}"
+
+    def result(self) -> "RunHandle":
+        return self
+
+    def cancel(self) -> None:
+        if self.client is None:
+            raise RuntimeError("RunHandle.cancel requires a Gradient client")
+        if self.machine_id:
+            self.client.delete_machine(self.machine_id)
+        if self.run_id:
+            try:
+                self.client.finish_trace_run(self.run_id)
+            except Exception:
+                pass
+        self.status = "cancelled"
+
+    def replay(
+        self,
+        *,
+        seed: int | None = None,
+        freeze_time: str | None = None,
+        egress: str | None = None,
+        trace: TracePolicy | Mapping[str, Any] | str | None = None,
+    ) -> "RunHandle":
+        if self.client is None:
+            raise RuntimeError("RunHandle.replay requires a Gradient client")
+        policy = TracePolicy.from_value(trace)
+        replay_config = {
+            key: value
+            for key, value in {
+                "seed": seed,
+                "freeze_time": freeze_time,
+                "egress": egress,
+            }.items()
+            if value is not None
+        }
+        replayed = self.client.replay_machine(
+            self.run_id,
+            trace_mode=policy.trace_mode,
+            capture_policy=policy.capture_policy(),
+            replay_config=replay_config or None,
+            metadata={"project": self.project, "sdk_surface": "RunHandle.replay"},
+        )
+        run = replayed["trace_run"]
+        machine = replayed["machine"]
+        return RunHandle(
+            run_id=str(run["id"]),
+            machine_id=str(machine["id"]),
+            environment_version_id=self.environment_version_id,
+            project=self.project,
+            stdout="",
+            stderr="",
+            exit_code=0,
+            proxy_url=str(replayed.get("full_proxy_url", "")),
+            progress=[],
+            timings={},
+            status=str(run.get("status", "active")),
+            client=self.client,
+            input=self.input,
+        )
+
+    def events(self, *, limit: int = 1000) -> list[Any]:
+        if self.client is None:
+            return []
+        return self.client.list_trace_events(self.run_id, limit=limit)
+
+    def logs(self) -> list[dict[str, str]]:
+        rows = []
+        if self.stdout:
+            rows.append({"stream": "stdout", "message": self.stdout})
+        if self.stderr:
+            rows.append({"stream": "stderr", "message": self.stderr})
+        return rows
+
+    def spans(self, **filters: Any) -> list[Any]:
+        if self.client is None:
+            return []
+        return self.client.list_trace_spans(self.run_id, **filters)
+
+    def artifacts(self) -> list[dict[str, Any]]:
+        return []
+
+    def to_dataset(
+        self,
+        *,
+        name: str | None = None,
+        dataset_id: str | None = None,
+        expected_output: Any | None = None,
+        metadata: Mapping[str, Any] | None = None,
+    ) -> dict[str, Any]:
+        if self.client is None:
+            raise RuntimeError("RunHandle.to_dataset requires a Gradient client")
+        if not dataset_id:
+            if not name:
+                name = f"{self.project}-runs"
+            dataset_id = str(self.client.create_dataset(name)["id"])
+        return self.client.add_dataset_example(
+            dataset_id,
+            input=self.input,
+            expected_output=expected_output if expected_output is not None else self.stdout,
+            source_trace_id=self.trace_id,
+            metadata={"source_run_id": self.run_id, **dict(metadata or {})},
+        )
+
+
+RemoteRunResult = RunHandle
+
+
+@dataclass(slots=True)
+class RemoteDeployment:
+    name: str
+    project: str
+    environment_version_id: str
+    machine_ids: list[str]
+    proxy_url: str
+    stable_url: str
+    trace_run_id: str | None
+    status: str
+    progress: list[ProgressEvent]
+    timings: dict[str, int]
+    client: GradientClient = field(repr=False)
+
+    @property
+    def url(self) -> str:
+        return self.stable_url or self.proxy_url
+
+    @property
+    def revision(self) -> str:
+        return self.environment_version_id
+
+    def refresh(self) -> "RemoteDeployment":
+        return _deployment_from_response(
+            self.client.get_deployment(self.name),
+            self.client,
+            self.progress,
+            self.timings,
+        )
+
+    def stop(self) -> "RemoteDeployment":
+        return _deployment_from_response(
+            self.client.stop_deployment(self.name),
+            self.client,
+            self.progress,
+            self.timings,
+        )
+
+    def start(self) -> "RemoteDeployment":
+        return _deployment_from_response(
+            self.client.start_deployment(self.name),
+            self.client,
+            self.progress,
+            self.timings,
+        )
+
+    def scale(self, machines: int) -> "RemoteDeployment":
+        return _deployment_from_response(
+            self.client.scale_deployment(self.name, machines),
+            self.client,
+            self.progress,
+            self.timings,
+        )
+
+    def delete(self) -> dict[str, Any]:
+        return self.client.delete_deployment(self.name)
+
+    def runs(self, *, limit: int = 50) -> list[Any]:
+        return self.client.list_trace_runs(limit=limit)
+
+
+DeploymentHandle = RemoteDeployment
+
+
+class Gradient:
+    def __init__(
+        self,
+        project: str | None = None,
+        environment: Environment | Mapping[str, Any] | None = None,
+        source: str | Path | None = None,
+        *,
+        client: GradientClient | None = None,
+        show_progress: bool = True,
+        keep_machine: bool = False,
+        source_root: str | Path | None = None,
+        excludes: Iterable[str] | None = None,
+        env_file: bool | str = True,
+        env_files: Iterable[str | Path] | None = None,
+        trace: TracePolicy | Mapping[str, Any] | str | None = None,
+    ) -> None:
+        if project is None:
+            raise TypeError("project is required")
+        if env_file or env_files:
+            for path in _env_file_candidates(env_file, env_files):
+                self.load_env(path)
+        self.project = project
+        if isinstance(environment, Environment):
+            self.environment = environment
+        else:
+            env_values = dict(environment or {})
+            if "python" in env_values and "python_version" not in env_values:
+                env_values["python_version"] = env_values.pop("python")
+            if "node" in env_values and "node_version" not in env_values:
+                env_values["node_version"] = env_values.pop("node")
+            self.environment = Environment(**env_values)
+        self.client = client or GradientClient.from_credentials()
+        self.show_progress = show_progress
+        self.keep_machine = keep_machine
+        self.source_root = Path(source_root or source or Path.cwd()).resolve()
+        self.excludes = [*DEFAULT_EXCLUDES, *(excludes or [])]
+        self.progress: list[ProgressEvent] = []
+        self.default_trace = TracePolicy.from_value(trace)
+
+    def load_env(self, path: str | Path = ".env", *, override: bool = False) -> dict[str, str]:
+        loaded = _load_env_file(Path(path), override=override)
+        return loaded
+
+    def build(
+        self,
+        target: Callable[[Any], Any] | None = None,
+        *,
+        display_name: str | None = None,
+    ) -> BuildHandle:
+        timings: dict[str, int] = {}
+        self.progress = []
+        self._stage("credentials", "ready", "Loaded Gradient credentials")
+
+        source = _package_source(self.source_root, target, self.excludes)
+        timings["package_source_ms"] = source.duration_ms
+        self._stage(
+            "source",
+            "ready",
+            f"Packaged {source.file_count} files ({len(source.archive_b64)} base64 bytes), source hash {source.source_hash[:12]}",
+            source.duration_ms,
+        )
+
+        t0 = time.perf_counter()
+        resolved = self.client.resolve_environment(
+            project=self.project,
+            environment=self.environment.to_spec(),
+            source=source.manifest,
+            display_name=display_name,
+        )
+        timings["resolve_environment_ms"] = _elapsed_ms(t0)
+        data = resolved["data"] if "data" in resolved and "environmentVersion" in resolved["data"] else resolved
+        for item in data.get("progress", []):
+            self._stage(
+                str(item.get("stage", "environment")),
+                str(item.get("status", "ready")),
+                str(item.get("message", "")),
+                int(item.get("duration_ms", 0) or 0),
+            )
+        env_version = data["environmentVersion"]
+        environment_spec = dict(env_version.get("environment_spec") or self.environment.to_spec())
+        source_manifest = dict(env_version.get("source_manifest") or source.manifest)
+        cache = dict(data.get("cache") or {})
+        self._stage(
+            "build",
+            "cached" if cache.get("environment") == "hit" else "ready",
+            f"Resolved build {env_version['id']} for source {source.source_hash[:12]}",
+            0,
+            cached=cache.get("environment") == "hit",
+        )
+        return BuildHandle(
+            id=str(env_version["id"]),
+            environment_version_id=str(env_version["id"]),
+            environment_hash=str(env_version.get("spec_hash") or _stable_hash(environment_spec)),
+            source_hash=str(env_version.get("source_hash") or source.source_hash),
+            image_ref=str(env_version.get("image_ref") or env_version.get("base_image") or ""),
+            cache=cache,
+            progress=list(self.progress),
+            timings=timings,
+            source_manifest=source_manifest,
+            environment_spec=environment_spec,
+            client=self.client,
+        )
+
+    def run(
+        self,
+        target: Callable[[Any], Any],
+        input: Any | None = None,
+        *,
+        trace: TracePolicy | Mapping[str, Any] | str | None = None,
+        trace_mode: str | None = None,
+        keep: bool | None = None,
+        wait_timeout_seconds: int = 60,
+        run_timeout_seconds: int = 300,
+        watch: Iterable[str] | None = None,
+    ) -> RunHandle:
+        if os.getenv("GRADIENT_REMOTE_EXECUTION") == "1":
+            return target(input)  # type: ignore[return-value]
+
+        trace_policy = _trace_policy(trace, trace_mode, self.default_trace)
+        keep_machine = self.keep_machine if keep is None else keep
+        timings: dict[str, int] = {}
+        overall = time.perf_counter()
+        self.progress = []
+
+        self._stage("credentials", "ready", "Loaded GRADIENT_API_KEY and default Gradient endpoints")
+        t0 = time.perf_counter()
+        self.client.ensure_routable()
+        timings["ensure_routable_ms"] = _elapsed_ms(t0)
+        self._stage("routing", "ready", "Gradient proxy routing is ready", timings["ensure_routable_ms"])
+
+        source = _package_source(self.source_root, target, self.excludes)
+        timings["package_source_ms"] = source.duration_ms
+        self._stage(
+            "source",
+            "ready",
+            f"Packaged {source.file_count} files ({len(source.archive_b64)} base64 bytes), source hash {source.source_hash[:12]}",
+            source.duration_ms,
+        )
+
+        t0 = time.perf_counter()
+        resolved = self.client.resolve_environment(
+            project=self.project,
+            environment=self.environment.to_spec(),
+            source=source.manifest,
+        )
+        timings["resolve_environment_ms"] = _elapsed_ms(t0)
+        data = resolved["data"] if "data" in resolved and "environmentVersion" in resolved["data"] else resolved
+        for item in data.get("progress", []):
+            self._stage(
+                str(item.get("stage", "environment")),
+                str(item.get("status", "ready")),
+                str(item.get("message", "")),
+                int(item.get("duration_ms", 0) or 0),
+            )
+        env_version = data["environmentVersion"]
+        template = env_version["machine_template"]
+
+        t0 = time.perf_counter()
+        traced = self.client.create_traced_machine(
+            template,
+            machine_template=_with_source_bundle(template, source),
+            trace_mode=trace_policy.trace_mode,
+            capture_policy=trace_policy.capture_policy(),
+            metadata={
+                "project": self.project,
+                "environment_version_id": env_version["id"],
+                "source_hash": source.source_hash,
+                "entrypoint": source.entrypoint,
+                "target": source.target_ref,
+                "trace_preset": trace_policy.preset,
+                "sdk_surface": "Gradient.run",
+            },
+            wait=True,
+            wait_timeout_seconds=wait_timeout_seconds,
+        )
+        timings["cold_start_ms"] = _elapsed_ms(t0)
+        machine = traced["machine"]
+        machine_id = machine["id"]
+        run_id = traced["trace_run"]["id"]
+        proxy_url = traced["full_proxy_url"]
+        self._stage("cold-start", "ready", f"Started machine {machine_id} for trace {run_id}", timings["cold_start_ms"])
+
+        try:
+            timings["upload_source_ms"] = 0
+            self._stage("upload", "ready", "Mounted source bundle when the machine was created")
+
+            setup_command = env_version.get("setup_command")
+            if setup_command and self.environment.install_at_runtime:
+                t0 = time.perf_counter()
+                setup = self._exec_machine_with_retry(
+                    machine_id,
+                    command=["sh", "-lc", f"cd /workspace && {setup_command}"],
+                    timeout_seconds=run_timeout_seconds,
+                    trace=False,
+                    stage="dependencies",
+                )
+                timings["runtime_setup_ms"] = _elapsed_ms(t0)
+                self._stage(
+                    "dependencies",
+                    "ready" if setup.exit_code == 0 else "failed",
+                    "Runtime dependency setup completed" if setup.exit_code == 0 else setup.stderr,
+                    timings["runtime_setup_ms"],
+                )
+                if setup.exit_code != 0:
+                    raise RuntimeError(setup.stderr or setup.stdout)
+            else:
+                self._stage("dependencies", "cached", "Skipped runtime install; using cached environment image", cached=True)
+
+            payload = base64.b64encode(json.dumps(input).encode("utf-8")).decode("ascii")
+            remote_code = _remote_bootstrap(source.target_ref, payload)
+            t0 = time.perf_counter()
+            result = self._exec_machine_with_retry(
+                machine_id,
+                command=["python", "-c", remote_code],
+                timeout_seconds=run_timeout_seconds,
+                trace=trace_policy.trace_exec,
+                watch=list(watch or ["/workspace"]),
+                stage="run",
+            )
+            timings["remote_run_ms"] = _elapsed_ms(t0)
+            self._stage(
+                "run",
+                "ready" if result.exit_code == 0 else "failed",
+                f"Remote agent exited with {result.exit_code}",
+                timings["remote_run_ms"],
+            )
+
+            timings["total_ms"] = _elapsed_ms(overall)
+            return RunHandle(
+                run_id=run_id,
+                machine_id=machine_id,
+                environment_version_id=str(env_version["id"]),
+                project=self.project,
+                stdout=result.stdout,
+                stderr=result.stderr,
+                exit_code=result.exit_code,
+                proxy_url=proxy_url,
+                progress=list(self.progress),
+                timings=timings,
+                status="completed" if result.exit_code == 0 else "failed",
+                client=self.client,
+                input=input,
+            )
+        finally:
+            if not keep_machine:
+                t0 = time.perf_counter()
+                try:
+                    self.client.delete_machine(machine_id)
+                    self._stage("cleanup", "ready", f"Deleted machine {machine_id}", _elapsed_ms(t0))
+                except Exception as exc:
+                    self._stage("cleanup", "failed", f"Could not delete machine {machine_id}: {exc}", _elapsed_ms(t0))
+
+    def deploy(
+        self,
+        target: Callable[[Any], Any],
+        *,
+        name: str | None = None,
+        trace: TracePolicy | Mapping[str, Any] | str | None = None,
+        trace_mode: str | None = None,
+        scale: int | None = None,
+        wait_timeout_seconds: int = 60,
+        run_timeout_seconds: int = 300,
+        watch: Iterable[str] | None = None,
+        min_machines: int = 1,
+        max_machines: int = 1,
+    ) -> RemoteDeployment:
+        if os.getenv("GRADIENT_REMOTE_EXECUTION") == "1":
+            raise RuntimeError("Gradient.deploy cannot be called from inside a remote Gradient machine")
+
+        trace_policy = _trace_policy(trace, trace_mode, self.default_trace)
+        if scale is not None:
+            min_machines = scale
+            max_machines = scale
+        timings: dict[str, int] = {}
+        overall = time.perf_counter()
+        self.progress = []
+
+        deployment_name = name or self.project
+        self._stage("credentials", "ready", "Loaded GRADIENT_API_KEY and default Gradient endpoints")
+        t0 = time.perf_counter()
+        self.client.ensure_routable()
+        timings["ensure_routable_ms"] = _elapsed_ms(t0)
+        self._stage("routing", "ready", "Gradient proxy routing is ready", timings["ensure_routable_ms"])
+
+        source = _package_source(self.source_root, target, self.excludes)
+        timings["package_source_ms"] = source.duration_ms
+        self._stage(
+            "source",
+            "ready",
+            f"Packaged {source.file_count} files ({len(source.archive_b64)} base64 bytes), source hash {source.source_hash[:12]}",
+            source.duration_ms,
+        )
+
+        t0 = time.perf_counter()
+        resolved = self.client.resolve_environment(
+            project=self.project,
+            environment=self.environment.to_spec(),
+            source=source.manifest,
+        )
+        timings["resolve_environment_ms"] = _elapsed_ms(t0)
+        data = resolved["data"] if "data" in resolved and "environmentVersion" in resolved["data"] else resolved
+        for item in data.get("progress", []):
+            self._stage(
+                str(item.get("stage", "environment")),
+                str(item.get("status", "ready")),
+                str(item.get("message", "")),
+                int(item.get("duration_ms", 0) or 0),
+            )
+        env_version = data["environmentVersion"]
+
+        t0 = time.perf_counter()
+        deployment_data = self.client.create_deployment(
+            name=deployment_name,
+            project=self.project,
+            template=_with_source_bundle(
+                env_version["machine_template"],
+                source,
+                service_code=_remote_service_bootstrap(source.target_ref),
+            ),
+            environment_version_id=str(env_version["id"]),
+            trace_mode=trace_policy.trace_mode,
+            capture_policy=trace_policy.capture_policy(),
+            metadata={
+                "source_hash": source.source_hash,
+                "entrypoint": source.entrypoint,
+                "target": source.target_ref,
+                "trace_preset": trace_policy.preset,
+            },
+            min_machines=min_machines,
+            max_machines=max_machines,
+        )
+        timings["deploy_create_ms"] = _elapsed_ms(t0)
+        deployment = _deployment_from_response(
+            deployment_data,
+            self.client,
+            self.progress,
+            timings,
+        )
+        self._stage(
+            "deploy",
+            "ready",
+            f"Created deployment {deployment.name} with {len(deployment.machine_ids)} machine(s)",
+            timings["deploy_create_ms"],
+        )
+
+        if deployment.machine_ids:
+            machine_id = deployment.machine_ids[0]
+            t0 = time.perf_counter()
+            try:
+                self.client.wait_machine_state(machine_id, timeout_seconds=wait_timeout_seconds)
+            except Exception:
+                pass
+            timings["deploy_wait_ms"] = _elapsed_ms(t0)
+            self._stage("cold-start", "ready", f"Deployment machine {machine_id} is accepting setup", timings["deploy_wait_ms"])
+
+            timings["upload_source_ms"] = 0
+            self._stage("upload", "ready", "Mounted deployment source bundle at machine creation")
+
+            setup_command = env_version.get("setup_command")
+            if setup_command and self.environment.install_at_runtime:
+                t0 = time.perf_counter()
+                setup = self._exec_machine_with_retry(
+                    machine_id,
+                    command=["sh", "-lc", f"cd /workspace && {setup_command}"],
+                    timeout_seconds=run_timeout_seconds,
+                    trace=False,
+                    stage="dependencies",
+                )
+                timings["runtime_setup_ms"] = _elapsed_ms(t0)
+                self._stage(
+                    "dependencies",
+                    "ready" if setup.exit_code == 0 else "failed",
+                    "Runtime dependency setup completed" if setup.exit_code == 0 else setup.stderr,
+                    timings["runtime_setup_ms"],
+                )
+                if setup.exit_code != 0:
+                    raise RuntimeError(setup.stderr or setup.stdout)
+            else:
+                self._stage("dependencies", "cached", "Skipped runtime install; using cached environment image", cached=True)
+
+            t0 = time.perf_counter()
+            started = self._exec_machine_with_retry(
+                machine_id,
+                command=[
+                    "sh",
+                    "-lc",
+                    "cd /workspace && nohup python -u /tmp/gradient-service.py >/tmp/gradient-service.log 2>&1 &",
+                ],
+                timeout_seconds=30,
+                trace=trace_policy.trace_exec,
+                watch=list(watch or ["/workspace"]),
+                stage="serve",
+            )
+            timings["serve_start_ms"] = _elapsed_ms(t0)
+            self._stage(
+                "serve",
+                "ready" if started.exit_code == 0 else "failed",
+                f"Deployment service is listening at {deployment.proxy_url}",
+                timings["serve_start_ms"],
+            )
+            if started.exit_code != 0:
+                raise RuntimeError(started.stderr or started.stdout)
+
+        timings["total_ms"] = _elapsed_ms(overall)
+        deployment.timings.update(timings)
+        deployment.progress[:] = list(self.progress)
+        return deployment
+
+    def replay(
+        self,
+        run_id: str,
+        *,
+        seed: int | None = None,
+        freeze_time: str | None = None,
+        egress: str | None = None,
+        trace: TracePolicy | Mapping[str, Any] | str | None = None,
+        wait_timeout_seconds: int = 30,
+    ) -> RunHandle:
+        trace_policy = TracePolicy.from_value(trace, fallback=self.default_trace)
+        replay_config = {
+            key: value
+            for key, value in {
+                "seed": seed,
+                "freeze_time": freeze_time,
+                "egress": egress,
+            }.items()
+            if value is not None
+        }
+        replayed = self.client.replay_machine(
+            run_id,
+            trace_mode=trace_policy.trace_mode,
+            capture_policy=trace_policy.capture_policy(),
+            replay_config=replay_config or None,
+            metadata={
+                "project": self.project,
+                "trace_preset": trace_policy.preset,
+                "sdk_surface": "Gradient.replay",
+            },
+            wait=True,
+            wait_timeout_seconds=wait_timeout_seconds,
+        )
+        run = replayed["trace_run"]
+        machine = replayed["machine"]
+        return RunHandle(
+            run_id=str(run["id"]),
+            machine_id=str(machine["id"]),
+            environment_version_id="",
+            project=self.project,
+            stdout="",
+            stderr="",
+            exit_code=0,
+            proxy_url=str(replayed.get("full_proxy_url", "")),
+            progress=[],
+            timings={},
+            status=str(run.get("status", "active")),
+            client=self.client,
+        )
+
+    def instrument(self, target: Any) -> Any:
+        from .integrations import wrap_anthropic, wrap_openai
+
+        module_name = target.__class__.__module__.lower()
+        class_name = target.__class__.__name__.lower()
+        if "openai" in module_name or "openai" in class_name:
+            return wrap_openai(target)
+        if "anthropic" in module_name or "anthropic" in class_name:
+            return wrap_anthropic(target)
+        if hasattr(target, "invoke") or hasattr(target, "stream"):
+            return _InstrumentedRunnable(target)
+        return target
+
+    def _exec_machine_with_retry(
+        self,
+        machine_id: str,
+        *,
+        command: list[str],
+        stdin: str | None = None,
+        timeout_seconds: int,
+        trace: bool,
+        watch: list[str] | None = None,
+        stage: str,
+        attempts: int = 4,
+    ) -> Any:
+        last_error: Exception | None = None
+        for attempt in range(1, attempts + 1):
+            try:
+                return self.client.exec_machine(
+                    machine_id,
+                    command,
+                    stdin=stdin,
+                    timeout_seconds=timeout_seconds,
+                    trace=trace,
+                    watch=watch,
+                )
+            except GradientAPIError as error:
+                last_error = error
+                if not _retryable_exec_error(error) or attempt == attempts:
+                    raise
+                if _machine_not_running_error(error):
+                    try:
+                        self.client.wait_machine_state(machine_id, timeout_seconds=15)
+                    except Exception:
+                        pass
+                delay_ms = attempt * 1500
+                self._stage(
+                    stage,
+                    "ready",
+                    f"Machine reported started but exec was not ready yet, retrying in {delay_ms}ms (attempt {attempt + 1}/{attempts})",
+                )
+                time.sleep(delay_ms / 1000)
+        if last_error is not None:
+            raise last_error
+        raise RuntimeError("unreachable exec retry state")
+
+    def benchmark(
+        self,
+        target: Callable[[Any], Any],
+        input: Any | None = None,
+        *,
+        iterations: int = 3,
+        trace_mode: str = "full",
+    ) -> list[RemoteRunResult]:
+        results: list[RemoteRunResult] = []
+        for index in range(max(1, iterations)):
+            self._stage("benchmark", "ready", f"Starting iteration {index + 1}/{iterations}")
+            results.append(self.run(target, input, trace_mode=trace_mode))
+        cold_starts = [item.timings.get("cold_start_ms", 0) for item in results]
+        if cold_starts:
+            avg = sum(cold_starts) // len(cold_starts)
+            self._stage(
+                "benchmark",
+                "ready",
+                f"Cold-start timings ms: min={min(cold_starts)} avg={avg} max={max(cold_starts)}",
+            )
+        return results
+
+    def _stage(
+        self,
+        stage: str,
+        status: str,
+        message: str,
+        duration_ms: int = 0,
+        *,
+        cached: bool | None = None,
+    ) -> None:
+        event = ProgressEvent(
+            stage=stage,
+            status=status,
+            message=message,
+            duration_ms=duration_ms,
+            cached=(cached if cached is not None else status == "cached"),
+        )
+        self.progress.append(event)
+        if self.show_progress:
+            suffix = f" {duration_ms}ms" if duration_ms else ""
+            marker = "cached" if event.cached else status
+            print(f"[gradient] {stage}: {marker} - {message}{suffix}", flush=True)
+
+
+@dataclass(slots=True)
+class SourceBundle:
+    archive_b64: str
+    source_hash: str
+    manifest: dict[str, Any]
+    entrypoint: str
+    target_ref: str
+    file_count: int
+    duration_ms: int
+
+
+class _InstrumentedRunnable:
+    def __init__(self, target: Any) -> None:
+        self.target = target
+
+    def invoke(self, *args: Any, **kwargs: Any) -> Any:
+        from .decorators import trace_context
+
+        with trace_context(f"{self.target.__class__.__name__}.invoke", kind="CHAIN"):
+            return self.target.invoke(*args, **kwargs)
+
+    def stream(self, *args: Any, **kwargs: Any) -> Any:
+        from .decorators import trace_context
+
+        with trace_context(f"{self.target.__class__.__name__}.stream", kind="CHAIN"):
+            yield from self.target.stream(*args, **kwargs)
+
+    def __getattr__(self, name: str) -> Any:
+        return getattr(self.target, name)
+
+
+def _with_source_bundle(
+    template: Mapping[str, Any],
+    source: SourceBundle,
+    *,
+    service_code: str | None = None,
+) -> dict[str, Any]:
+    result = copy.deepcopy(dict(template))
+    config = dict(result.get("config") or {})
+    env = dict(config.get("env") or {})
+    env["GRADIENT_SOURCE_ARCHIVE_B64"] = source.archive_b64
+    init_steps = [
+        "rm -rf /workspace",
+        "mkdir -p /workspace",
+        "printf '%s' \"$GRADIENT_SOURCE_ARCHIVE_B64\" | base64 -d | tar -xzf - -C /workspace",
+    ]
+    if service_code is not None:
+        env["GRADIENT_SERVICE_CODE_B64"] = base64.b64encode(
+            service_code.encode("utf-8")
+        ).decode("ascii")
+        init_steps.append(
+            "printf '%s' \"$GRADIENT_SERVICE_CODE_B64\" | base64 -d > /tmp/gradient-service.py"
+        )
+    init_steps.append("exec sleep infinity")
+    config["env"] = env
+    config["init"] = {
+        "cmd": [
+            "sh",
+            "-lc",
+            " && ".join(init_steps),
+        ]
+    }
+    result["config"] = config
+    return result
+
+
+def _package_source(root: Path, target: Callable[..., Any] | None, excludes: list[str]) -> SourceBundle:
+    started = time.perf_counter()
+    unwrapped_target: Callable[..., Any] | None = (
+        inspect.unwrap(target)
+        if target is not None and (inspect.isfunction(target) or inspect.ismethod(target))
+        else target
+    )
+    entrypoint = ""
+    target_ref = ""
+    if unwrapped_target is not None:
+        entrypoint, target_ref = _target_entrypoint_and_ref(root, unwrapped_target)
+
+    files: list[tuple[Path, str]] = []
+    for path in root.rglob("*"):
+        if path.is_dir():
+            continue
+        rel = path.relative_to(root).as_posix()
+        if _excluded(rel, excludes):
+            continue
+        files.append((path, rel))
+
+    digest = hashlib.sha256()
+    raw = io.BytesIO()
+    with tarfile.open(fileobj=raw, mode="w:gz") as archive:
+        for path, rel in sorted(files, key=lambda item: item[1]):
+            data = path.read_bytes()
+            digest.update(rel.encode("utf-8"))
+            digest.update(b"\0")
+            digest.update(hashlib.sha256(data).digest())
+            info = tarfile.TarInfo(rel)
+            info.size = len(data)
+            info.mode = path.stat().st_mode & 0o777
+            info.mtime = 0
+            archive.addfile(info, io.BytesIO(data))
+    encoded = base64.b64encode(raw.getvalue()).decode("ascii")
+    if len(encoded) > 950_000:
+        raise ValueError(
+            "source bundle is larger than the current 1MB upload limit; add ignores or use a prebuilt environment image"
+        )
+    source_hash = digest.hexdigest()
+    manifest = {
+        "hash": source_hash,
+        "entrypoint": entrypoint,
+        "target": target_ref,
+        "file_count": len(files),
+        "archive_bytes": len(raw.getvalue()),
+        "ignored": excludes,
+    }
+    return SourceBundle(
+        archive_b64=encoded,
+        source_hash=source_hash,
+        manifest=manifest,
+        entrypoint=entrypoint,
+        target_ref=target_ref,
+        file_count=len(files),
+        duration_ms=_elapsed_ms(started),
+    )
+
+
+def _target_entrypoint_and_ref(root: Path, target: Callable[..., Any]) -> tuple[str, str]:
+    if inspect.isfunction(target) or inspect.ismethod(target):
+        target_file = Path(inspect.getsourcefile(target) or "").resolve()
+        if not target_file.exists():
+            raise ValueError("target must be a function defined in a source file")
+        try:
+            entrypoint = target_file.relative_to(root).as_posix()
+        except ValueError as exc:
+            raise ValueError(f"target file {target_file} is not under source root {root}") from exc
+        module_name = target.__module__
+        if module_name == "__main__":
+            module_name = entrypoint[:-3].replace("/", ".") if entrypoint.endswith(".py") else entrypoint.replace("/", ".")
+        if "<locals>" in target.__qualname__:
+            raise ValueError("remote Gradient.run targets must be top-level functions so the sandbox can import them")
+        return entrypoint, f"{module_name}:{target.__qualname__}"
+
+    found = _find_global_object_ref(root, target)
+    if found is not None:
+        return found
+    raise ValueError(
+        "remote Gradient.run targets must be top-level functions or global callable Agent objects"
+    )
+
+
+def _find_global_object_ref(root: Path, target: Any) -> tuple[str, str] | None:
+    for module in list(sys.modules.values()):
+        module_file = getattr(module, "__file__", None)
+        if not module_file:
+            continue
+        path = Path(module_file).resolve()
+        try:
+            entrypoint = path.relative_to(root).as_posix()
+        except ValueError:
+            continue
+        for name, value in vars(module).items():
+            if value is target and not name.startswith("_"):
+                return entrypoint, f"{module.__name__}:{name}"
+    return None
+
+
+def _remote_bootstrap(target_ref: str, payload_b64: str) -> str:
+    module, _, qualname = target_ref.partition(":")
+    if not module or not qualname:
+        raise ValueError("target ref must be module:qualname")
+    return "\n".join(
+        [
+            "import base64, importlib, json, os, sys",
+            "sys.path.insert(0, '/workspace')",
+            "os.environ['GRADIENT_REMOTE_EXECUTION'] = '1'",
+            f"mod = importlib.import_module({module!r})",
+            "obj = mod",
+            f"for part in {qualname.split('.')!r}: obj = getattr(obj, part)",
+            _remote_trace_setup_code(),
+            f"payload = json.loads(base64.b64decode({payload_b64!r}).decode('utf-8'))",
+            f"emit_trace_event('gradient.sdk.remote.start', {{'target': {target_ref!r}}})",
+            "try:",
+            "    result = obj(payload)",
+            f"    emit_trace_event('gradient.sdk.remote.success', {{'target': {target_ref!r}, 'output': result}})",
+            "    print(json.dumps(result, default=str))",
+            "except Exception as exc:",
+            f"    emit_trace_event('gradient.sdk.remote.error', {{'target': {target_ref!r}, 'error': str(exc), 'error_type': type(exc).__name__}})",
+            "    raise",
+            "finally:",
+            "    tracer.flush()",
+            "    tracer.shutdown()",
+        ]
+    )
+
+
+def _remote_service_bootstrap(target_ref: str) -> str:
+    module, _, qualname = target_ref.partition(":")
+    if not module or not qualname:
+        raise ValueError("target ref must be module:qualname")
+    return "\n".join(
+        [
+            "import importlib, json, os, sys",
+            "from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer",
+            "sys.path.insert(0, '/workspace')",
+            "os.environ['GRADIENT_REMOTE_EXECUTION'] = '1'",
+            f"mod = importlib.import_module({module!r})",
+            "obj = mod",
+            f"for part in {qualname.split('.')!r}: obj = getattr(obj, part)",
+            _remote_trace_setup_code(),
+            "class Handler(BaseHTTPRequestHandler):",
+            "    def do_GET(self):",
+            "        self.send_response(200)",
+            "        self.send_header('content-type', 'application/json')",
+            "        self.end_headers()",
+            "        self.wfile.write(json.dumps({'ok': True, 'service': 'gradient'}).encode())",
+            "    def do_POST(self):",
+            "        try:",
+            "            length = int(self.headers.get('content-length') or '0')",
+            "            raw = self.rfile.read(length) if length else b'{}'",
+            "            payload = json.loads(raw.decode('utf-8') or '{}')",
+            f"            emit_trace_event('gradient.sdk.remote.request', {{'target': {target_ref!r}, 'path': self.path}})",
+            "            result = obj(payload)",
+            f"            emit_trace_event('gradient.sdk.remote.success', {{'target': {target_ref!r}, 'path': self.path, 'output': result}})",
+            "            tracer.flush()",
+            "            body = json.dumps({'ok': True, 'result': result}, default=str).encode()",
+            "            self.send_response(200)",
+            "        except Exception as exc:",
+            f"            emit_trace_event('gradient.sdk.remote.error', {{'target': {target_ref!r}, 'path': self.path, 'error': str(exc), 'error_type': type(exc).__name__}})",
+            "            body = json.dumps({'ok': False, 'error': str(exc)}).encode()",
+            "            self.send_response(500)",
+            "        self.send_header('content-type', 'application/json')",
+            "        self.end_headers()",
+            "        self.wfile.write(body)",
+            "    def log_message(self, fmt, *args):",
+            "        print(fmt % args, flush=True)",
+            "port = int(os.environ.get('PORT') or os.environ.get('GRADIENT_PORT') or '8080')",
+            "ThreadingHTTPServer(('0.0.0.0', port), Handler).serve_forever()",
+        ]
+    )
+
+
+def _remote_trace_setup_code() -> str:
+    return "\n".join(
+        [
+            "from gradient_sdk.tracing import SpanExporter, Tracer, set_tracer",
+            "import urllib.request",
+            "trace_token = os.environ.get('GRADIENT_TRACE_TOKEN', '')",
+            "trace_run_id = os.environ.get('GRADIENT_TRACE_RUN_ID', '')",
+            "trace_event_endpoint = os.environ.get('GRADIENT_TRACE_ENDPOINT', '')",
+            "trace_endpoint = os.environ.get('OTEL_EXPORTER_OTLP_TRACES_ENDPOINT', '')",
+            "if not trace_endpoint and os.environ.get('GRADIENT_TRACE_ENDPOINT', '').endswith('/api/v1/runs/ingest'):",
+            "    trace_endpoint = os.environ['GRADIENT_TRACE_ENDPOINT'][:-len('/api/v1/runs/ingest')] + '/v1/traces'",
+            "project_name = os.environ.get('GRADIENT_PROJECT') or os.environ.get('OTEL_SERVICE_NAME') or 'gradient'",
+            "def emit_trace_event(event_type, data):",
+            "    if not trace_token or not trace_run_id or not trace_event_endpoint:",
+            "        print('[gradient remote] trace event skipped: missing trace env', file=sys.stderr, flush=True)",
+            "        return",
+            "    try:",
+            "        body = json.dumps({'run_id': trace_run_id, 'events': [{'source': 'sdk', 'type': event_type, 'data': data}]}, default=str).encode('utf-8')",
+            "        req = urllib.request.Request(trace_event_endpoint, data=body, method='POST', headers={'Authorization': 'Bearer ' + trace_token, 'Content-Type': 'application/json', 'Accept': 'application/json'})",
+            "        with urllib.request.urlopen(req, timeout=5) as res:",
+            "            res.read()",
+            "        print('[gradient remote] emitted ' + event_type, file=sys.stderr, flush=True)",
+            "    except Exception as emit_exc:",
+            "        print('[gradient remote] trace event failed: ' + str(emit_exc), file=sys.stderr, flush=True)",
+            "if trace_token and trace_endpoint:",
+            "    exporter = SpanExporter(endpoint=trace_endpoint, ingest_token=trace_token, resource={'service.name': project_name, 'openinference.project.name': project_name, 'gradient.run.id': os.environ.get('GRADIENT_TRACE_RUN_ID', '')})",
+            "    tracer = Tracer(project=project_name, exporter=exporter)",
+            "    set_tracer(tracer)",
+            "else:",
+            "    from gradient_sdk.tracing import get_tracer",
+            "    tracer = get_tracer()",
+        ]
+    )
+
+
+def _deployment_from_response(
+    data: Mapping[str, Any],
+    client: GradientClient,
+    progress: list[ProgressEvent],
+    timings: dict[str, int],
+) -> RemoteDeployment:
+    payload = data["data"] if "data" in data and isinstance(data["data"], Mapping) else data
+    return RemoteDeployment(
+        name=str(payload.get("name") or payload.get("id") or ""),
+        project=str(payload.get("project") or ""),
+        environment_version_id=str(payload.get("environment_version_id") or ""),
+        machine_ids=[str(item) for item in payload.get("machine_ids", [])],
+        proxy_url=str(payload.get("proxy_url") or payload.get("stable_url") or ""),
+        stable_url=str(payload.get("stable_url") or payload.get("proxy_url") or ""),
+        trace_run_id=str(payload.get("trace_run_id")) if payload.get("trace_run_id") else None,
+        status=str(payload.get("status") or "unknown"),
+        progress=list(progress),
+        timings=dict(timings),
+        client=client,
+    )
+
+
+def _excluded(rel: str, patterns: list[str]) -> bool:
+    normalized = rel.replace(os.sep, posixpath.sep)
+    return any(
+        fnmatch.fnmatch(normalized, pattern) or fnmatch.fnmatch(normalized, pattern.rstrip("/**"))
+        for pattern in patterns
+    )
+
+
+def _elapsed_ms(started: float) -> int:
+    return int((time.perf_counter() - started) * 1000)
+
+
+def _stable_hash(value: Any) -> str:
+    return hashlib.sha256(
+        json.dumps(value, sort_keys=True, separators=(",", ":"), default=str).encode("utf-8")
+    ).hexdigest()
+
+
+def _trace_policy(
+    trace: TracePolicy | Mapping[str, Any] | str | None,
+    trace_mode: str | None,
+    fallback: TracePolicy,
+) -> TracePolicy:
+    if trace is not None and trace_mode is not None:
+        raise ValueError("pass either trace or trace_mode, not both")
+    if trace_mode is not None:
+        return TracePolicy.from_value(trace_mode, fallback=fallback)
+    return TracePolicy.from_value(trace, fallback=fallback)
+
+
+def _env_file_candidates(
+    env_file: bool | str,
+    env_files: Iterable[str | Path] | None,
+) -> list[Path]:
+    paths: list[Path] = []
+    if isinstance(env_file, str):
+        paths.append(Path(env_file))
+    elif env_file:
+        paths.extend([Path(".env.local"), Path(".env"), Path(".env.gradient")])
+    paths.extend(Path(path) for path in env_files or [])
+    return paths
+
+
+def _load_env_file(path: Path, *, override: bool = False) -> dict[str, str]:
+    if not path.exists() or not path.is_file():
+        return {}
+    loaded: dict[str, str] = {}
+    for raw_line in path.read_text(encoding="utf-8").splitlines():
+        line = raw_line.strip()
+        if not line or line.startswith("#"):
+            continue
+        if line.startswith("export "):
+            line = line[len("export ") :].lstrip()
+        key, separator, raw_value = line.partition("=")
+        if not separator:
+            continue
+        key = key.strip()
+        if not key:
+            continue
+        value = raw_value.strip()
+        if len(value) >= 2 and value[0] == value[-1] and value[0] in {"'", '"'}:
+            value = value[1:-1]
+        if override or key not in os.environ:
+            os.environ[key] = value
+            loaded[key] = value
+    return loaded
+
+
+def _retryable_exec_error(error: GradientAPIError) -> bool:
+    if error.status != 412:
+        return False
+    body = (error.response_body or error.message).lower()
+    return "exec request failed: eof" in body or "machine not running" in body
+
+
+def _machine_not_running_error(error: GradientAPIError) -> bool:
+    body = (error.response_body or error.message).lower()
+    return "machine not running" in body