highflame-forge 0.0.3__py3-none-any.whl

highflame_forge/__init__.py

@@ -0,0 +1,140 @@
+"""Highflame Forge - Unified sandbox platform for ML training and secure execution.
+
+Example:
+    ```python
+    from highflame_forge import Forge
+
+    forge = Forge()
+
+    # Simple sandbox usage
+    with forge.sandbox(gpu="A40", memory_gb=32) as sb:
+        sb.run_sync("python train.py --epochs 3")
+
+    # Training job with automatic tracking
+    from highflame_forge import TrainingJob
+
+    job = TrainingJob(
+        forge=forge,
+        script="train.py",
+        args={"epochs": 3, "lr": 2e-5},
+        gpu="A40",
+    )
+    result = await job.run()
+    print(f"Final loss: {result.final_loss}")
+    ```
+"""
+
+from highflame_forge.backends.base import (
+    Backend,
+    CredentialPolicyError,
+    IsolationUnavailableError,
+    create_backend,
+    get_backend_class,
+    register_backend,
+    registered_backends,
+)
+from highflame_forge.backends.gvisor_backend import GVisorBackend, GVisorSandbox
+from highflame_forge.backends.local_backend import LocalBackend, LocalSandbox
+from highflame_forge.config import (
+    BackendType,
+    GPUType,
+    IsolationTier,
+    NetworkConfig,
+    Presets,
+    SandboxConfig,
+    SandboxStatus,
+)
+from highflame_forge.credentials import CredentialFinding, find_long_lived_credentials
+from highflame_forge.enforcement import (
+    EnforcementSpec,
+    build_tracing_policy,
+    derive_enforcement,
+)
+from highflame_forge.forge import BackendSelector, Forge
+from highflame_forge.git import GitAuthMethod, GitRepo
+from highflame_forge.identity import (
+    MECHANISM_CAPABILITIES,
+    SCOPE_DEV_GPU,
+    SCOPE_FS_READ,
+    SCOPE_FS_WRITE,
+    SCOPE_NET_EGRESS,
+    SCOPE_SYS_BPF,
+    SCOPE_SYS_PTRACE,
+    SCOPE_SYS_RAW_SOCKET,
+    SCOPE_TIER_HARDWARE,
+    SCOPE_TIER_KERNEL,
+    SCOPE_TIER_OS,
+    SCOPE_TOOLS_EXECUTE,
+    SCOPE_VIRT_NESTED,
+    Capabilities,
+    CapabilityResolver,
+    SandboxIdentity,
+    ScopeCapabilityResolver,
+)
+from highflame_forge.jobs.training import (
+    HyperparameterSweep,
+    SweepResult,
+    TrainingEvent,
+    TrainingEventType,
+    TrainingJob,
+    TrainingResult,
+)
+from highflame_forge.sandbox import CommandResult, Sandbox, SandboxMetrics
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "MECHANISM_CAPABILITIES",
+    "SCOPE_DEV_GPU",
+    "SCOPE_FS_READ",
+    "SCOPE_FS_WRITE",
+    "SCOPE_NET_EGRESS",
+    "SCOPE_SYS_BPF",
+    "SCOPE_SYS_PTRACE",
+    "SCOPE_SYS_RAW_SOCKET",
+    "SCOPE_TIER_HARDWARE",
+    "SCOPE_TIER_KERNEL",
+    "SCOPE_TIER_OS",
+    "SCOPE_TOOLS_EXECUTE",
+    "SCOPE_VIRT_NESTED",
+    "Backend",
+    "BackendSelector",
+    "BackendType",
+    "Capabilities",
+    "CapabilityResolver",
+    "CommandResult",
+    "CredentialFinding",
+    "CredentialPolicyError",
+    "EnforcementSpec",
+    "Forge",
+    "GPUType",
+    "GVisorBackend",
+    "GVisorSandbox",
+    "GitAuthMethod",
+    "GitRepo",
+    "HyperparameterSweep",
+    "IsolationTier",
+    "IsolationUnavailableError",
+    "LocalBackend",
+    "LocalSandbox",
+    "NetworkConfig",
+    "Presets",
+    "Sandbox",
+    "SandboxConfig",
+    "SandboxIdentity",
+    "SandboxMetrics",
+    "SandboxStatus",
+    "ScopeCapabilityResolver",
+    "SweepResult",
+    "TrainingEvent",
+    "TrainingEventType",
+    "TrainingJob",
+    "TrainingResult",
+    "build_tracing_policy",
+    "create_backend",
+    "derive_enforcement",
+    "find_long_lived_credentials",
+    "get_backend_class",
+    "register_backend",
+    "registered_backends",
+]

highflame_forge/backends/__init__.py

@@ -0,0 +1,18 @@
+"""Backend implementations for Highflame Forge."""
+
+from highflame_forge.backends.base import Backend, CostEstimate, GPUAvailability
+from highflame_forge.backends.gcp_backend import GCPBackend
+from highflame_forge.backends.local_backend import LocalBackend, LocalSandbox
+from highflame_forge.backends.modal_backend import ModalBackend
+from highflame_forge.backends.runpod_backend import RunPodBackend
+
+__all__ = [
+    "Backend",
+    "CostEstimate",
+    "GCPBackend",
+    "GPUAvailability",
+    "LocalBackend",
+    "LocalSandbox",
+    "ModalBackend",
+    "RunPodBackend",
+]

highflame_forge/backends/base.py

@@ -0,0 +1,444 @@
+"""Abstract base class for compute backends."""
+
+from __future__ import annotations
+
+import logging
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from highflame_forge.config import BackendType, GPUType, IsolationTier, SandboxConfig
+
+if TYPE_CHECKING:
+    from datetime import datetime
+
+    from highflame_forge.identity import CapabilityResolver
+    from highflame_forge.sandbox import Sandbox
+
+logger = logging.getLogger(__name__)
+
+
+class IsolationUnavailableError(RuntimeError):
+    """The isolation tier an execution requires is not available on this
+    backend. Raised at provision time — the boundary fails closed rather than
+    silently downgrading to a weaker tier (ADR 0007 D5 / ADR 0016 D1)."""
+
+
+class CredentialPolicyError(ValueError):
+    """A strongly-isolated (Tier 2+) sandbox was asked to carry a long-lived
+    credential in its environment, violating INV-ENF-004 (ADR 0016). Raised at
+    provision time — fail closed. The message names the offending variables and
+    the credential kind, never the secret value."""
+
+
+@dataclass
+class GPUAvailability:
+    """GPU availability information from a backend."""
+
+    gpu_type: GPUType
+    available: int
+    total: int
+    price_per_hour: float
+    regions: list[str]
+
+
+@dataclass
+class CostEstimate:
+    """Cost estimate for running a sandbox."""
+
+    backend: BackendType
+    hourly_cost: float
+    estimated_total: float
+    duration_hours: float
+    breakdown: dict[str, float]  # e.g., {"gpu": 2.0, "cpu": 0.5, "storage": 0.1}
+
+
+# ── Backend registry (open-closed: add a provider without editing core) ───────
+# Keyed by string name so a new adapter self-registers — no BackendType enum
+# edit, no Forge if/elif branch. An entry is either a Backend subclass or a lazy
+# "module:ClassName" string (so a built-in like modal doesn't import its SDK
+# until actually used). A third-party/plugin backend calls
+# register_backend("daytona", DaytonaBackend) and becomes usable via
+# Forge(backend="daytona").
+_BACKEND_REGISTRY: dict[str, type[Backend] | str] = {}
+
+
+def register_backend(name, cls=None):  # type: ignore[no-untyped-def]
+    """Register a Backend under ``name`` — a class or a lazy ``"module:Class"``
+    string. Usable as a decorator (``@register_backend("daytona")``) or a call
+    (``register_backend("daytona", DaytonaBackend)``)."""
+
+    def _register(target):  # type: ignore[no-untyped-def]
+        _BACKEND_REGISTRY[name] = target
+        return target
+
+    return _register if cls is None else _register(cls)
+
+
+def get_backend_class(name: str) -> type[Backend]:
+    """Resolve a registered Backend class by name (lazily importing a
+    ``"module:Class"`` entry). Raises ValueError if unknown."""
+    try:
+        entry = _BACKEND_REGISTRY[name]
+    except KeyError:
+        known = ", ".join(sorted(_BACKEND_REGISTRY)) or "(none registered)"
+        raise ValueError(f"unknown backend {name!r}; registered: {known}") from None
+    if isinstance(entry, str):
+        import importlib
+
+        module_name, _, class_name = entry.partition(":")
+        entry = getattr(importlib.import_module(module_name), class_name)
+        _BACKEND_REGISTRY[name] = entry  # cache the resolved class — import once
+    return entry
+
+
+def create_backend(name: str, **kwargs: object) -> Backend:
+    """Instantiate a registered backend by name."""
+    return get_backend_class(name)(**kwargs)
+
+
+def registered_backends() -> list[str]:
+    """Names of all registered backends."""
+    return sorted(_BACKEND_REGISTRY)
+
+
+# Built-in backends, registered lazily (no SDK import until used).
+for _name, _path in {
+    "local": "highflame_forge.backends.local_backend:LocalBackend",
+    "gvisor": "highflame_forge.backends.gvisor_backend:GVisorBackend",
+    "modal": "highflame_forge.backends.modal_backend:ModalBackend",
+    "runpod": "highflame_forge.backends.runpod_backend:RunPodBackend",
+    "gcp": "highflame_forge.backends.gcp_backend:GCPBackend",
+}.items():
+    register_backend(_name, _path)
+
+
+@dataclass(frozen=True)
+class VolumeRecord:
+    """A durable workspace volume, as seen from the control plane."""
+
+    #: Backend-scoped volume name. For Forge-created volumes this is the hash
+    #: of the tenancy tuple (see ``service.volumes.volume_name_for``).
+    name: str
+    #: When this workspace was last launched, read from the ``.forge/last_used``
+    #: marker inside it. ``None`` when the marker is absent or unparseable —
+    #: which ALSO describes a volume Forge did not create, so callers must treat
+    #: None as "not ours / unknown", never as "infinitely old".
+    last_used: datetime | None = None
+    #: The repo this workspace holds, from ``.forge/repo``. Display only —
+    #: ownership is proved by recomputing the name from it, never by trusting
+    #: it. ``None`` when absent.
+    repo: str | None = None
+
+
+class Backend(ABC):
+    """Abstract interface for compute backends.
+
+    Each backend (Modal, RunPod, GCP, Vertex AI) implements this interface
+    to provide a unified API for creating and managing sandboxes.
+    """
+
+    #: Absolute in-sandbox path that is writable, is the agent's working area,
+    #: and is where a persistent volume gets mounted. Declared by the BACKEND
+    #: because only it knows: Modal makes /workspace its scratch and mounts
+    #: volumes there, but another substrate may confine writes elsewhere
+    #: entirely — the local bwrap backend mounts /workspace READ-ONLY, so a
+    #: clone into it fails.
+    #:
+    #: Callers must read this rather than assuming "/workspace". The checkout
+    #: path derives from it, and that path has to stay stable across launches
+    #: of the same repo on the same backend, because harnesses key their
+    #: session state by cwd.
+    workspace_root: str = "/workspace"
+
+    async def list_workspace_volumes(self, prefix: str) -> list[VolumeRecord]:
+        """Durable workspace volumes whose name starts with ``prefix``.
+
+        The prefix is supplied by the caller rather than known here: naming is
+        a service-layer concern (``service.volumes``), and a backend that
+        invented its own would put the two out of step.
+
+        Must be a CONTROL-PLANE listing: the reaper runs on a schedule over
+        every volume in the deployment, so needing a sandbox to inspect one
+        would make garbage collection cost more than the garbage.
+
+        Only called on a backend declaring ``supports_volumes``.
+        """
+        raise NotImplementedError(
+            f"{type(self).__name__} declares supports_volumes but cannot list them"
+        )
+
+    async def reset_volume(self, name: str) -> None:
+        """Destroy the durable volume ``name`` so the next launch starts empty.
+
+        This is what ``--fresh`` means. Only ever called on a backend that
+        declares ``supports_volumes``, and only once the caller has established
+        that nothing is currently attached to it.
+
+        A CONTROL-PLANE delete, deliberately, rather than deleting files from
+        inside a sandbox. Doing it in-band needs compute just to erase, runs
+        shell commands over a mount whose semantics the backend controls (Modal
+        mounts a volume as a SYMLINK, which silently defeats `find`), and can
+        only run AFTER the sandbox has booted — leaving a window in which the
+        agent is already looking at the tree the user asked to destroy.
+        Deleting the volume before provisioning has none of those problems.
+
+        Must be idempotent: ``--fresh`` on a repo that has never been launched
+        is a normal request, not an error.
+        """
+        raise NotImplementedError(
+            f"{type(self).__name__} declares supports_volumes but cannot reset one"
+        )
+
+    #: Whether this backend can mount a durable volume for
+    #: ``SandboxConfig.volume_name`` (forge#56 slice 3). Declared rather than
+    #: inferred so callers can tell "no persistence here" from "persistence
+    #: requested and silently dropped" — the latter would let `--fresh` report
+    #: success while wiping nothing, and let a resume quietly start blank.
+    supports_volumes: bool = False
+
+    def __init__(self, resolver: CapabilityResolver | None = None, **kwargs: object):
+        """Initialize the backend.
+
+        Args:
+            resolver: Capability resolver to derive per-sandbox capabilities from
+                the acting identity. Injected by the service layer with the
+                Cedar-backed resolver in production (ADR 0019). Library/CLI callers
+                (and tests) leave it ``None`` and get the process-wide scope
+                resolver (``DEFAULT_RESOLVER``).
+            **kwargs: Backend-specific configuration.
+        """
+        self.logger = logging.getLogger(f"{__name__}.{self.__class__.__name__}")
+        self._initialized = False
+        #: May be None here; backends that derive capabilities fall back to
+        #: ``DEFAULT_RESOLVER`` at use so no import-time global is captured.
+        self._resolver: CapabilityResolver | None = resolver
+
+    @property
+    @abstractmethod
+    def backend_type(self) -> BackendType:
+        """Return the backend type identifier."""
+        pass
+
+    @property
+    @abstractmethod
+    def is_available(self) -> bool:
+        """Check if the backend is available and properly configured."""
+        pass
+
+    @property
+    def max_isolation(self) -> IsolationTier:
+        """The strongest isolation tier this backend can currently provide.
+
+        Cloud container backends (Modal/RunPod/GCP) are OS-tier equivalent per
+        ADR 0016 — override when a backend provides a kernel or hardware
+        boundary (gVisor → KERNEL, Firecracker/TEE → HARDWARE), or, like the
+        local backend, when the achievable tier depends on the host.
+        """
+        return IsolationTier.OS
+
+    def check_isolation_floor(self, required: IsolationTier) -> None:
+        """Refuse to provision below the required isolation floor (fail closed).
+
+        Called by every backend at sandbox-creation time. Never downgrade:
+        an unavailable tier is a refusal, not a warning (ADR 0007 D5).
+
+        Raises:
+            IsolationUnavailableError: If this backend cannot meet the floor.
+        """
+        if required > self.max_isolation:
+            raise IsolationUnavailableError(
+                f"execution requires isolation tier {required.name} but backend "
+                f"{self.backend_type.value!r} provides at most "
+                f"{self.max_isolation.name} — refusing to run (fail closed; "
+                "use a backend that meets the floor)"
+            )
+
+    def check_credential_policy(self, config: SandboxConfig, tier: IsolationTier) -> None:
+        """Enforce INV-ENF-004: a Tier 2+ sandbox MUST NOT carry a long-lived
+        credential in its declared env/secrets. Called by every backend at
+        creation time, after the isolation floor is resolved.
+
+        Below KERNEL this is a no-op — trusted-local / OS-tier execution on the
+        operator's own machine may legitimately pass a key through. At KERNEL+
+        (untrusted / multi-tenant) a match fails closed: the credential belongs
+        on an SVID-authenticated channel (ADR 0016 D3), not in the environment.
+
+        Raises:
+            CredentialPolicyError: If a long-lived credential shape is found.
+        """
+        if tier < IsolationTier.KERNEL:
+            return
+        from highflame_forge.credentials import find_long_lived_credentials
+
+        findings = find_long_lived_credentials({**config.env, **config.secrets})
+        if findings:
+            offenders = ", ".join(str(f) for f in findings)
+            # INV-ENF-004 — keep the invariant id in code, NOT in the API message.
+            raise CredentialPolicyError(
+                f"tier {tier.name} sandbox may not carry long-lived credentials "
+                f"in its environment: {offenders}. Deliver these over an "
+                "SVID-authenticated channel, or scope them to a short-lived "
+                "(<=15m) token."
+            )
+
+    @abstractmethod
+    async def initialize(self) -> None:
+        """Initialize the backend (authenticate, validate config, etc.)
+
+        This is called lazily on first use.
+
+        Raises:
+            RuntimeError: If initialization fails
+        """
+        pass
+
+    def _effective_floor(self, config: SandboxConfig) -> IsolationTier:
+        """The minimum isolation tier this execution requires. Delegates to the
+        SINGLE shared floor computation so this pre-provision guard and the
+        capability resolver can never disagree (they'd otherwise be two copies of
+        the same logic — a latent way for the guard to under-check)."""
+        from highflame_forge.identity import resolve_isolation_floor
+
+        return resolve_isolation_floor(config, config.identity)
+
+    async def create_sandbox(self, config: SandboxConfig) -> Sandbox:
+        """Create a new sandbox — TEMPLATE METHOD (do not override).
+
+        Runs the fail-closed security guards that apply to EVERY backend, then
+        delegates provisioning to :meth:`_provision`. This makes the guards
+        unskippable by construction: a new adapter implements ``_provision`` and
+        inherits the isolation-floor + credential-policy enforcement — it cannot
+        forget to call them (ADR 0016 D1 / INV-ENF-004 / ADR 0017).
+
+        Raises:
+            IsolationUnavailableError: backend can't meet the required tier.
+            CredentialPolicyError: long-lived credential in a Tier 2+ env.
+        """
+        floor = self._effective_floor(config)
+        self.check_isolation_floor(floor)
+        self.check_credential_policy(config, floor)
+        return await self._provision(config)
+
+    @abstractmethod
+    async def _provision(self, config: SandboxConfig) -> Sandbox:
+        """Provision the sandbox on this backend's substrate. Called by
+        :meth:`create_sandbox` AFTER the universal guards have passed — so an
+        implementation only does provisioning, never security enforcement.
+
+        Args:
+            config: Sandbox configuration (guards already applied)
+
+        Returns:
+            Sandbox handle for the created instance
+        """
+        ...
+
+    @abstractmethod
+    async def get_sandbox(self, sandbox_id: str) -> Sandbox | None:
+        """Get an existing sandbox by ID.
+
+        Args:
+            sandbox_id: Unique sandbox identifier
+
+        Returns:
+            Sandbox handle if found, None otherwise
+        """
+        pass
+
+    @abstractmethod
+    async def list_sandboxes(self) -> list[Sandbox]:
+        """List all active sandboxes for this backend.
+
+        Returns:
+            List of active sandbox handles
+        """
+        pass
+
+    @abstractmethod
+    async def terminate_sandbox(self, sandbox_id: str) -> bool:
+        """Terminate a sandbox by ID.
+
+        Args:
+            sandbox_id: Unique sandbox identifier
+
+        Returns:
+            True if terminated, False if not found
+        """
+        pass
+
+    @abstractmethod
+    def get_available_gpus(self) -> list[GPUAvailability]:
+        """Get list of available GPU types and their availability.
+
+        Returns:
+            List of GPU availability information
+        """
+        pass
+
+    @abstractmethod
+    def estimate_cost(
+        self,
+        config: SandboxConfig,
+        duration_hours: float,
+    ) -> CostEstimate:
+        """Estimate cost for running a sandbox.
+
+        Args:
+            config: Sandbox configuration
+            duration_hours: Expected duration in hours
+
+        Returns:
+            Cost estimate with breakdown
+        """
+        pass
+
+    @abstractmethod
+    def map_gpu_type(self, gpu: GPUType | str) -> str:
+        """Map generic GPU type to backend-specific identifier.
+
+        Args:
+            gpu: Generic GPU type
+
+        Returns:
+            Backend-specific GPU identifier
+        """
+        pass
+
+    async def ensure_initialized(self) -> None:
+        """Ensure backend is initialized, initializing if needed."""
+        if not self._initialized:
+            await self.initialize()
+            self._initialized = True
+
+    def validate_config(self, config: SandboxConfig) -> list[str]:
+        """Validate configuration for this backend.
+
+        Args:
+            config: Sandbox configuration to validate
+
+        Returns:
+            List of validation error messages (empty if valid)
+        """
+        errors = []
+
+        # Check GPU availability
+        if config.gpu and config.gpu != GPUType.NONE:
+            available_gpus = {g.gpu_type for g in self.get_available_gpus()}
+            gpu_type = config.gpu if isinstance(config.gpu, GPUType) else GPUType(config.gpu)
+            if gpu_type not in available_gpus:
+                errors.append(f"GPU type {config.gpu} not available on {self.backend_type.value}")
+
+        # Check memory limits (backend-specific, override in subclass)
+        if config.memory_gb > 1024:
+            errors.append(f"Memory {config.memory_gb}GB exceeds maximum")
+
+        # Check timeout
+        if config.timeout_seconds > 86400:  # 24 hours
+            errors.append("Timeout exceeds maximum of 24 hours")
+
+        return errors
+
+    def __repr__(self) -> str:
+        """String representation."""
+        return f"{self.__class__.__name__}(type={self.backend_type.value})"