inline-core 1.1.1__py3-none-any.whl

inline_core/__init__.py

@@ -0,0 +1,7 @@
+"""Inline Core: the generation engine behind Inline.
+
+Takes a typed node graph and returns immutable takes. See PLAN.md for the architecture and
+docs/contract.md for the Storyline API.
+"""
+
+__version__ = "0.0.0"

inline_core/components/__init__.py

@@ -0,0 +1 @@
+"""Swappable model components, each behind a narrow interface and registered by name."""

inline_core/components/conditioning.py

@@ -0,0 +1,20 @@
+"""Opaque conditioning and latents. Conditioning is family-defined; the graph never inspects it."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    import torch
+
+
+class Conditioning:
+    """Opaque output of a TextEncoder, consumed by a Denoiser. No fixed tensor shape."""
+
+
+@dataclass
+class Latents:
+    """A latent tensor. Its device and dtype live on the tensor, set by the policy at load."""
+
+    tensor: torch.Tensor

inline_core/components/interfaces.py

@@ -0,0 +1,91 @@
+"""The five component interfaces. Device- and dtype-agnostic; placement comes from the context.
+
+The Sampler carries an on_step callback from day one so progress streams without retrofitting.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from collections.abc import Callable, Sequence
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any
+
+from .conditioning import Conditioning, Latents
+
+if TYPE_CHECKING:
+    import torch
+
+    from ..runtime.context import ExecutionContext
+
+
+@dataclass(frozen=True)
+class StepInfo:
+    """Reported once per denoise step so the executor can stream sample progress."""
+
+    step: int
+    total: int
+
+
+StepCallback = Callable[[StepInfo], None]
+
+
+class TextEncoder(ABC):
+    """prompt -> opaque Conditioning."""
+
+    @abstractmethod
+    def encode(
+        self, prompt: str, params: dict[str, Any], ctx: ExecutionContext
+    ) -> Conditioning: ...
+
+
+class Scheduler(ABC):
+    """Owns the timesteps and the prediction parameterization."""
+
+    @abstractmethod
+    def timesteps(self, steps: int) -> Sequence[torch.Tensor]: ...
+
+    @abstractmethod
+    def scale_model_input(self, latents: Latents, t: torch.Tensor) -> Latents: ...
+
+    @abstractmethod
+    def step(self, model_output: torch.Tensor, t: torch.Tensor, latents: Latents) -> Latents: ...
+
+
+class Denoiser(ABC):
+    """(latents, timestep, conditioning) -> predicted output. The heavy model."""
+
+    @abstractmethod
+    def predict(
+        self,
+        latents: Latents,
+        timestep: torch.Tensor,
+        conditioning: Conditioning,
+        ctx: ExecutionContext,
+    ) -> torch.Tensor: ...
+
+
+class Sampler(ABC):
+    """The stepping loop, independent of the denoiser. Reports each step via on_step."""
+
+    @abstractmethod
+    def sample(
+        self,
+        denoiser: Denoiser,
+        scheduler: Scheduler,
+        latents: Latents,
+        conditioning: Conditioning,
+        *,
+        steps: int,
+        ctx: ExecutionContext,
+        on_step: StepCallback | None = None,
+    ) -> Latents: ...
+
+
+class VAE(ABC):
+    """Pixel and latent conversion, both directions."""
+
+    @abstractmethod
+    def encode(self, image: torch.Tensor, ctx: ExecutionContext) -> Latents: ...
+
+    @abstractmethod
+    def decode(self, latents: Latents, ctx: ExecutionContext) -> torch.Tensor: ...

inline_core/config.py

@@ -0,0 +1,33 @@
+"""Engine configuration from the environment. Small and explicit."""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+
+def models_dir() -> Path:
+    """The models root scanned on start (category subfolders inside). `INLINE_MODELS_DIR`, else
+    `./models`. Users drop their own weight files here; nothing is downloaded."""
+    env = os.environ.get("INLINE_MODELS_DIR")
+    return Path(env).expanduser() if env else Path("models")
+
+
+def data_dir() -> Path:
+    """Engine-owned working data (the run DB, takes). `INLINE_DATA_DIR`, else `./.inline`."""
+    env = os.environ.get("INLINE_DATA_DIR")
+    return Path(env).expanduser() if env else Path(".inline")
+
+
+def server_host() -> str:
+    """Address the /v1 server binds. `INLINE_HOST`, else loopback (`127.0.0.1`)."""
+    return os.environ.get("INLINE_HOST", "127.0.0.1")
+
+
+def server_port() -> int:
+    """Port the /v1 server binds. `INLINE_PORT`, else 8848; a non-numeric value falls back."""
+    raw = os.environ.get("INLINE_PORT", "")
+    try:
+        return int(raw)
+    except ValueError:
+        return 8848

inline_core/device/__init__.py

@@ -0,0 +1 @@
+"""Device and memory policy: the single owner of placement, dtype, offload, and attention."""

inline_core/device/auto.py

@@ -0,0 +1,35 @@
+"""A minimal default policy: whole graph on the best device, fp16 on cuda, fp32 elsewhere, SDPA.
+
+This is the always-works path (PLAN.md). Phase 3 replaces it with real budgets, offload, profiles.
+"""
+
+from __future__ import annotations
+
+from .detect import available_device
+from .policy import AttentionBackend, DevicePolicy, Placement, Profile
+from .types import Device, DeviceKind, DType
+
+_PROFILE = {
+    DeviceKind.CUDA: Profile.GPU_MAX,
+    DeviceKind.MPS: Profile.GPU_MAX,
+    DeviceKind.CPU: Profile.CPU,
+}
+
+
+class AutoDevicePolicy(DevicePolicy):
+    def __init__(self, device: Device | None = None) -> None:
+        self._device = device or available_device()
+
+    @property
+    def profile(self) -> Profile:
+        return _PROFILE[self._device.kind]
+
+    def placement(self, role: str) -> Placement:
+        dtype = DType.FP16 if self._device.kind is DeviceKind.CUDA else DType.FP32
+        return Placement(self._device, dtype)
+
+    def attention_backend(self) -> AttentionBackend:
+        return AttentionBackend.SDPA
+
+    def vae_tiling(self) -> bool:
+        return self._device.kind is DeviceKind.CPU

inline_core/device/detect.py

@@ -0,0 +1,41 @@
+"""Best-available device detection. Torch is imported lazily so the core imports without it."""
+
+from __future__ import annotations
+
+from .types import Device, DeviceKind
+
+
+def available_device() -> Device:
+    """The single best device: cuda:0, else mps, else cpu."""
+    return available_devices()[0]
+
+
+def available_devices() -> tuple[Device, ...]:
+    """Every usable device: all CUDA GPUs, else one MPS, else one CPU."""
+    try:
+        import torch
+    except ModuleNotFoundError:
+        return (Device(DeviceKind.CPU),)
+    if torch.cuda.is_available():
+        return tuple(Device(DeviceKind.CUDA, i) for i in range(torch.cuda.device_count()))
+    if torch.backends.mps.is_available():
+        return (Device(DeviceKind.MPS),)
+    return (Device(DeviceKind.CPU),)
+
+
+def has_nvlink() -> bool:
+    """Best-effort: True only when NVLink is confirmed between two GPUs. Conservative (False) on any
+    uncertainty, so the policy defaults to the PCIe-friendly PipeFusion split."""
+    try:
+        import pynvml
+
+        pynvml.nvmlInit()
+        try:
+            if pynvml.nvmlDeviceGetCount() < 2:
+                return False
+            handle = pynvml.nvmlDeviceGetHandleByIndex(0)
+            return bool(pynvml.nvmlDeviceGetNvLinkState(handle, 0))
+        finally:
+            pynvml.nvmlShutdown()
+    except Exception:
+        return False

inline_core/device/memory.py

@@ -0,0 +1,168 @@
+"""The memory-aware device policy: measure RAM/VRAM, pick a profile, own dtype/offload/quant/tiling.
+
+Profiles: gpu-max (ample VRAM), lowvram (tight VRAM), cpu (fp32, tiling, int8 to fit RAM). We always
+prefer the GPU: even under lowvram, weights stay resident on the GPU (tiling + attention slicing +
+int8 do the memory saving) and we do NOT auto-offload to CPU — offloading is slow and defeats "use
+the GPU we have". Set INLINE_ALLOW_CPU_OFFLOAD=1 to opt back into streaming modules on/off the GPU
+for extreme cases. Override the profile/budget with INLINE_PROFILE and INLINE_VRAM_BUDGET_GB.
+Detection is lazy so the core imports without torch or psutil; an unavailable measurement keeps the
+policy conservative.
+"""
+
+from __future__ import annotations
+
+import os
+
+from .detect import available_devices, has_nvlink
+from .policy import AttentionBackend, DevicePolicy, Parallel, Placement, Profile, Quantization
+from .types import Device, DeviceKind, DType
+
+_GPU_MAX_MIN_VRAM_GB = 16.0  # at or above -> gpu-max, else lowvram
+_QUANT_VRAM_GB = 10.0  # lowvram below this -> int8
+_QUANT_RAM_GB = 48.0  # cpu below this -> int8
+
+
+def _system_ram_gb() -> float | None:
+    try:
+        return os.sysconf("SC_PHYS_PAGES") * os.sysconf("SC_PAGE_SIZE") / 1e9
+    except (ValueError, OSError, AttributeError):
+        pass
+    try:
+        import psutil
+
+        return psutil.virtual_memory().total / 1e9
+    except ModuleNotFoundError:
+        return None
+
+
+def _vram_gb(device: Device) -> float | None:
+    if device.kind is not DeviceKind.CUDA:
+        return None
+    try:
+        import torch
+
+        return torch.cuda.mem_get_info(device.index)[1] / 1e9
+    except Exception:
+        return None
+
+
+def _env_profile() -> Profile | None:
+    value = os.environ.get("INLINE_PROFILE", "").strip().lower()
+    return next((p for p in Profile if p.value == value), None)
+
+
+def _env_budget() -> float | None:
+    value = os.environ.get("INLINE_VRAM_BUDGET_GB", "").strip()
+    try:
+        return float(value) if value else None
+    except ValueError:
+        return None
+
+
+def _env_allow_offload() -> bool:
+    """Opt back into CPU offload under lowvram. Off by default: we keep weights on the GPU."""
+    value = os.environ.get("INLINE_ALLOW_CPU_OFFLOAD", "").strip().lower()
+    return value in ("1", "true", "yes", "on")
+
+
+def _env_parallel() -> Parallel | None:
+    """Parse INLINE_PARALLEL like `pipefusion=2,ulysses=2` into a degree spec."""
+    value = os.environ.get("INLINE_PARALLEL", "").strip()
+    if not value:
+        return None
+    valid = {"pipefusion", "ulysses", "ring", "cfg", "tensor"}
+    degrees: dict[str, int] = {}
+    for part in value.split(","):
+        key, _, raw = part.partition("=")
+        if key.strip() in valid:
+            try:
+                degrees[key.strip()] = int(raw)
+            except ValueError:
+                continue
+    return Parallel(**degrees) if degrees else None
+
+
+class MemoryPolicy(DevicePolicy):
+    def __init__(
+        self,
+        device: Device | None = None,
+        *,
+        ram_gb: float | None = None,
+        vram_gb: float | None = None,
+        profile: Profile | None = None,
+        devices: tuple[Device, ...] | None = None,
+        nvlink: bool | None = None,
+        parallel: Parallel | None = None,
+        allow_offload: bool | None = None,
+    ) -> None:
+        self._devices = devices if devices is not None else available_devices()
+        self._device = device or self._devices[0]
+        self._ram_gb = ram_gb if ram_gb is not None else _system_ram_gb()
+        self._vram_gb = vram_gb if vram_gb is not None else _vram_gb(self._device)
+        self._profile = profile or _env_profile() or self._choose_profile()
+        self._nvlink = nvlink if nvlink is not None else has_nvlink()
+        self._parallel = parallel if parallel is not None else _env_parallel()
+        self._allow_offload = allow_offload if allow_offload is not None else _env_allow_offload()
+
+    def _choose_profile(self) -> Profile:
+        if self._device.kind is DeviceKind.CPU:
+            return Profile.CPU
+        if self._device.kind is DeviceKind.MPS:
+            return Profile.GPU_MAX  # unified memory; offload semantics differ
+        budget = _env_budget() or self._vram_gb
+        if budget is not None and budget < _GPU_MAX_MIN_VRAM_GB:
+            return Profile.LOWVRAM
+        return Profile.GPU_MAX
+
+    @property
+    def profile(self) -> Profile:
+        return self._profile
+
+    def placement(self, role: str) -> Placement:
+        if self._profile is Profile.CPU:
+            return Placement(self._device, DType.FP32)
+        # Always prefer the GPU: even under lowvram, keep weights resident (tiling/slicing/int8 save
+        # memory instead). Only stream modules to CPU when explicitly opted in via env.
+        offload = self._profile is Profile.LOWVRAM and self._allow_offload
+        if role == "denoiser":
+            parallel = self._denoiser_parallel()
+            if parallel is not None:
+                cuda = tuple(d for d in self._devices if d.kind is DeviceKind.CUDA)
+                return Placement(
+                    self._device,
+                    DType.BF16,
+                    offload=offload,
+                    devices=cuda[: parallel.world_size],
+                    parallel=parallel,
+                )
+        return Placement(self._device, DType.BF16, offload=offload)
+
+    def _denoiser_parallel(self) -> Parallel | None:
+        """Split the denoiser across GPUs when there are 2+. An explicit override wins; else auto:
+        PipeFusion on PCIe (no NVLink), sequence-parallel (Ulysses) on NVLink."""
+        if self._parallel is not None:
+            return self._parallel if self._parallel.world_size > 1 else None
+        cuda = [d for d in self._devices if d.kind is DeviceKind.CUDA]
+        if len(cuda) < 2:
+            return None
+        return Parallel(ulysses=len(cuda)) if self._nvlink else Parallel(pipefusion=len(cuda))
+
+    def attention_backend(self) -> AttentionBackend:
+        return AttentionBackend.SDPA
+
+    def vae_tiling(self) -> bool:
+        return self._profile in (Profile.LOWVRAM, Profile.CPU)
+
+    def attention_slicing(self) -> bool:
+        return self._profile in (Profile.LOWVRAM, Profile.CPU)
+
+    def quantization(self) -> Quantization:
+        if self._profile is Profile.LOWVRAM and _below(self._vram_gb, _QUANT_VRAM_GB):
+            return Quantization.INT8
+        if self._profile is Profile.CPU and _below(self._ram_gb, _QUANT_RAM_GB):
+            return Quantization.INT8
+        return Quantization.NONE
+
+
+def _below(measured: float | None, threshold: float) -> bool:
+    return measured is not None and measured < threshold

inline_core/device/policy.py

@@ -0,0 +1,82 @@
+"""The device and memory policy interface. Components never self-assign a device; they ask here."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from enum import Enum
+
+from .types import Device, DType
+
+
+class Profile(str, Enum):
+    GPU_MAX = "gpu-max"
+    LOWVRAM = "lowvram"
+    CPU = "cpu"
+
+
+class AttentionBackend(str, Enum):
+    FLASH = "flash"
+    XFORMERS = "xformers"
+    SDPA = "sdpa"
+
+
+class Quantization(str, Enum):
+    NONE = "none"
+    INT8 = "int8"  # torch-native weight-only, portable
+    NF4 = "nf4"  # bitsandbytes, cuda-only
+
+
+@dataclass(frozen=True)
+class Parallel:
+    """How the denoiser is split across GPUs (xDiT degrees). Product of degrees = world size (GPUs).
+    PipeFusion is PCIe-friendly; Ulysses/Ring want NVLink; CFG applies only to guided models."""
+
+    pipefusion: int = 1
+    ulysses: int = 1
+    ring: int = 1
+    cfg: int = 1
+    tensor: int = 1
+
+    @property
+    def world_size(self) -> int:
+        return self.pipefusion * self.ulysses * self.ring * self.cfg * self.tensor
+
+
+@dataclass(frozen=True)
+class Placement:
+    """Where and how a component runs. Chosen by the policy, never by the component."""
+
+    device: Device
+    dtype: DType
+    offload: bool = False
+    # Multi-GPU (denoiser only): the device group + how it is split. Empty/None = single device.
+    devices: tuple[Device, ...] = ()
+    parallel: Parallel | None = None
+
+
+class DevicePolicy(ABC):
+    """Owns dtype, device, offload, attention backend, and tiling for a worker."""
+
+    @property
+    @abstractmethod
+    def profile(self) -> Profile: ...
+
+    @abstractmethod
+    def placement(self, role: str) -> Placement:
+        """Placement for a component role: text_encoder, denoiser, vae, and so on."""
+
+    @abstractmethod
+    def attention_backend(self) -> AttentionBackend: ...
+
+    @abstractmethod
+    def vae_tiling(self) -> bool:
+        """Whether to tile VAE decode to cap peak memory."""
+
+    def attention_slicing(self) -> bool:
+        """Whether to slice attention to cap peak memory. Default off."""
+        return False
+
+    def quantization(self) -> Quantization:
+        """Weight quantization to fit low memory. Default none."""
+        return Quantization.NONE

inline_core/device/types.py

@@ -0,0 +1,31 @@
+"""Device and dtype abstractions. Kept independent of torch so nothing load-bearing needs it."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import Enum
+
+
+class DeviceKind(str, Enum):
+    CUDA = "cuda"
+    MPS = "mps"
+    CPU = "cpu"
+
+
+class DType(str, Enum):
+    FP32 = "fp32"
+    FP16 = "fp16"
+    BF16 = "bf16"
+    FP8 = "fp8"
+    INT8 = "int8"
+
+
+@dataclass(frozen=True)
+class Device:
+    kind: DeviceKind
+    index: int = 0
+
+    def __str__(self) -> str:
+        if self.kind is DeviceKind.CPU:
+            return self.kind.value
+        return f"{self.kind.value}:{self.index}"

inline_core/errors.py

@@ -0,0 +1,42 @@
+"""Typed engine errors. Fail loudly with a clear message; never catch and hide."""
+
+from __future__ import annotations
+
+
+class InlineCoreError(Exception):
+    """Base for all engine errors."""
+
+
+class GraphValidationError(InlineCoreError):
+    """A graph failed validation before execution. Carries the offending node and port."""
+
+    def __init__(
+        self, message: str, *, node_id: str | None = None, port: str | None = None
+    ) -> None:
+        super().__init__(message)
+        self.node_id = node_id
+        self.port = port
+
+
+class UnknownNodeType(GraphValidationError):
+    """A node references a type absent from the registry."""
+
+
+class PortTypeError(GraphValidationError):
+    """An edge connects incompatible port kinds, or a required port is unwired."""
+
+
+class CycleError(GraphValidationError):
+    """The graph is not acyclic."""
+
+
+class DeviceError(InlineCoreError):
+    """A device or placement could not be satisfied."""
+
+
+class ComponentError(InlineCoreError):
+    """A component failed to load or execute."""
+
+
+class CancelledError(InlineCoreError):
+    """A run was cancelled cooperatively via its cancel token."""

inline_core/graph/__init__.py

@@ -0,0 +1 @@
+"""The graph engine: parse JSON to a typed DAG, validate, cache by content, and execute lazily."""

inline_core/graph/cache.py

@@ -0,0 +1,83 @@
+"""The content-hash node cache. Identity = (type, canonical params, upstream keys, asset content).
+
+Determinism rules from docs/contract.md section 4, including seed-based cache eligibility.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from abc import ABC, abstractmethod
+from typing import Any
+
+from ..takes import Take
+from .registry import Registry
+from .schema import Graph, Node
+
+
+class NodeCache(ABC):
+    @abstractmethod
+    def get(self, key: str) -> list[Take] | None: ...
+
+    @abstractmethod
+    def put(self, key: str, takes: list[Take]) -> None: ...
+
+
+class InMemoryCache(NodeCache):
+    def __init__(self) -> None:
+        self._store: dict[str, list[Take]] = {}
+
+    def get(self, key: str) -> list[Take] | None:
+        return self._store.get(key)
+
+    def put(self, key: str, takes: list[Take]) -> None:
+        self._store[key] = list(takes)
+
+
+def _canonical_params(node: Node, registry: Registry) -> dict[str, Any]:
+    merged = {**registry.get(node.type).defaults(), **node.params}
+    return {key: merged[key] for key in sorted(merged)}
+
+
+def is_cache_eligible(node: Node, registry: Registry) -> bool:
+    """False when any seed param resolves to a negative (random) value."""
+    descriptor = registry.get(node.type)
+    defaults = descriptor.defaults()
+    for key in descriptor.seed_keys():
+        value = node.params.get(key, defaults.get(key))
+        try:
+            if int(value) < 0:
+                return False
+        except (TypeError, ValueError):
+            return False
+    return True
+
+
+def node_cache_key(
+    graph: Graph,
+    node_id: str,
+    registry: Registry,
+    asset_hashes: dict[str, str],
+    _memo: dict[str, str] | None = None,
+) -> str:
+    """A stable content hash for a node's output. Asset refs contribute their byte hash."""
+    memo = _memo if _memo is not None else {}
+    if node_id in memo:
+        return memo[node_id]
+    node = graph.node(node_id)
+    upstream = {
+        port_id: [
+            [edge.output, node_cache_key(graph, edge.from_node, registry, asset_hashes, memo)]
+            for edge in edges
+        ]
+        for port_id, edges in sorted(node.inputs.items())
+    }
+    payload = {
+        "type": node.type,
+        "params": _canonical_params(node, registry),
+        "inputs": upstream,
+        "asset": asset_hashes.get(node_id),
+    }
+    digest = hashlib.sha256(json.dumps(payload, sort_keys=True, default=str).encode()).hexdigest()
+    memo[node_id] = digest
+    return digest