ramllm 0.1.0__py3-none-any.whl

ramllm/__init__.py

@@ -0,0 +1,3 @@
+"""RAMLLM: Run bigger models on smaller machines."""
+
+__version__ = "0.1.0"

ramllm/__main__.py

@@ -0,0 +1,6 @@
+"""Main entry point for python -m ramllm."""
+
+from ramllm.cli.main import app
+
+if __name__ == "__main__":
+    app()

ramllm/api/__init__.py

@@ -0,0 +1,5 @@
+"""API package."""
+
+from ramllm.api.server import api_app, create_app
+
+__all__ = ["api_app", "create_app"]

ramllm/api/server.py

@@ -0,0 +1,306 @@
+"""FastAPI server providing OpenAI-compatible endpoints and RAMLLM extensions."""
+
+import asyncio
+from collections.abc import AsyncGenerator
+from contextlib import asynccontextmanager
+from typing import Any
+
+import httpx
+from fastapi import FastAPI, HTTPException, Request, Response
+from fastapi.responses import JSONResponse, StreamingResponse
+from pydantic import BaseModel
+
+from ramllm.backends import LlamaCppAdapter
+from ramllm.backends.base import BackendAdapter, BackendProcess
+from ramllm.core.analyzer.gguf import analyze_gguf
+from ramllm.core.governor.monitor import MemoryGovernor, TierMetrics
+from ramllm.core.governor.recovery import RecoveryExhaustedError, launch_with_recovery
+from ramllm.core.planner import create_execution_plans
+from ramllm.core.profiler import probe_hardware
+from ramllm.core.schemas import ExecutionPlan, HardwareProfile, ModelInfo, UserRequest
+
+
+class ReplanRequest(BaseModel):
+    """Payload for dynamic runtime replanning."""
+
+    vram_budget_bytes: dict[int, int] | None = None
+    ram_budget_bytes: int | None = None
+    context: int | None = None
+    gpu_indices: list[int] | None = None
+    allow_streaming: bool | None = None
+
+
+class ServerState:
+    """Shared state for the RAMLLM API server."""
+
+    def __init__(self) -> None:
+        self.model_path: str = ""
+        self.model_info: ModelInfo | None = None
+        self.hw_profile: HardwareProfile | None = None
+        self.user_request: UserRequest | None = None
+        self.active_plan: ExecutionPlan | None = None
+        self.backend_process: BackendProcess | None = None
+        self.governor: MemoryGovernor | None = None
+        self.internal_port: int = 18080
+        self.adapter: BackendAdapter = LlamaCppAdapter()
+        self.lock: asyncio.Lock = asyncio.Lock()
+        self.replan_pending: bool = False
+        self.auto_retry: bool = True
+        self.max_retries: int = 3
+
+
+state = ServerState()
+
+
+def _handle_server_memory_pressure(metrics: TierMetrics) -> None:
+    """Invoked when memory governor detects external memory pressure during server runtime."""
+    state.replan_pending = True
+
+
+def launch_active_session(auto_retry: bool | None = None) -> None:
+    """Plan and launch the backend session with reactive recovery and memory governor."""
+    if not state.model_info or not state.hw_profile or not state.user_request:
+        raise RuntimeError("Server state not initialized.")
+
+    plans = create_execution_plans(state.model_info, state.hw_profile, state.user_request)
+    chosen = plans[0]
+    if chosen.verdict == "INFEASIBLE":
+        raise ValueError(f"Plan is INFEASIBLE: {'; '.join(chosen.warnings)}")
+
+    use_auto_retry = state.auto_retry if auto_retry is None else auto_retry
+
+    proc, final_plan, _ = launch_with_recovery(
+        adapter=state.adapter,
+        plan=chosen,
+        model=state.model_info,
+        hw=state.hw_profile,
+        req=state.user_request,
+        port=state.internal_port,
+        timeout_s=40.0,
+        max_retries=state.max_retries,
+        auto_retry=use_auto_retry,
+    )
+
+    state.active_plan = final_plan
+    state.backend_process = proc
+    state.governor = MemoryGovernor(
+        proc,
+        state.hw_profile,
+        on_pressure=_handle_server_memory_pressure,
+    )
+    state.governor.start()
+    state.replan_pending = False
+
+
+def shutdown_active_session() -> None:
+    """Stop memory governor and terminate backend engine."""
+    if state.governor:
+        state.governor.stop()
+        state.governor = None
+    if state.backend_process:
+        state.adapter.stop(state.backend_process)
+        state.backend_process = None
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
+    """FastAPI application lifespan managing backend lifecycle."""
+    if state.model_path:
+        launch_active_session()
+    yield
+    shutdown_active_session()
+
+
+api_app = FastAPI(
+    title="RAMLLM Server",
+    description="OpenAI-compatible LLM inference server with strict RAM/VRAM budget governor.",
+    version="0.1.0",
+    lifespan=lifespan,
+)
+
+
+@api_app.get("/healthz")
+async def healthz() -> dict[str, str]:
+    """Liveness check endpoint."""
+    return {"status": "ok"}
+
+
+@api_app.get("/ramllm/plan")
+async def get_plan() -> ExecutionPlan:
+    """Return the currently executing ExecutionPlan."""
+    if not state.active_plan:
+        raise HTTPException(status_code=503, detail="No active plan scheduled.")
+    return state.active_plan
+
+
+@api_app.get("/ramllm/metrics")
+async def get_metrics() -> TierMetrics:
+    """Return real-time per-tier memory metrics and governor warnings."""
+    if not state.governor:
+        raise HTTPException(status_code=503, detail="Governor not running.")
+    return state.governor.get_latest_metrics()
+
+
+@api_app.post("/ramllm/replan")
+async def replan(req: ReplanRequest) -> ExecutionPlan:
+    """Change memory budgets or context at runtime and replan execution."""
+    async with state.lock:
+        if not state.user_request or not state.model_info or not state.hw_profile:
+            raise HTTPException(status_code=503, detail="Server not initialized with model.")
+
+        shutdown_active_session()
+
+        # Update request
+        ctx = req.context or state.user_request.min_context
+        vram_dict = (
+            req.vram_budget_bytes
+            if req.vram_budget_bytes is not None
+            else state.user_request.vram_budget_bytes
+        )
+        ram_bytes = (
+            req.ram_budget_bytes
+            if req.ram_budget_bytes is not None
+            else state.user_request.ram_budget_bytes
+        )
+
+        gpu_idxs = (
+            req.gpu_indices if req.gpu_indices is not None else state.user_request.gpu_indices
+        )
+
+        allow_streaming = (
+            req.allow_streaming
+            if req.allow_streaming is not None
+            else state.user_request.allow_streaming
+        )
+
+        state.user_request = UserRequest(
+            model=state.model_path,
+            vram_budget_bytes=vram_dict,
+            ram_budget_bytes=ram_bytes,
+            min_context=ctx,
+            target_context=ctx,
+            gpu_indices=gpu_idxs,
+            allow_streaming=allow_streaming,
+        )
+
+        try:
+            launch_active_session()
+        except (ValueError, RecoveryExhaustedError) as err:
+            raise HTTPException(
+                status_code=422,
+                detail=f"Cannot replan under requested constraints: {err}",
+            ) from err
+        except Exception as err:
+            raise HTTPException(
+                status_code=500,
+                detail=f"Replan failed unexpectedly: {err}",
+            ) from err
+
+        if not state.active_plan:
+            raise HTTPException(status_code=500, detail="Failed to establish new plan.")
+        return state.active_plan
+
+
+@api_app.get("/v1/models")
+async def list_models_api() -> dict[str, Any]:
+    """OpenAI-compatible models list endpoint."""
+    model_name = state.model_info.arch if state.model_info else "ramllm-model"
+    return {
+        "object": "list",
+        "data": [
+            {
+                "id": model_name,
+                "object": "model",
+                "created": 1700000000,
+                "owned_by": "ramllm",
+            }
+        ],
+    }
+
+
+@api_app.post("/v1/chat/completions")
+async def chat_completions(request: Request) -> Response:
+    """OpenAI-compatible chat completions proxy with streaming SSE support."""
+    body = await request.json()
+    is_stream = body.get("stream", False)
+    target_url = f"http://127.0.0.1:{state.internal_port}/v1/chat/completions"
+
+    client = httpx.AsyncClient(timeout=120.0)
+
+    if is_stream:
+
+        async def event_generator() -> AsyncGenerator[bytes, None]:
+            try:
+                async with client.stream("POST", target_url, json=body) as resp:
+                    async for chunk in resp.aiter_bytes():
+                        yield chunk
+            finally:
+                await client.aclose()
+
+        return StreamingResponse(event_generator(), media_type="text/event-stream")
+    else:
+        try:
+            resp = await client.post(target_url, json=body)
+            return JSONResponse(status_code=resp.status_code, content=resp.json())
+        finally:
+            await client.aclose()
+
+
+@api_app.post("/v1/completions")
+async def completions(request: Request) -> Response:
+    """OpenAI-compatible text completions proxy."""
+    body = await request.json()
+    is_stream = body.get("stream", False)
+    target_url = f"http://127.0.0.1:{state.internal_port}/v1/completions"
+
+    client = httpx.AsyncClient(timeout=120.0)
+
+    if is_stream:
+
+        async def event_generator() -> AsyncGenerator[bytes, None]:
+            try:
+                async with client.stream("POST", target_url, json=body) as resp:
+                    async for chunk in resp.aiter_bytes():
+                        yield chunk
+            finally:
+                await client.aclose()
+
+        return StreamingResponse(event_generator(), media_type="text/event-stream")
+    else:
+        try:
+            resp = await client.post(target_url, json=body)
+            return JSONResponse(status_code=resp.status_code, content=resp.json())
+        finally:
+            await client.aclose()
+
+
+def create_app(
+    model_path: str,
+    vram_budget_bytes: dict[int, int] | None = None,
+    ram_budget_bytes: int | None = None,
+    context: int = 4096,
+    gpu_indices: list[int] | None = None,
+    internal_port: int = 18080,
+    adapter: BackendAdapter | None = None,
+    hw_profile: HardwareProfile | None = None,
+    auto_retry: bool = True,
+    max_retries: int = 3,
+) -> FastAPI:
+    """Configure server state and return FastAPI application instance."""
+    state.model_path = model_path
+    state.internal_port = internal_port
+    state.auto_retry = auto_retry
+    state.max_retries = max_retries
+    if adapter:
+        state.adapter = adapter
+    state.model_info = analyze_gguf(model_path)
+    state.hw_profile = hw_profile or probe_hardware(quick=True)
+    state.user_request = UserRequest(
+        model=model_path,
+        vram_budget_bytes=vram_budget_bytes,
+        ram_budget_bytes=ram_budget_bytes,
+        min_context=context,
+        target_context=context,
+        gpu_indices=gpu_indices,
+    )
+    return api_app

ramllm/backends/__init__.py

@@ -0,0 +1,21 @@
+"""Backends package."""
+
+from ramllm.backends.base import (
+    BackendAdapter,
+    BackendInstall,
+    BackendProcess,
+    Capabilities,
+    MemoryReport,
+)
+from ramllm.backends.llamacpp import LlamaCppAdapter
+from ramllm.backends.mock import MockBackendAdapter
+
+__all__ = [
+    "BackendAdapter",
+    "BackendInstall",
+    "BackendProcess",
+    "Capabilities",
+    "MemoryReport",
+    "LlamaCppAdapter",
+    "MockBackendAdapter",
+]

ramllm/backends/base.py

@@ -0,0 +1,98 @@
+"""Abstract backend interface, capabilities, and process models."""
+
+import subprocess
+from abc import ABC, abstractmethod
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from ramllm.core.schemas import ExecutionPlan
+
+
+class Capabilities(BaseModel):
+    """Features supported by a specific backend installation."""
+
+    model_config = ConfigDict(frozen=True)
+
+    partial_gpu_offload: bool = True
+    per_tensor_placement: bool = False
+    kv_on_gpu_or_cpu_choice: bool = True
+    kv_quantization: bool = True
+    multi_gpu_layer_split: bool = True
+    multi_gpu_tensor_split: bool = False
+    flash_attention: bool = True
+    mmap_control: bool = True
+    async_weight_streaming: bool = False
+    expert_level_eviction: bool = False
+    apis: list[str] = Field(default_factory=list)
+
+
+class BackendInstall(BaseModel):
+    """Metadata describing a discovered backend binary on the system."""
+
+    model_config = ConfigDict(frozen=True)
+
+    name: str
+    binary_path: str
+    version: str | None = None
+    build_info: str | None = None
+    capabilities: Capabilities = Field(default_factory=Capabilities)
+
+
+class MemoryReport(BaseModel):
+    """Reported peak and current memory usage from runtime process sampling and logs."""
+
+    vram_used_bytes: dict[int, int] = Field(default_factory=dict)
+    ram_rss_bytes: int = 0
+    model_buffer_bytes: int = 0
+    kv_buffer_bytes: int = 0
+    compute_buffer_bytes: int = 0
+
+
+class BackendProcess(BaseModel):
+    """Handle to a running backend server process."""
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    pid: int
+    port: int
+    process: subprocess.Popen[str] | None = None
+    plan: ExecutionPlan
+    initial_memory_report: MemoryReport = Field(default_factory=MemoryReport)
+
+
+class BackendAdapter(ABC):
+    """Abstract interface for all RAMLLM execution engines."""
+
+    name: str
+
+    @abstractmethod
+    def detect(self) -> BackendInstall | None:
+        """Discover backend binary on system PATH or configured locations."""
+        ...
+
+    @abstractmethod
+    def capabilities(self, install: BackendInstall) -> Capabilities:
+        """Inspect backend capabilities from its binary help output."""
+        ...
+
+    @abstractmethod
+    def build_args(self, plan: ExecutionPlan, port: int = 8080) -> list[str]:
+        """Convert an ExecutionPlan into concrete command-line arguments."""
+        ...
+
+    @abstractmethod
+    def launch(
+        self, plan: ExecutionPlan, port: int = 8080, timeout_s: float = 30.0
+    ) -> BackendProcess:
+        """Launch the backend process, await healthiness, and extract parsed buffer sizes."""
+        ...
+
+    @abstractmethod
+    def memory_report(self, proc: BackendProcess) -> MemoryReport:
+        """Collect current memory consumption across hardware tiers."""
+        ...
+
+    @abstractmethod
+    def stop(self, proc: BackendProcess) -> None:
+        """Gracefully terminate the backend process."""
+        ...

ramllm/backends/llamacpp/__init__.py

@@ -0,0 +1,7 @@
+"""llama.cpp backend package."""
+
+from ramllm.backends.llamacpp.adapter import LlamaCppAdapter
+from ramllm.backends.llamacpp.flags import build_llamacpp_args
+from ramllm.backends.llamacpp.install import detect_llamacpp
+
+__all__ = ["LlamaCppAdapter", "detect_llamacpp", "build_llamacpp_args"]

ramllm/backends/llamacpp/adapter.py

@@ -0,0 +1,75 @@
+"""Llama.cpp Backend Adapter implementation."""
+
+import psutil
+
+from ramllm.backends.base import (
+    BackendAdapter,
+    BackendInstall,
+    BackendProcess,
+    Capabilities,
+    MemoryReport,
+)
+from ramllm.backends.llamacpp.flags import build_llamacpp_args
+from ramllm.backends.llamacpp.install import detect_llamacpp
+from ramllm.backends.llamacpp.launcher import launch_llamacpp_server
+from ramllm.core.schemas import ExecutionPlan
+
+
+class LlamaCppAdapter(BackendAdapter):
+    """Execution engine adapter for llama.cpp via managed llama-server subprocess."""
+
+    name = "llamacpp"
+
+    def __init__(self, binary_path: str | None = None):
+        self.binary_path = binary_path
+        self._install: BackendInstall | None = None
+
+    def detect(self) -> BackendInstall | None:
+        if self.binary_path:
+            self._install = detect_llamacpp(self.binary_path)
+        else:
+            from ramllm.services.backend_manager import discover_backend
+
+            res = discover_backend()
+            self._install = res.install if res else None
+        return self._install
+
+    def capabilities(self, install: BackendInstall) -> Capabilities:
+        return install.capabilities
+
+    def build_args(self, plan: ExecutionPlan, port: int = 8080) -> list[str]:
+        if not self._install:
+            self.detect()
+        return build_llamacpp_args(plan, install=self._install, port=port)
+
+    def launch(
+        self, plan: ExecutionPlan, port: int = 8080, timeout_s: float = 30.0
+    ) -> BackendProcess:
+        cmd = self.build_args(plan, port=port)
+        return launch_llamacpp_server(cmd, plan=plan, port=port, timeout_s=timeout_s)
+
+    def memory_report(self, proc: BackendProcess) -> MemoryReport:
+        rss = 0
+        try:
+            p = psutil.Process(proc.pid)
+            rss = p.memory_info().rss
+            for child in p.children(recursive=True):
+                rss += child.memory_info().rss
+        except Exception:
+            rss = proc.initial_memory_report.ram_rss_bytes
+
+        return MemoryReport(
+            vram_used_bytes=proc.plan.memory.vram_peak_bytes,
+            ram_rss_bytes=rss,
+            model_buffer_bytes=proc.initial_memory_report.model_buffer_bytes,
+            kv_buffer_bytes=proc.initial_memory_report.kv_buffer_bytes,
+            compute_buffer_bytes=proc.initial_memory_report.compute_buffer_bytes,
+        )
+
+    def stop(self, proc: BackendProcess) -> None:
+        if proc.process and proc.process.poll() is None:
+            proc.process.terminate()
+            try:
+                proc.process.wait(timeout=3)
+            except Exception:
+                proc.process.kill()

ramllm/backends/llamacpp/flags.py

@@ -0,0 +1,78 @@
+"""Flag translation from ExecutionPlan to llama-server CLI arguments."""
+
+from ramllm.backends.base import BackendInstall
+from ramllm.core.schemas import ExecutionPlan
+
+
+def build_llamacpp_args(
+    plan: ExecutionPlan,
+    install: BackendInstall | None = None,
+    host: str = "127.0.0.1",
+    port: int = 8080,
+) -> list[str]:
+    """Map an ExecutionPlan to concrete arguments for llama-server."""
+    binary = install.binary_path if install else "llama-server"
+    args = [binary, "--host", host, "--port", str(port)]
+
+    # Extract model path from backend_args in plan
+    model_path = ""
+    for i, a in enumerate(plan.backend_args):
+        if a in ("-m", "--model") and i + 1 < len(plan.backend_args):
+            model_path = plan.backend_args[i + 1]
+            break
+
+    if model_path:
+        args.extend(["-m", model_path])
+
+    # GPU layers
+    gpu_layer_count = sum(len(layer_list) for layer_list in plan.placement.gpu_layers.values())
+    args.extend(["-ngl", str(gpu_layer_count)])
+
+    # Context size
+    args.extend(["-c", str(plan.kv.context)])
+
+    # Predictable single-slot memory usage
+    args.extend(["-np", "1"])
+
+    # KV quantization
+    if plan.kv.type_k != "f16":
+        args.extend(["-ctk", plan.kv.type_k])
+    if plan.kv.type_v != "f16":
+        args.extend(["-ctv", plan.kv.type_v])
+
+    # KV offload to CPU if requested
+    if plan.kv.location == "cpu":
+        args.append("--no-kv-offload")
+
+    # Multi-GPU split
+    if len(plan.placement.gpu_layers) > 1:
+        total_offloaded = sum(len(layers) for layers in plan.placement.gpu_layers.values())
+        if total_offloaded > 0:
+            gpu_indices = sorted(plan.placement.gpu_layers.keys())
+            max_gpu_idx = max(gpu_indices)
+            fractions = [
+                f"{len(plan.placement.gpu_layers.get(idx, [])) / total_offloaded:.2f}"
+                for idx in range(max_gpu_idx + 1)
+            ]
+            main_gpu = gpu_indices[0]
+            for i, a in enumerate(plan.backend_args):
+                if a in ("-mg", "--main-gpu") and i + 1 < len(plan.backend_args):
+                    main_gpu = int(plan.backend_args[i + 1])
+                    break
+            args.extend(["-sm", "layer", "-ts", ",".join(fractions), "-mg", str(main_gpu)])
+
+    # MoE CPU offload flags
+    if plan.placement.expert_placement == "cpu":
+        args.append("-cmoe")
+
+    # Partial MoE CPU offload (-ncmoe)
+    for i, a in enumerate(plan.backend_args):
+        if a in ("-ncmoe", "--n-cpu-moe") and i + 1 < len(plan.backend_args):
+            args.extend(["-ncmoe", plan.backend_args[i + 1]])
+            break
+
+    # Disk streaming / no-mmap
+    if plan.placement.disk_streamed_layers:
+        args.extend(["-lm", "none"])
+
+    return args

ramllm/backends/llamacpp/install.py

@@ -0,0 +1,75 @@
+"""Discovery and capability introspection for llama.cpp installations."""
+
+import shutil
+import subprocess
+
+from ramllm.backends.base import BackendInstall, Capabilities
+
+
+def detect_llamacpp(custom_path: str | None = None) -> BackendInstall | None:
+    """Discover llama-server binary and introspect its capabilities.
+
+    Returns None if no working llama-server executable is found.
+    """
+    binary = custom_path or shutil.which("llama-server") or shutil.which("llama-cli")
+    if not binary:
+        return None
+
+    # Check version
+    version_str: str | None = None
+    build_info: str | None = None
+    try:
+        ver_res = subprocess.run([binary, "--version"], capture_output=True, text=True, timeout=5)
+        out = ver_res.stdout + ver_res.stderr
+        for line in out.splitlines():
+            if "version:" in line.lower() or "build" in line.lower():
+                version_str = line.strip()
+            if "built with" in line.lower():
+                build_info = line.strip()
+    except Exception:
+        pass
+
+    # Check help output for supported flags
+    help_text = ""
+    try:
+        help_res = subprocess.run([binary, "--help"], capture_output=True, text=True, timeout=5)
+        help_text = help_res.stdout + help_res.stderr
+    except Exception:
+        pass
+
+    # Derive capabilities from help text
+    has_ngl = "-ngl" in help_text or "--gpu-layers" in help_text
+    has_cpu_moe = "-cmoe" in help_text or "--cpu-moe" in help_text
+    has_kv_quant = "-ctk" in help_text or "--cache-type-k" in help_text
+    has_override_tensor = "-ot" in help_text or "--override-tensor" in help_text
+    has_split_mode = "-sm" in help_text or "--split-mode" in help_text
+    has_flash_attn = "-fa" in help_text or "--flash-attn" in help_text
+
+    apis: list[str] = []
+    if "metal" in help_text.lower() or "metal" in (build_info or "").lower():
+        apis.append("metal")
+    if "cuda" in help_text.lower() or "cuda" in (build_info or "").lower():
+        apis.append("cuda")
+    if "vulkan" in help_text.lower():
+        apis.append("vulkan")
+    if "rocm" in help_text.lower():
+        apis.append("rocm")
+
+    caps = Capabilities(
+        partial_gpu_offload=has_ngl,
+        per_tensor_placement=has_override_tensor or has_cpu_moe,
+        kv_on_gpu_or_cpu_choice=True,
+        kv_quantization=has_kv_quant,
+        multi_gpu_layer_split=has_split_mode,
+        flash_attention=has_flash_attn,
+        mmap_control=True,
+        apis=apis,
+    )
+
+    return BackendInstall(
+        name="llamacpp",
+        binary_path=binary,
+        version=version_str,
+        build_info=build_info,
+        capabilities=caps,
+    )