lcode-cli 0.15.0__py3-none-any.whl → 0.16.0__py3-none-any.whl

lcode/__init__.py

@@ -1,3 +1,3 @@
 """lcode — a local-first terminal coding agent powered by open-weight models via Ollama."""
 
-__version__ = "0.15.0"
+__version__ = "0.16.0"

lcode/agent.py

@@ -570,7 +570,8 @@ class Agent:
                 self.console.print(f"[dim]{self.settings.model} does not support reasoning; continuing without.[/]")
                 yield from self.chat(messages, tools, False)
                 return
-            if any(marker in error for marker in GPU_MEMORY_ERRORS):
+            # Only Ollama loads the model with lcode's settings; other servers manage their own memory.
+            if getattr(self.ollama, "kind", "ollama") == "ollama" and any(m in error for m in GPU_MEMORY_ERRORS):
                 batch = self.settings.num_batch
                 if not started and batch and batch > SAFE_NUM_BATCH:
                     # Larger batches read prompts faster but need extra VRAM that isn't always free.

lcode/backends.py

@@ -0,0 +1,430 @@
+"""Model servers other than Ollama: anything with an OpenAI-compatible API.
+
+LM Studio, llama.cpp's llama-server, vLLM and MLX servers (mlx_lm.server, LM Studio's MLX engine)
+all speak the OpenAI chat completions API, with streaming and tool calls. `OpenAICompatible` gives
+lcode the same interface as its Ollama client, translating on the way: tool calls get ids and
+JSON-string arguments, streamed tool calls arrive in pieces, reasoning text comes as
+`reasoning_content`, and Ollama's `format` becomes `response_format`.
+
+What only Ollama can do degrades cleanly: lcode can't unload the model to free the GPU, can't ask
+for a context size (the server has its own; lcode reads it where the server says), and doesn't know
+whether a model can see images unless you set `vision_model`.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import time
+from collections.abc import Callable, Iterator
+from urllib.parse import quote
+
+import requests
+
+from lcode.ollama import OllamaError, redact
+
+PRESETS = {  # backend setting -> (the server's name, its default address)
+    "lmstudio": ("LM Studio", "http://localhost:1234/v1"),
+    "llama.cpp": ("llama-server", "http://localhost:8080/v1"),
+    "vllm": ("vLLM", "http://localhost:8000/v1"),
+    "mlx": ("MLX server", "http://localhost:8080/v1"),
+    "openai": ("OpenAI-compatible server", ""),
+}
+BACKENDS = ("ollama", *PRESETS)
+DEFAULT_CONTEXT = 32768  # when the server doesn't say its context size
+EMBEDDING_HINTS = ("embed", "bge-", "minilm", "e5-")  # model ids that are embedding models, not chat models
+GRAMMAR_ERRORS = ("grammar", "sampler", "response_format", "json_schema")  # a server that can't use a schema
+CONTEXT_ERRORS = ("context size", "context length", "context window", "maximum context", "exceeds the")
+
+
+class BackendError(OllamaError):
+    """The model server failed; the agent handles it like an Ollama error."""
+
+
+IMAGE_TYPES = {"iVBOR": "image/png", "/9j/": "image/jpeg", "R0lGOD": "image/gif", "UklGR": "image/webp"}
+
+
+def with_images(text: str, images: list[str]) -> list[dict]:
+    """A message's text and base64 images (Ollama's `images`) as OpenAI content parts."""
+    parts: list[dict] = [{"type": "text", "text": text}]
+    for data in images:
+        mime = next((t for prefix, t in IMAGE_TYPES.items() if data.startswith(prefix)), "image/png")
+        parts.append({"type": "image_url", "image_url": {"url": f"data:{mime};base64,{data}"}})
+    return parts
+
+
+def to_openai(messages: list[dict]) -> list[dict]:
+    """lcode's messages (Ollama's format) as OpenAI chat messages."""
+    out: list[dict] = []
+    pending: list[str] = []  # ids of the latest tool calls, for the tool results that follow
+    for n, m in enumerate(messages):
+        role = m.get("role")
+        if role in ("system", "user"):
+            text = m.get("content") or ""
+            out.append({"role": role, "content": with_images(text, m["images"]) if m.get("images") else text})
+        elif role == "assistant":
+            message: dict = {"role": "assistant", "content": m.get("content") or ""}
+            calls = m.get("tool_calls") or []
+            if calls:
+                ids = [c.get("id") or f"call_{n}_{i}" for i, c in enumerate(calls)]
+                message["tool_calls"] = []
+                for call_id, call in zip(ids, calls, strict=False):
+                    fn = call.get("function") or {}
+                    args = fn.get("arguments") or {}
+                    message["tool_calls"].append(
+                        {
+                            "id": call_id,
+                            "type": "function",
+                            "function": {
+                                "name": fn.get("name", ""),
+                                "arguments": args if isinstance(args, str) else json.dumps(args),
+                            },
+                        }
+                    )
+                pending = list(ids)
+            out.append(message)
+        elif role == "tool":
+            call_id = m.get("tool_call_id") or (pending.pop(0) if pending else f"call_{n}")
+            if m.get("tool_call_id") and call_id in pending:
+                pending.remove(call_id)
+            out.append({"role": "tool", "tool_call_id": call_id, "content": m.get("content") or ""})
+    return out
+
+
+class ThinkSplitter:
+    """Moves a leading <think>…</think> block out of the answer, for servers that don't separate reasoning.
+
+    vLLM without a reasoning parser, MLX's server and some LM Studio setups stream a reasoning model's
+    thinking as ordinary text. The tags can arrive split across chunks.
+    """
+
+    OPEN, CLOSE = "<think>", "</think>"
+
+    def __init__(self) -> None:
+        self.buffer = ""
+        self.state = "start"  # start -> thinking -> answer, or start -> answer
+        self.trim = False  # drop the blank lines between the thinking and the answer
+
+    def feed(self, text: str) -> list[tuple[str, str]]:
+        """("thinking" or "content", text) pieces that are ready to show."""
+        self.buffer += text
+        out: list[tuple[str, str]] = []
+        if self.state == "start":
+            head = self.buffer.lstrip()
+            if head.startswith(self.OPEN):
+                self.state, self.buffer = "thinking", head[len(self.OPEN) :]
+            elif not head or self.OPEN.startswith(head):
+                return out  # maybe the start of the tag: wait for more
+            else:
+                self.state = "answer"
+        if self.state == "thinking":
+            end = self.buffer.find(self.CLOSE)
+            if end >= 0:
+                out.append(("thinking", self.buffer[:end]))
+                self.state, self.trim, self.buffer = "answer", True, self.buffer[end + len(self.CLOSE) :]
+            else:
+                keep = next((n for n in range(len(self.CLOSE) - 1, 0, -1) if self.buffer.endswith(self.CLOSE[:n])), 0)
+                ready, self.buffer = self.buffer[: len(self.buffer) - keep], self.buffer[len(self.buffer) - keep :]
+                out.append(("thinking", ready))
+        if self.state == "answer":
+            if self.trim:
+                self.buffer = self.buffer.lstrip("\n")
+                self.trim = not self.buffer
+            out.append(("content", self.buffer))
+            self.buffer = ""
+        return [(kind, piece) for kind, piece in out if piece]
+
+    def flush(self) -> list[tuple[str, str]]:
+        rest, self.buffer = self.buffer, ""
+        return [("thinking" if self.state == "thinking" else "content", rest)] if rest else []
+
+
+def strip_thinking(text: str) -> str:
+    """An answer without its leading <think>…</think> block."""
+    splitter = ThinkSplitter()
+    pieces = splitter.feed(text) + splitter.flush()
+    return "".join(piece for kind, piece in pieces if kind == "content")
+
+
+class OpenAICompatible:
+    """A model server with an OpenAI-compatible API, behind the same interface as lcode's Ollama client."""
+
+    kind = "openai"
+
+    def __init__(self, base_url: str, api_key: str | None = None, name: str = "OpenAI-compatible server"):
+        self.name = name
+        self.host = base_url.rstrip("/")
+        self.root = self.host.removesuffix("/v1")
+        key = api_key if api_key is not None else os.environ.get("LCODE_API_KEY") or os.environ.get("OPENAI_API_KEY")
+        self.headers = {"Authorization": f"Bearer {key}"} if key else {}
+
+    # -- plumbing
+    def _url(self, path: str) -> str:
+        return f"{self.host}{path}"
+
+    def describe(self) -> str:
+        return f"{self.name} at {redact(self.host)}"
+
+    def _get(self, url: str, timeout: float = 10) -> dict:
+        try:
+            r = requests.get(url, headers=self.headers, timeout=timeout)
+        except requests.RequestException as e:
+            raise BackendError(f"cannot reach the model server at {redact(self.host)}: {e}") from e
+        if r.status_code != 200:
+            raise BackendError(f"{url.removeprefix(self.root)} failed ({r.status_code}): {r.text[:300]}")
+        return r.json()
+
+    def _payload(self, payload: dict, stream: bool) -> dict:
+        body: dict = {"model": payload["model"], "messages": to_openai(payload.get("messages") or []), "stream": stream}
+        if payload.get("tools") and not payload.get("format"):
+            # An answer in a given format has no tool calls, and llama-server can't combine the two grammars.
+            body["tools"] = payload["tools"]
+        if stream:
+            body["stream_options"] = {"include_usage": True}
+        if payload.get("think") is False:
+            body["chat_template_kwargs"] = {"enable_thinking": False}  # Qwen-style models on vLLM and llama.cpp
+        schema = payload.get("format")
+        if isinstance(schema, dict):
+            body["response_format"] = {"type": "json_schema", "json_schema": {"name": "answer", "schema": schema}}
+        elif schema == "json":
+            body["response_format"] = {"type": "json_object"}
+        if (payload.get("options") or {}).get("num_predict"):
+            body["max_tokens"] = payload["options"]["num_predict"]
+        return body
+
+    def _failed(self, status: int, text: str) -> BackendError:
+        message = f"{self.name} error {status}: {text[:500]}"
+        if any(marker in text.lower() for marker in CONTEXT_ERRORS):
+            message += (
+                "\nThe conversation no longer fits the server's context window. /compact summarizes it; lcode's "
+                "context setting (/ctx) should be no larger than the server's."
+            )
+        return BackendError(message)
+
+    # -- what the agent uses
+    def version(self) -> str:
+        self._get(self._url("/models"))
+        return self.name
+
+    def list_models(self) -> list[dict]:
+        return [{"name": m.get("id", ""), **m} for m in self._get(self._url("/models")).get("data") or []]
+
+    def installed_names(self) -> set[str]:
+        return {m["name"] for m in self.list_models()}
+
+    def chat_models(self) -> list[str]:
+        """The models served for chat (servers also list their embedding models)."""
+        return sorted(n for n in self.installed_names() if not is_embedding(n))
+
+    def show(self, model: str) -> dict:
+        return {"capabilities": ["completion", "tools"]}  # the API doesn't say; vision_model can name a model
+
+    def max_context(self, model: str) -> int | None:
+        """The server's context window, where it tells: vLLM, LM Studio or llama-server."""
+        try:
+            for entry in self.list_models():
+                if entry.get("name") == model and entry.get("max_model_len"):  # vLLM
+                    return int(entry["max_model_len"])
+        except BackendError:
+            return None
+        for url, read in (
+            # LM Studio: the size the model was loaded with (it loads models with its own default otherwise)
+            (f"{self.root}/api/v0/models/{quote(model, safe='')}", lambda d: d.get("loaded_context_length")),
+            (f"{self.root}/props", lambda d: (d.get("default_generation_settings") or {}).get("n_ctx")),  # llama-server
+        ):  # fmt: skip
+            try:
+                value = read(self._get(url, timeout=5))
+            except (BackendError, ValueError, AttributeError):
+                continue
+            if value:
+                return int(value)
+        return None
+
+    def running(self) -> list[dict]:
+        return []  # the server decides what it keeps loaded
+
+    def load(self, model: str, options: dict, keep_alive: str) -> None:
+        pass
+
+    def unload(self, model: str) -> None:
+        pass  # not possible through this API: free_gpu tools share the GPU with the model
+
+    def chat(self, payload: dict, timeout: float = 900) -> dict:
+        try:
+            r = requests.post(
+                self._url("/chat/completions"),
+                json=self._payload(payload, False),
+                headers=self.headers,
+                timeout=timeout,
+            )
+        except requests.RequestException as e:
+            raise BackendError(f"cannot reach the model server at {redact(self.host)}: {e}") from e
+        if r.status_code == 400 and payload.get("format") and any(w in r.text.lower() for w in GRAMMAR_ERRORS):
+            # The server can't hold the answer to a schema (llama-server can't with some settings):
+            # ask without one. The prompt asks for the same JSON, and lcode reads it out of the text.
+            return self.chat({k: v for k, v in payload.items() if k not in ("format", "tools")}, timeout)
+        if r.status_code != 200:
+            raise self._failed(r.status_code, r.text)
+        choice = (r.json().get("choices") or [{}])[0].get("message") or {}
+        message: dict = {"role": "assistant", "content": strip_thinking(choice.get("content") or "")}
+        if choice.get("tool_calls"):
+            message["tool_calls"] = [self._ollama_call(c) for c in choice["tool_calls"]]
+        return {"message": message}
+
+    @staticmethod
+    def _ollama_call(call: dict) -> dict:
+        fn = call.get("function") or {}
+        raw = fn.get("arguments") or "{}"
+        try:
+            args = json.loads(raw) if isinstance(raw, str) else raw
+        except json.JSONDecodeError as e:
+            # The same wording as Ollama's, so the agent asks the model to try again.
+            raise BackendError(f"error parsing tool call: raw='{raw[:200]}', err={e}") from e
+        return {"id": call.get("id", ""), "function": {"name": fn.get("name", ""), "arguments": args}}
+
+    def chat_stream(self, payload: dict, on_open: Callable | None = None) -> Iterator[dict]:
+        """Ollama-style chunks from an OpenAI-style stream: content, thinking, then tool calls and counts."""
+        started = time.monotonic_ns()
+        try:
+            r = requests.post(
+                self._url("/chat/completions"), json=self._payload(payload, True), headers=self.headers,
+                stream=True, timeout=(10, None),
+            )  # fmt: skip
+        except requests.RequestException as e:
+            raise BackendError(f"cannot reach the model server at {redact(self.host)}: {e}") from e
+        if on_open:
+            on_open(r)
+        calls: dict[int, dict] = {}
+        usage: dict = {}
+        timings: dict = {}  # llama-server measures prompt and generation time itself
+        splitter = ThinkSplitter()
+        first_token = None
+        with r:
+            if r.status_code != 200:
+                raise self._failed(r.status_code, r.text)
+            for line in r.iter_lines():
+                if not line or not line.startswith(b"data:"):
+                    continue
+                data = line[5:].strip()
+                if data == b"[DONE]":
+                    break
+                try:
+                    chunk = json.loads(data)
+                except ValueError:
+                    continue
+                if chunk.get("error"):
+                    error = chunk["error"]
+                    raise self._failed(0, error.get("message", str(error)) if isinstance(error, dict) else str(error))
+                usage = chunk.get("usage") or usage
+                timings = chunk.get("timings") or timings
+                for choice in chunk.get("choices") or []:
+                    delta = choice.get("delta") or {}
+                    thinking = delta.get("reasoning_content") or delta.get("reasoning")
+                    if thinking:
+                        first_token = first_token or time.monotonic_ns()
+                        yield {"message": {"thinking": thinking}, "done": False}
+                    if delta.get("content"):
+                        first_token = first_token or time.monotonic_ns()
+                        for kind, piece in splitter.feed(delta["content"]):
+                            yield {"message": {kind: piece}, "done": False}
+                    for piece in delta.get("tool_calls") or []:
+                        first_token = first_token or time.monotonic_ns()
+                        slot = calls.setdefault(piece.get("index", len(calls)), {"id": "", "name": "", "arguments": ""})
+                        fn = piece.get("function") or {}
+                        slot["id"] = piece.get("id") or slot["id"]
+                        slot["name"] += fn.get("name") or ""
+                        slot["arguments"] += fn.get("arguments") or ""
+        for kind, piece in splitter.flush():
+            yield {"message": {kind: piece}, "done": False}
+        if calls:
+            tool_calls = [
+                self._ollama_call({"id": c["id"], "function": {"name": c["name"], "arguments": c["arguments"] or "{}"}})
+                for _, c in sorted(calls.items())
+            ]
+            yield {"message": {"tool_calls": tool_calls}, "done": False}
+        ended = time.monotonic_ns()
+        first_token = first_token or ended
+        final = {
+            "message": {},
+            "done": True,
+            "prompt_eval_count": int(usage.get("prompt_tokens") or 0),
+            "prompt_eval_duration": first_token - started,
+            "eval_count": int(usage.get("completion_tokens") or 0),
+            "eval_duration": max(1, ended - first_token),
+        }
+        if timings.get("prompt_ms") is not None and timings.get("predicted_ms"):
+            # The whole prompt counts (it's what fills the context), cached tokens included.
+            prompt = final["prompt_eval_count"] or int(timings.get("prompt_n", 0)) + int(timings.get("cache_n", 0))
+            final.update(
+                prompt_eval_count=prompt,
+                prompt_eval_duration=int(timings["prompt_ms"] * 1e6),
+                eval_count=final["eval_count"] or int(timings.get("predicted_n", 0)),
+                eval_duration=max(1, int(timings["predicted_ms"] * 1e6)),
+            )
+        yield final
+
+    def embed(self, model: str, inputs: list[str], options: dict | None = None, keep_alive: str = "5m") -> list:
+        try:
+            r = requests.post(
+                self._url("/embeddings"), json={"model": model, "input": inputs}, headers=self.headers, timeout=600
+            )
+        except requests.RequestException as e:
+            raise BackendError(f"cannot reach the model server at {redact(self.host)}: {e}") from e
+        if r.status_code != 200:
+            raise BackendError(f"{self.name} embeddings failed ({r.status_code}): {r.text[:300]}")
+        return [item["embedding"] for item in sorted(r.json().get("data") or [], key=lambda d: d.get("index", 0))]
+
+
+def is_openai(client) -> bool:
+    return getattr(client, "kind", "ollama") == "openai"
+
+
+def is_embedding(model: str) -> bool:
+    return any(hint in model.lower() for hint in EMBEDDING_HINTS)
+
+
+def pick_model(client: OpenAICompatible, name: str) -> str:
+    """The served model `name` stands for: the same id, else the one id that contains it, else the only one."""
+    served = client.chat_models()
+    if name in served or not served:
+        return name  # nothing listed: some servers load models on request
+    matches = [m for m in served if name.lower() in m.lower()]
+    if len(matches) == 1:
+        return matches[0]
+    if len(served) == 1:
+        return served[0]  # llama-server serves one model, whatever it's called
+    listed = ", ".join(served) if served else "none"
+    raise BackendError(
+        f"{client.describe()} doesn't serve '{name}'. Its models: {listed}. "
+        "Choose one with --model or `lcode config set model <id>`."
+    )
+
+
+def context_for(client: OpenAICompatible, model: str, requested: int | None) -> tuple[int, str]:
+    """lcode's context budget: the server's window (lcode can't change it), or less when asked."""
+    try:
+        limit = client.max_context(model)
+    except BackendError:
+        limit = None
+    if limit and requested and requested > limit:
+        return limit, f"capped at the server's context window of {limit:,} tokens"
+    if requested or limit:
+        return requested or limit, ""  # type: ignore[return-value]
+    return DEFAULT_CONTEXT, (
+        f"{client.name} doesn't say how large its context window is; lcode assumes {DEFAULT_CONTEXT:,} tokens. "
+        "Set the real size with --context or `lcode config set context`"
+    )
+
+
+def connect(cfg: dict):
+    """The model server the settings point at: Ollama, or a server with an OpenAI-compatible API."""
+    from lcode.ollama import Ollama
+
+    backend = cfg.get("backend") or "ollama"
+    if backend == "ollama":
+        return Ollama(cfg["ollama_host"])
+    name, default_url = PRESETS[backend]
+    url = cfg.get("base_url") or default_url
+    if not url:
+        raise BackendError("backend = openai needs the server's address: lcode config set base_url http://host:port/v1")
+    return OpenAICompatible(url, name=name)

lcode/bench.py

@@ -1010,7 +1010,10 @@ def run_model(
         return run
     where = f" · {run.memory_gb} GB, {run.gpu_percent}% on the GPU" if run.memory_gb else ""
     reads = f" · reads prompts at {run.prompt_tps:,.0f} tok/s" if run.prompt_tps else ""
-    console.print(f"  [dim]Loaded in {run.load_seconds:.0f}s{where}{reads}[/]")
+    if getattr(ollama, "kind", "ollama") == "ollama":
+        console.print(f"  [dim]Loaded in {run.load_seconds:.0f}s{where}{reads}[/]")
+    elif reads:  # the server loads the model itself
+        console.print(f"  [dim]{reads.removeprefix(' · ').capitalize()}[/]")
     for task in tasks:
         log = console if verbose else Console(file=io.StringIO(), width=120, force_terminal=False)
         if session and shared is None:
@@ -1099,12 +1102,13 @@ def print_summary(console: Console, runs: list[ModelRun], tasks: list[Task]) ->
             console.print(f"[yellow]{escape(run.model)}: {escape(run.error)}[/]")
 
 
-def report(runs: list[ModelRun], hardware: Hardware, ollama_version: str) -> dict:
+def report(runs: list[ModelRun], hardware: Hardware, ollama_version: str, server: str = "") -> dict:
     """The JSON document written by --json (schema version 1)."""
     return {
         "schema": SCHEMA_VERSION,
         "lcode": __version__,
-        "ollama": ollama_version,
+        "ollama": None if server else ollama_version,
+        "server": server or f"Ollama {ollama_version}",
         "date": dt.datetime.now().astimezone().isoformat(timespec="seconds"),
         "hardware": {**asdict(hardware), "description": hardware.describe()},
         "tasks": [{"id": t.id, "title": t.title} for t in TASKS],
@@ -1112,11 +1116,12 @@ def report(runs: list[ModelRun], hardware: Hardware, ollama_version: str) -> dic
     }
 
 
-def markdown(runs: list[ModelRun], tasks: list[Task], hardware: Hardware, ollama_version: str) -> str:
+def markdown(runs: list[ModelRun], tasks: list[Task], hardware: Hardware, ollama_version: str, server: str = "") -> str:
     """A table to paste into a model test report."""
     date = dt.date.today().isoformat()
+    server = server or f"Ollama {ollama_version}"
     lines = [
-        f"**lcode bench** · lcode {__version__} · Ollama {ollama_version} · {hardware.describe()} · {date}",
+        f"**lcode bench** · lcode {__version__} · {server} · {hardware.describe()} · {date}",
         "",
         "| | " + " | ".join(f"`{r.model}`" for r in runs) + " |",
         "|---|" + "---|" * len(runs),

lcode/cli.py

@@ -12,11 +12,12 @@ from pathlib import Path
 
 import requests
 from rich.console import Console
+from rich.markup import escape
 from rich.progress import BarColumn, DownloadColumn, Progress, TextColumn, TimeRemainingColumn, TransferSpeedColumn
 from rich.prompt import Confirm
 from rich.table import Table
 
-from lcode import __version__, catalog, config, limits, web
+from lcode import __version__, backends, catalog, config, limits, web
 from lcode.agent import Agent, Settings
 from lcode.catalog import ModelSpec
 from lcode.config import ConfigError, format_tokens, parse_context
@@ -40,7 +41,25 @@ def fail(message: str, code: int = 1):
     sys.exit(code)
 
 
+def connect(cfg: dict) -> Ollama:
+    """The model server the settings point at (an OpenAI-compatible one has the same interface)."""
+    if cfg["backend"] == "ollama":
+        return Ollama(cfg["ollama_host"])
+    try:
+        return backends.connect(cfg)
+    except OllamaError as e:
+        fail(str(e))
+
+
 def check_ollama(ollama: Ollama, hw: Hardware | None = None) -> str:
+    if backends.is_openai(ollama):
+        try:
+            return ollama.version()
+        except OllamaError as e:
+            fail(
+                f"{e}\n\nIs {ollama.name} running, with its server started? lcode expects its OpenAI-compatible "
+                "API there; change the address with: lcode config set base_url http://host:port/v1"
+            )
     try:
         version = ollama.version()
     except OllamaError as e:
@@ -52,7 +71,12 @@ def check_ollama(ollama: Ollama, hw: Hardware | None = None) -> str:
 
 
 def resolve_model(ollama: Ollama, name: str) -> tuple[str, ModelSpec | None]:
-    """Map a catalog key or Ollama tag to an installed Ollama model name."""
+    """Map a catalog key or Ollama tag to an installed Ollama model name (or a model the server serves)."""
+    if backends.is_openai(ollama):
+        try:
+            return backends.pick_model(ollama, name), None
+        except OllamaError as e:
+            raise NotInstalled(str(e)) from e
     spec = catalog.find(name)
     installed = ollama.installed_names()
     if spec:
@@ -72,6 +96,8 @@ def choose_context(
     ollama: Ollama, model: str, spec: ModelSpec | None, requested: int | None, hw: Hardware
 ) -> tuple[int, str]:
     """Pick the context window: requested > largest that fits (catalog models) > 32K. Capped at the model max."""
+    if backends.is_openai(ollama):
+        return backends.context_for(ollama, model, requested)
     try:
         limit = ollama.max_context(model) or (spec.max_context if spec else None)
     except OllamaError:
@@ -96,7 +122,29 @@ def choose_context(
 # ----------------------------------------------------------------------------- lcode models
 
 
+def print_served(client: backends.OpenAICompatible, current: str | None) -> None:
+    try:
+        served = client.chat_models()
+        current = backends.pick_model(client, current) if current else None
+    except OllamaError as e:
+        if not isinstance(e, backends.BackendError) or "doesn't serve" not in str(e):
+            console.print(f"[red]{e}[/]")
+            return
+    console.print(f"[bold]Models served by {escape(client.describe())}[/]")
+    for name in served:
+        console.print(f"  {escape(name)}" + ("  [bold]current[/]" if name == current else ""))
+    if not served:
+        console.print("  none listed")
+    console.print(
+        "[dim]Models are downloaded and loaded in the server itself. Choose one with [/]lcode --model <id>[dim] "
+        "or [/]lcode config set model <id>[dim].[/]"
+    )
+
+
 def print_models(ollama: Ollama | None, hw: Hardware, current: str | None = None) -> None:
+    if ollama is not None and backends.is_openai(ollama):
+        print_served(ollama, current)  # type: ignore[arg-type]
+        return
     try:
         installed = ollama.installed_names() if ollama else set()
     except OllamaError:
@@ -141,8 +189,7 @@ def print_models(ollama: Ollama | None, hw: Hardware, current: str | None = None
 
 def cmd_models(args) -> None:
     cfg = config.load()
-    ollama = Ollama(cfg["ollama_host"])
-    print_models(ollama, detect(), cfg["model"])
+    print_models(connect(cfg), detect(), cfg["model"])
 
 
 # ----------------------------------------------------------------------------- lcode setup
@@ -173,6 +220,13 @@ def pull_with_progress(ollama: Ollama, tag: str) -> None:
 
 def cmd_setup(args) -> None:
     cfg = config.load()
+    if cfg["backend"] != "ollama":
+        name = backends.PRESETS[cfg["backend"]][0]
+        fail(
+            f"lcode setup downloads models into Ollama, but lcode uses {name} (backend = {cfg['backend']}). "
+            f"Download and load a model in {name}, then: lcode config set model <id> (see lcode models). "
+            "Back to Ollama: lcode config set backend ollama"
+        )
     ollama = Ollama(cfg["ollama_host"])
     hw = detect()
     console.print(f"[bold]Machine:[/] {hw.describe()}")
@@ -257,15 +311,24 @@ def cmd_doctor(args) -> None:
         f"{config.CONFIG_PATH}" + ("" if config.CONFIG_PATH.exists() else " (not created yet)"),
         True if config.CONFIG_PATH.exists() else None,
     )
-    ollama = Ollama(cfg["ollama_host"])
+    ollama = connect(cfg)
+    openai = backends.is_openai(ollama)
     try:
-        version = ollama.version()
-        fresh = version_tuple(version) >= MIN_VERSION
-        line("Ollama", f"{redact(ollama.host)} · version {version}" + ("" if fresh else " (too old)"), fresh)
-        ok &= fresh
+        if openai:
+            ollama.version()
+            served = ollama.chat_models()  # type: ignore[attr-defined]
+            line("Server", f"{ollama.describe()} · {len(served)} model(s) served", True)
+        else:
+            version = ollama.version()
+            fresh = version_tuple(version) >= MIN_VERSION
+            line("Ollama", f"{redact(ollama.host)} · version {version}" + ("" if fresh else " (too old)"), fresh)
+            ok &= fresh
     except OllamaError as e:
-        line("Ollama", f"{e}", False)
-        line("", f"install: {OLLAMA_INSTALL.get(hw.os, OLLAMA_INSTALL['linux'])}", None)
+        line("Server" if openai else "Ollama", f"{e}", False)
+        if openai:
+            line("", f"start {ollama.name}'s server, or set its address: lcode config set base_url <url>", None)
+        else:
+            line("", f"install: {OLLAMA_INSTALL.get(hw.os, OLLAMA_INSTALL['linux'])}", None)
         sys.exit(1)
     try:
         model, spec = resolve_model(ollama, cfg["model"])
@@ -288,6 +351,8 @@ def cmd_doctor(args) -> None:
                 f"{seer} looks at screenshots and images" + ("" if seer == model else " (loaded when needed)"),
                 True,
             )
+        elif openai:
+            line("Vision", "off; set vision_model to a model the server serves that can see images", None)
         else:
             line("Vision", "off" if cfg["vision_model"] == "off" else vision.INSTALL_HINT, None)
         if limits.get(model):
@@ -475,10 +540,11 @@ def cmd_bench(args) -> None:
         fail(str(e))
     if args.timeout <= 0:
         fail("--timeout must be a positive number of seconds")
-    ollama = Ollama(cfg["ollama_host"])
+    ollama = connect(cfg)
     hw = detect()
     version = check_ollama(ollama, hw)
-    console.print(f"[bold]lcode bench[/] · {hw.describe()} · Ollama {version}")
+    server = ollama.describe() if backends.is_openai(ollama) else ""
+    console.print(f"[bold]lcode bench[/] · {hw.describe()} · {server or f'Ollama {version}'}")
     console.print("[dim]Each task runs in a new temporary folder with every permission granted and web access off.[/]")
     names = args.models or [cfg["model"]]
     resolved: dict[str, tuple[str, ModelSpec | None] | NotInstalled] = {}
@@ -535,10 +601,10 @@ def cmd_bench(args) -> None:
     bench.print_summary(console, runs, tasks)
     if args.json:
         path = Path(args.json).expanduser()
-        bench.write_json(path, bench.report(runs, hw, version))
+        bench.write_json(path, bench.report(runs, hw, version, server))
         console.print(f"Results saved to {path}")
     if args.markdown:
-        print("\n" + bench.markdown(runs, tasks, hw, version))
+        print("\n" + bench.markdown(runs, tasks, hw, version, server))
     if args.keep:
         console.print(f"[dim]Task folders are kept in {tempfile.gettempdir()} (lcode-bench-*).[/]")
 
@@ -560,13 +626,14 @@ def cmd_chat(args) -> None:
     cwd = Path(args.repo).expanduser().resolve()
     if not cwd.is_dir():
         fail(f"not a directory: {cwd}")
-    ollama = Ollama(cfg["ollama_host"])
+    ollama = connect(cfg)
     hw = detect()
     check_ollama(ollama, hw)
     try:
         model, spec = resolve_model(ollama, args.model or cfg["model"])
     except NotInstalled as e:
-        if not args.model and cfg["model"] == config.DEFAULT_MODEL and not config.CONFIG_PATH.exists():
+        first_run = not args.model and cfg["model"] == config.DEFAULT_MODEL and not config.CONFIG_PATH.exists()
+        if first_run and not backends.is_openai(ollama):
             fail("lcode isn't set up yet. Run:  lcode setup")
         fail(str(e))
     context, note = choose_context(ollama, model, spec, requested_ctx, hw)
@@ -744,11 +811,13 @@ def cmd_index(args) -> None:
 
     cfg = config.load()
     root = work_tree_for(Path(args.repo).expanduser().resolve())
-    ollama = Ollama(cfg["ollama_host"])
+    ollama = connect(cfg)
     setting = args.model or cfg["embed_model"]
     if setting == "off":
         fail("semantic code search is off (lcode config set embed_model auto)")
     model = codesearch.pick_model(ollama, "auto" if setting == "off" else setting)
+    if model is None and backends.is_openai(ollama):
+        fail(f"{ollama.describe()} serves no embedding model: load one there (e.g. qwen3-embedding-0.6b)")
     if model is None:
         fail(
             "no embedding model is installed. Install one, for example:\n  ollama pull qwen3-embedding:0.6b\n"

lcode/codesearch.py

@@ -53,7 +53,12 @@ def pick_model(ollama: Ollama, setting: str) -> str | None:
         return None
     if setting != "auto":
         return setting if setting in installed or f"{setting}:latest" in installed else None
-    return next((m for m in MODELS if m in installed), None)
+    found = next((m for m in MODELS if m in installed), None)
+    if found is None and getattr(ollama, "kind", "ollama") == "openai":  # e.g. LM Studio's text-embedding-…
+        from lcode.backends import is_embedding
+
+        found = next((m for m in sorted(installed) if is_embedding(m)), None)
+    return found
 
 
 def index_dir(root: Path) -> Path:

lcode/config.py

@@ -7,6 +7,7 @@ import os
 import re
 import sys
 from pathlib import Path
+from urllib.parse import urlsplit
 
 if sys.version_info >= (3, 11):
     import tomllib
@@ -36,6 +37,8 @@ SETTINGS: dict[str, tuple[object, type, str]] = {
     "num_batch": (None, int, "prompt batch size; larger reads prompts faster but needs more VRAM"),
     "keep_alive": ("30m", str, "how long Ollama keeps the model loaded after the last request"),
     "ollama_host": ("http://localhost:11434", str, "Ollama server URL"),
+    "backend": ("ollama", str, "model server: ollama | lmstudio | llama.cpp | vllm | mlx | openai (any OpenAI API)"),
+    "base_url": (None, str, "OpenAI-compatible server address, e.g. http://localhost:1234/v1 (default: the backend's)"),
     "permission_mode": ("ask", str, "ask | plan | auto-edit | yolo"),
     "think": (True, bool, "let the model reason before answering (slower, better)"),
     "web": ("on", str, "web search and page fetching for the model: on | ask | off"),
@@ -62,6 +65,8 @@ ENV_OVERRIDES = {
     "LCODE_NUM_BATCH": "num_batch",
     "LCODE_KEEP_ALIVE": "keep_alive",
     "OLLAMA_HOST": "ollama_host",
+    "LCODE_BACKEND": "backend",
+    "LCODE_BASE_URL": "base_url",
     "LCODE_WEB": "web",
     "LCODE_SANDBOX": "sandbox",
     "LCODE_MEMORY": "memory",
@@ -111,6 +116,18 @@ def coerce(key: str, value: object) -> object:
         return parse_context(value)  # type: ignore[arg-type]
     if key in ("ollama_host", "searxng_url"):
         return normalize_host(str(value))
+    if key == "base_url":
+        url = normalize_host(str(value))
+        return url if urlsplit(url).path.strip("/") else url + "/v1"  # the OpenAI API lives under /v1
+    if key == "backend":
+        from lcode.backends import BACKENDS
+
+        value = {"lm-studio": "lmstudio", "llamacpp": "llama.cpp", "llama-server": "llama.cpp"}.get(
+            str(value).lower(), str(value).lower()
+        )
+        if value not in BACKENDS:
+            raise ConfigError(f"backend must be one of {', '.join(BACKENDS)}")
+        return value
     if key == "web":
         value = {"true": "on", "false": "off", "yes": "on", "no": "off"}.get(str(value).lower(), str(value).lower())
         if value not in WEB_MODES:

lcode/ollama.py

@@ -32,9 +32,15 @@ def version_tuple(version: str) -> tuple[int, ...]:
 
 
 class Ollama:
+    kind = "ollama"
+    name = "Ollama"
+
     def __init__(self, host: str):
         self.host = host.rstrip("/")
 
+    def describe(self) -> str:
+        return f"Ollama at {redact(self.host)}"
+
     def _url(self, path: str) -> str:
         return f"{self.host}{path}"
 

lcode/repl.py

@@ -26,7 +26,7 @@ from rich.rule import Rule
 from rich.table import Table
 from rich.text import Text
 
-from lcode import __version__, catalog, extensions, limits, sessions, web
+from lcode import __version__, backends, catalog, extensions, limits, sessions, web
 from lcode import context as context_tools
 from lcode import memory as memory_notes
 from lcode.agent import AUTO_COMPACT_RATIO, INIT_PROMPT, Agent
@@ -149,7 +149,9 @@ def banner(agent: Agent) -> None:
             Group(
                 Text.from_markup(f"[bold cyan]lcode[/] [dim]v{__version__}[/] — local coding agent\n"),
                 Text.from_markup(
-                    f"[dim]model[/]    {s.model}\n"
+                    f"[dim]model[/]    {escape(s.model)}"
+                    + (f" [dim]· {escape(agent.ollama.describe())}[/]" if backends.is_openai(agent.ollama) else "")
+                    + "\n"
                     f"[dim]context[/]  {format_tokens(s.context)} tokens\n"
                     f"[dim]cwd[/]      {agent.cwd}\n"
                     f"[dim]mode[/]     {agent.perms.mode} [dim](Shift+Tab to cycle)[/]\n"
@@ -221,6 +223,12 @@ def context_command(agent: Agent, arg: str, hardware: Hardware) -> None:
         c.print(f"MCP tools: ~{format_tokens(agent.mcp.cost())} tokens of definitions, {how}.")
     if not arg:
         print_breakdown(agent)
+    if not arg and backends.is_openai(agent.ollama):
+        c.print(
+            f"[dim]{agent.ollama.name} sets the context window when it loads the model; lcode uses "
+            f"{format_tokens(s.context)} of it. To summarize sooner: /ctx 16k[/]"
+        )
+        return
     if arg:
         try:
             apply_context(agent, parse_context(arg), hardware)
@@ -838,7 +846,7 @@ def handle_command(agent: Agent, line: str, hardware: Hardware) -> bool:
         print_models(agent.ollama, hardware, s.model)
     elif cmd == "/model":
         if not arg:
-            c.print(f"Model: {s.model}. Switch with /model <key or Ollama tag>; list with /models.")
+            c.print(f"Model: {s.model}. Switch with /model <name>; list with /models.")
         else:
             try:
                 name, spec = resolve_model(agent.ollama, arg)
@@ -849,6 +857,8 @@ def handle_command(agent: Agent, line: str, hardware: Hardware) -> bool:
             s.num_batch = spec.num_batch if spec and name == spec.local_name else None
             if spec:  # size the context for the new model: largest window that fits this machine
                 s.context = limits.cap(name, spec.fit(hardware)[0] or MIN_USEFUL_CONTEXT)
+            elif backends.is_openai(agent.ollama):
+                s.context = backends.context_for(agent.ollama, name, None)[0]  # the server's window
             agent.messages[0]["content"] = agent.system_prompt()
             c.print(f"[green]Switched to {name}[/] (context {format_tokens(s.context)}).")
     elif cmd == "/think":

{lcode_cli-0.15.0.dist-info → lcode_cli-0.16.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: lcode-cli
-Version: 0.15.0
+Version: 0.16.0
 Summary: A local-first terminal coding agent powered by open-weight models on your own GPU or Mac, via Ollama.
 Project-URL: Homepage, https://nasser1941.github.io/lcode/
 Project-URL: Documentation, https://nasser1941.github.io/lcode/
@@ -102,7 +102,7 @@ through [Ollama](https://ollama.com), and when it needs current information it c
   session as short notes, saved by the model, by you (`/remember`) or by a quick check at the end
   of a session. Not whole conversations, and never secrets.
 - **Bring your own model.** Eight curated open-weight models, all tested end to end, or any Ollama model
-  with tool calling.
+  with tool calling. Or use LM Studio, llama.cpp, vLLM or MLX instead of Ollama.
 - **MCP servers, ready to go.** Connect Jira and Confluence, GitHub, AWS, Google Drive, Grafana,
   Google Cloud, Sentry, Linear, Notion, Postgres, Kubernetes, Metabase, Encord, Valohai and more with one command
   (`lcode mcp add atlassian`), image generation with ComfyUI, or any other MCP server. Browser sign-in (OAuth) is built in.
@@ -208,8 +208,8 @@ fills up. Details: [How it works](https://nasser1941.github.io/lcode/how-it-work
 
 See the pinned [Roadmap issue](https://github.com/nasser1941/lcode/issues/10). Next up:
 
-- [More local backends: LM Studio, llama.cpp, vLLM and MLX](https://github.com/nasser1941/lcode/issues/47)
 - [Git workflow: /commit, /review, /pr and parallel worktrees](https://github.com/nasser1941/lcode/issues/50)
+- [Automation: JSON output, a Python API and a GitHub Action for self-hosted runners](https://github.com/nasser1941/lcode/issues/48)
 - More models in the catalog, and test reports from your hardware ([report one](https://github.com/nasser1941/lcode/issues/new?template=model_request.yml))
 
 ## Contributing

{lcode_cli-0.15.0.dist-info → lcode_cli-0.16.0.dist-info}/RECORD

@@ -1,13 +1,14 @@
-lcode/__init__.py,sha256=nt1m5TjCnM-J0wx2xHbNb3L4c1vNJmM3nIma1bcOi5E,118
+lcode/__init__.py,sha256=zacfVVvqhWmN_-V62Fl2VjpnxCnSmt1Y75XJ4_2KJ64,118
 lcode/__main__.py,sha256=s5N5pryX2FAu93GGZ3FxUrIXHWGzxz66lKUoaV_h8Ws,35
-lcode/agent.py,sha256=7eWmKnxRAYEvGYGkukmd2EGfWg_LYhEQdBJQljMK_Uk,46113
-lcode/bench.py,sha256=7XHJZmRVUt3JYYEyIIMX9fcqOFIvfKB9Iy-2aPo00VU,40406
+lcode/agent.py,sha256=ZApMmW8TE4SMD5nc2TCE7M0U_VCwHq1xIl97hmFGcfo,46262
+lcode/backends.py,sha256=dtJHrC6ys73bRDg64z0tQz1PavmeAkiODZu1fOO9ExQ,20635
+lcode/bench.py,sha256=XVdk0hDxmRcjgROOCgwwOsExcUhIxgh0B2vnxtGnA3o,40742
 lcode/bench_backend.py,sha256=X2fo-zRX92Xp3KMxSsFlWioc2vRupBTtowj5kLME8mw,14287
 lcode/catalog.py,sha256=pZ1azX9OH2r4JE2c6E5OfwywpgS39-W2oXzoCHJhmtM,4141
 lcode/checkpoints.py,sha256=eqPtmPe44-aweWqc2H21ROogwzIt3NpvcUobsaCHoGY,18354
-lcode/cli.py,sha256=bcASJ0qpSbAi-rmqqLqpySHtckf6rQ777CjB82ll8C0,37091
-lcode/codesearch.py,sha256=_Z7D_Snpl0NwfXk6P4yUJ0VmwLKCy74YLc52DjUUdTU,10510
-lcode/config.py,sha256=5JdLA04cxB7gEf6JGJYbIlqIgjo9tNNiT26osEdJ9zI,9190
+lcode/cli.py,sha256=QAnmYqJ36Wyhp5WLI0DC6S-KIVgzx8vZ8Iv-rQLmqEk,40289
+lcode/codesearch.py,sha256=Io8leN1nb_Y-Dvlm9xaGtwUWYXX-ooswuznJV6dGU1s,10766
+lcode/config.py,sha256=nFljzjHTFaCTTbagycagbh2X_PpLtsUs-yRODiy4QcU,10065
 lcode/context.py,sha256=qJymLE3eYeaF6qVOzlUwDURqlky__8cskl_6vXeU3AA,10701
 lcode/extensions.py,sha256=W0HFDl0BvROzr7i_qtabHdIbacBgP5o5qIJLzoM2oNE,16806
 lcode/frontmatter.py,sha256=Bx0n4vLPhp7ZCWx-g7xqzqNy5VoKBxyeeikh5UFB_gA,2441
@@ -17,11 +18,11 @@ lcode/limits.py,sha256=oJzVQoMyTZ0_Fkpu5t3qxrf9v7Hv9S8U8BRQR5M1aKA,1237
 lcode/lsp.py,sha256=NtSW7xapv7fg2HAQLuYK1nwIqT7nG0LK1hiTpOmla_4,26086
 lcode/memory.py,sha256=WZEHTp-lVxAwS-yyLQR2xB3TMZ9sejmX2Ivg7T-tl3w,21949
 lcode/models.toml,sha256=lt2G64zzo_Ryxv2ISRSNZoVD_JqCqYqsM01n8WoXGs8,4098
-lcode/ollama.py,sha256=edSckfbtK0iSH-Pm-IpUHO_73nPaRokh7cHiDBSKvWU,7594
+lcode/ollama.py,sha256=u5tnvL28cfaFS4GhmiEsHhTQe6w3Ng_Tej05UUzLQTQ,7715
 lcode/permissions.py,sha256=bSDpkSzq1tozC3LgzAzXmJX_DGRxoY9a3Kdx-AFQ3nI,5426
 lcode/planning.py,sha256=hZjVka7i9TswJrXLbjbZJ0PBc6zHpmO64JqMiaFYnMQ,6110
 lcode/render.py,sha256=gMvf4W3qkLUJ4Vot3az_hP_zRDAItJC9pP-68dHFkfA,1375
-lcode/repl.py,sha256=l3LHCNRZD7Kk03k5hk6GJKrm62ToQT9FPjKtW3xwsAc,45540
+lcode/repl.py,sha256=PRO4VG-s8kSowcHJ3svYHpHFBU0s4nsvsth8dyKG6mE,46117
 lcode/repomap.py,sha256=1GRJgEUjBl9XHSXsF7tSGXBCEgsY-3CzwMohaTudZ_M,13969
 lcode/sandbox.py,sha256=u_pkrVRXhCGhh-wFDPwuZ3jGG7UeXxXRAeBNsm37ZvE,8608
 lcode/sessions.py,sha256=wOR-J0EHgJxLSPboKPHnvBcp5VTUYTj3tZWgpsa6-K4,4013
@@ -44,8 +45,8 @@ lcode/mcp/stdio.py,sha256=kC6KHwzMMDjBr-yCZLmyVKS7PZXU299WpZdYRoX5eRQ,6292
 lcode/servers/__init__.py,sha256=qrIhxEodEB8cjKEQ4Cicc0GsQhE2OH-9qlVHzr3-tq8,77
 lcode/servers/encord.py,sha256=WSauawL1UKYZCUAG1oalcKL_An3DVof-JM6xIgFyQSE,12769
 lcode/servers/valohai.py,sha256=e5xZGIoa4cja4f5i85q1GpL9c7cglUjV8jN2co1CFMY,14430
-lcode_cli-0.15.0.dist-info/METADATA,sha256=BSaWKw3B8zJZ55eweihAEyScAXH3KT3fHgfBTdTQJQk,12040
-lcode_cli-0.15.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
-lcode_cli-0.15.0.dist-info/entry_points.txt,sha256=pMz2Byj2QRyt-ZZQYZvmWUtmKPCeE7f8iCvu-sjZPIo,133
-lcode_cli-0.15.0.dist-info/licenses/LICENSE,sha256=JWjmoL7nNIC1XUa-dZm2hwLUgjUQMFo6k8APb0idNzs,1074
-lcode_cli-0.15.0.dist-info/RECORD,,
+lcode_cli-0.16.0.dist-info/METADATA,sha256=vHaeTD3ZW6w2pDT_LhDDlfyVYrIa3lprUbWUof4WJtE,12126
+lcode_cli-0.16.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+lcode_cli-0.16.0.dist-info/entry_points.txt,sha256=pMz2Byj2QRyt-ZZQYZvmWUtmKPCeE7f8iCvu-sjZPIo,133
+lcode_cli-0.16.0.dist-info/licenses/LICENSE,sha256=JWjmoL7nNIC1XUa-dZm2hwLUgjUQMFo6k8APb0idNzs,1074
+lcode_cli-0.16.0.dist-info/RECORD,,