inferml 1.0.1__py3-none-any.whl

_win_compat.py

@@ -0,0 +1,52 @@
+"""Windows compatibility patches applied process-wide at sidecar boot.
+
+Currently:
+- os.symlink → transparent copy fallback when the caller lacks
+  SeCreateSymbolicLinkPrivilege (the WinError 1314 case). HuggingFace's
+  cache layout uses symlinks to dedup blobs across snapshots, and a
+  standard non-admin user without Developer Mode hits this on every
+  download. POSIX users keep the real os.symlink (symlinks always work
+  there).
+
+Import this module before any other library that may call os.symlink.
+On non-Windows platforms the import is a no-op.
+"""
+from __future__ import annotations
+
+import os
+import shutil
+import sys
+
+
+def _install_symlink_copy_fallback() -> None:
+    if sys.platform != "win32":
+        return
+
+    _orig = os.symlink
+
+    def _resolve(src, dst):
+        if os.path.isabs(src):
+            return src
+        return os.path.normpath(os.path.join(os.path.dirname(dst), src))
+
+    def _symlink(src, dst, target_is_directory=False, *, dir_fd=None):
+        try:
+            return _orig(src, dst, target_is_directory=target_is_directory, dir_fd=dir_fd)
+        except OSError as e:
+            is_privilege_error = (
+                getattr(e, "winerror", None) == 1314
+                or "privilege" in str(e).lower()
+            )
+            if not is_privilege_error:
+                raise
+            real_src = _resolve(src, dst)
+            if target_is_directory or os.path.isdir(real_src):
+                shutil.copytree(real_src, dst, dirs_exist_ok=True)
+            else:
+                shutil.copyfile(real_src, dst)
+            return None
+
+    os.symlink = _symlink
+
+
+_install_symlink_copy_fallback()

adapters/__init__.py

@@ -0,0 +1,65 @@
+"""Adapter base classes + shared catch-all adapters.
+
+Per-family inference code lives in `python/models/<family>/`. This package
+holds only the cross-cutting pieces:
+
+  - `Adapter`                  the base class every family inherits from
+  - `StandardPipelineAdapter`  fallback for repos with no dedicated family
+  - `DiffusersAdapter`         library passthrough for diffusers checkpoints
+
+Routing strategy lives in `routing.py` and is:
+
+    1. Named override (via model_overrides.json `"adapter"` field)
+    2. Plugin adapters (python/plugins/*.py)
+    3. models/<family>/ registry (per-family folders)
+    4. DiffusersAdapter (library == "diffusers")
+    5. StandardPipelineAdapter (pipeline_tag in its task list)
+"""
+from __future__ import annotations
+
+from .base import Adapter  # noqa: F401
+from .standard_pipeline import StandardPipelineAdapter
+from .diffusers_pipeline import DiffusersAdapter
+
+def _named_adapters() -> dict[str, type]:
+    """Build the name→class map used by `model_overrides.json "adapter"` pins.
+
+    Includes the cross-cutting fallbacks plus every family folder in
+    `python/models/`. Built LAZILY (see `__getattr__` below) so we don't
+    capture a partially-loaded `models.FAMILIES` if some caller imports
+    `models` before `adapters` and the family-folder-discovery chain
+    re-enters this module mid-load.
+    """
+    out: dict[str, type] = {
+        "standard":  StandardPipelineAdapter,
+        "diffusers": DiffusersAdapter,
+    }
+    try:
+        from models import FAMILIES
+        for fam_name, entry in FAMILIES.items():
+            cls = entry.get("adapter")
+            if cls is None:
+                continue
+            out[fam_name] = cls
+            short = cls.__name__.replace("Adapter", "").lower()
+            out.setdefault(short, cls)
+    except Exception:
+        pass
+    return out
+
+_NAMED_ADAPTERS_CACHE: "dict[str, type] | None" = None
+
+def __getattr__(name: str):
+    global _NAMED_ADAPTERS_CACHE
+    if name == "NAMED_ADAPTERS":
+        if _NAMED_ADAPTERS_CACHE is None:
+            _NAMED_ADAPTERS_CACHE = _named_adapters()
+        return _NAMED_ADAPTERS_CACHE
+    raise AttributeError(f"module 'adapters' has no attribute {name!r}")
+
+__all__ = [
+    "Adapter",
+    "StandardPipelineAdapter",
+    "DiffusersAdapter",
+    "NAMED_ADAPTERS",
+]

adapters/base.py

@@ -0,0 +1,37 @@
+"""Adapter base class.
+
+One instance of an adapter = one loaded model. The router picks the adapter,
+calls `load(info, device)` once, then `run(inputs, params)` per request.
+Instances are cached by (adapter_class, model_id) in the engine.
+"""
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+
+class Adapter(ABC):
+    override: dict = {}
+
+    @classmethod
+    def can_handle(cls, info: dict) -> bool:
+        """Return True if this adapter can run the described model.
+
+        `info` is the dict from routing.inspect_model. Implementations should
+        inspect `model_id`, `model_type`, `architectures`, `tags`, etc. -
+        *not* download any weights."""
+        return False
+
+    @abstractmethod
+    def load(self, info: dict, device) -> None:
+        """Instantiate the underlying model + any helpers (processor, tokenizer)."""
+
+    @abstractmethod
+    def run(self, inputs: dict, params: dict) -> dict:
+        """Execute inference. Must return a dict matching one of the kinds in
+        `output_kinds.py` (`boxes`, `masks`, `labels`, `text`, `image`,
+        `audio`, `vector`)."""
+
+    def unload(self) -> None:
+        """Hook for freeing GPU memory - default: drop references."""
+        for attr in list(self.__dict__.keys()):
+            if attr not in ("override",):
+                setattr(self, attr, None)

adapters/diffusers_pipeline.py

@@ -0,0 +1,57 @@
+"""Diffusers fallback. text-to-image, img2img, inpainting."""
+from __future__ import annotations
+
+from .base import Adapter
+import output_kinds as ok
+from io_utils import decode_image, resolve_device, torch_dtype_for_device
+
+class DiffusersAdapter(Adapter):
+    @classmethod
+    def can_handle(cls, info):
+        if info.get("library") == "diffusers":
+            return info.get("pipeline_tag") != "text-to-video"
+        tag = info.get("pipeline_tag")
+        return tag in ("text-to-image", "image-to-image", "inpainting")
+
+    def load(self, info, device):
+        self.info = info
+        self.device = device
+        self.task = info.get("pipeline_tag") or "text-to-image"
+        dtype = torch_dtype_for_device()
+
+        from diffusers import (
+            AutoPipelineForText2Image,
+            AutoPipelineForImage2Image,
+            AutoPipelineForInpainting,
+        )
+        cls = {
+            "image-to-image": AutoPipelineForImage2Image,
+            "inpainting":     AutoPipelineForInpainting,
+        }.get(self.task, AutoPipelineForText2Image)
+
+        kwargs = {"torch_dtype": dtype}
+        if self.override.get("trust_remote_code"):
+            kwargs["trust_remote_code"] = True
+        self.pipe = cls.from_pretrained(info["model_id"], **kwargs)
+        resolved = resolve_device()
+        if resolved is not False:
+            self.pipe = self.pipe.to(resolved)
+
+    def run(self, inputs, params):
+        prompt = (inputs.get("text") or "").strip()
+        if not prompt:
+            raise ValueError("Prompt required")
+        kwargs = {k: params[k] for k in
+                  ("num_inference_steps", "guidance_scale", "negative_prompt", "strength")
+                  if k in params}
+        kwargs.setdefault("num_inference_steps", 20)
+        kwargs.setdefault("guidance_scale", 7.5)
+
+        if self.task == "image-to-image" and inputs.get("dataUrl"):
+            kwargs["image"] = decode_image(inputs["dataUrl"])
+        elif self.task == "inpainting" and inputs.get("dataUrl"):
+            kwargs["image"] = decode_image(inputs["dataUrl"])
+
+        result = self.pipe(prompt, **kwargs)
+        image = result.images[0]
+        return ok.image(image)

adapters/standard_pipeline.py

@@ -0,0 +1,31 @@
+"""Standard HF pipeline adapter. Fallback when no `models/<family>/` matches.
+
+Thin dispatcher: the actual per-task logic lives in `python/tasks/`. When a
+model breaks, add a folder under `python/models/` (preferred) or a Variant
+in the relevant task file.
+"""
+from __future__ import annotations
+
+from .base import Adapter
+from tasks import TASK_REGISTRY, get_task
+
+
+class StandardPipelineAdapter(Adapter):
+    SUPPORTED_TASKS = set(TASK_REGISTRY.keys())
+
+    @classmethod
+    def can_handle(cls, info):
+        return info.get("pipeline_tag") in cls.SUPPORTED_TASKS
+
+    def load(self, info, device):
+        self.info = info
+        self.device = device
+        self.task_name = info["pipeline_tag"]
+        self.handler = get_task(self.task_name)
+        if self.handler is None:
+            raise ValueError(f"No task handler registered for {self.task_name!r}")
+        extra = {"trust_remote_code": True} if self.override.get("trust_remote_code") else {}
+        self.state = self.handler.load_pipeline(info, device, extra_kwargs=extra)
+
+    def run(self, inputs, params):
+        return self.handler.handle(self.state, inputs, params)

engine.py

@@ -0,0 +1,294 @@
+"""LocalML inference engine - the reusable core.
+
+Holds all model-loading and inference logic, driven in-process by the FastAPI
+web server (`server/`). Running in-process is what lets the OpenAI-compatible
+endpoint hold a live handle to the currently-loaded LLM and stream tokens from
+it.
+
+Design invariants:
+  - One model = one loaded pipeline. Adapter instances are cached by
+    (adapter_class_name, model_id); a second request reuses the loaded model.
+  - Inference is NOT thread-safe against itself (torch). Callers serialize.
+    The server runs `run()`/`download()` in a threadpool behind a single lock.
+"""
+from __future__ import annotations
+
+import fnmatch
+import re
+import sys
+import threading
+import time
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.resolve()))
+
+import _win_compat  # noqa: F401, E402
+
+from routing import inspect_model, pick_adapter, override_for  # noqa: E402
+from io_utils import resolve_device  # noqa: E402
+
+class DownloadCancelled(Exception):
+    """Raised from the custom tqdm when the user dismisses a download."""
+
+class Engine:
+    """Holds the adapter cache and drives load / run / download / unload."""
+
+    def __init__(self):
+        self._adapter_cache: dict = {}
+        self._current_llm_id: str | None = None
+        self._log = _default_log
+
+    def _get_adapter(self, info: dict):
+        model_id = info["model_id"]
+        adapter = pick_adapter(info)
+        cache_key = (type(adapter).__name__, model_id)
+        cached = self._adapter_cache.get(cache_key)
+        if cached is not None:
+            return cached
+        self._log(f"loading {type(adapter).__name__} for {model_id}")
+        dev = resolve_device()
+        adapter.load(info, dev)
+        self._adapter_cache[cache_key] = adapter
+        return adapter
+
+    def _resolve_info(self, model_id: str, task: str | None) -> dict:
+        info = inspect_model(model_id)
+        if not info.get("pipeline_tag") and task:
+            info["pipeline_tag"] = task
+        return info
+
+    def run(self, model_id: str, task: str | None, inputs: dict, params: dict | None) -> dict:
+        """Execute one inference. Returns an `output_kinds` dict.
+
+        Overrides merge under request params, the model is inspected + routed,
+        the adapter is loaded (or reused) and invoked. Records the model as the
+        current LLM when it's a text generator so the OpenAI endpoint can find
+        it.
+        """
+        if not model_id:
+            raise ValueError("Missing 'modelId' - the session isn't bound to a model")
+
+        inputs = inputs or {}
+
+        override = override_for(model_id) or {}
+        ovr_params = override.get("params") or {}
+        req_params = params or {}
+        merged_params = {**ovr_params, **req_params}
+
+        info = self._resolve_info(model_id, task)
+        adapter = self._get_adapter(info)
+
+        out = adapter.run(inputs, merged_params)
+
+        if _is_text_generation(info, task, out):
+            self._current_llm_id = model_id
+        return out
+
+    def current_llm_id(self) -> str | None:
+        return self._current_llm_id
+
+    def loaded_model_ids(self) -> list[str]:
+        seen = []
+        for (_cls, model_id) in self._adapter_cache.keys():
+            if model_id not in seen:
+                seen.append(model_id)
+        return seen
+
+    def get_cached_adapter(self, model_id: str):
+        """Return a loaded adapter instance for `model_id`, or None."""
+        for (_cls, mid), adapter in self._adapter_cache.items():
+            if mid == model_id:
+                return adapter
+        return None
+
+    def ensure_loaded(self, model_id: str, task: str | None = None):
+        """Load a model without running inference and return its adapter.
+
+        Used by the OpenAI endpoint to lazy-load a model named in the request
+        body when it isn't resident yet.
+        """
+        cached = self.get_cached_adapter(model_id)
+        if cached is not None:
+            return cached
+        info = self._resolve_info(model_id, task)
+        return self._get_adapter(info)
+
+    def unload(self, model_id: str | None = None) -> int:
+        """Drop cached adapter(s), freeing references (GPU memory). Returns the
+        number of adapters unloaded. `None` unloads everything."""
+        keys = [k for k in self._adapter_cache
+                if model_id is None or k[1] == model_id]
+        for k in keys:
+            adapter = self._adapter_cache.pop(k, None)
+            if adapter is not None:
+                try:
+                    adapter.unload()
+                except Exception:
+                    pass
+            if self._current_llm_id == k[1]:
+                self._current_llm_id = None
+        _empty_torch_cache()
+        return len(keys)
+
+    def download(self, model_id: str, on_progress=None, cancel_event: "threading.Event | None" = None) -> dict:
+        """Run `snapshot_download`, streaming byte-level progress via the
+        `on_progress(dict)` callback. Picks exactly one weight format so multi-
+        format repos don't download 4× the bytes. Raises DownloadCancelled if
+        `cancel_event` is set mid-flight.
+
+        Progress is delivered to the `on_progress` callback.
+        """
+        from huggingface_hub import snapshot_download, HfApi
+        from tqdm.auto import tqdm as _BaseTqdm
+
+        if not model_id:
+            raise ValueError("Missing 'modelId'")
+
+        on_progress = on_progress or (lambda _evt: None)
+
+        WEIGHT_FORMAT_ORDER = ["safetensors", "bin", "pt", "ckpt", "msgpack", "h5", "onnx", "ot"]
+        WEIGHT_EXT_RX = re.compile(r"\.(safetensors|bin|pt|ckpt|msgpack|h5|onnx|ot)$", re.IGNORECASE)
+
+        siblings = []
+        try:
+            info = HfApi().model_info(model_id, files_metadata=True)
+            siblings = list(info.siblings or [])
+        except Exception:
+            siblings = []
+
+        chosen_ext = None
+        by_ext: dict = {}
+        for s in siblings:
+            m = WEIGHT_EXT_RX.search((getattr(s, "rfilename", "") or "").lower())
+            if not m:
+                continue
+            by_ext.setdefault(m.group(1), []).append(s)
+        for ext in WEIGHT_FORMAT_ORDER:
+            if by_ext.get(ext):
+                chosen_ext = ext
+                break
+
+        ignore_patterns: list = []
+        if chosen_ext:
+            for ext in WEIGHT_FORMAT_ORDER:
+                if ext != chosen_ext and by_ext.get(ext):
+                    ignore_patterns.append(f"*.{ext}")
+            if chosen_ext != "onnx":
+                ignore_patterns.append("onnx/*")
+
+        total_bytes = 0
+        if siblings:
+            for s in siblings:
+                name = (getattr(s, "rfilename", "") or "").lower()
+                if any(fnmatch.fnmatch(name, p) for p in ignore_patterns):
+                    continue
+                sz = getattr(s, "size", None) or 0
+                if sz:
+                    total_bytes += sz
+
+        state = {"done": 0, "last_emit": 0.0}
+        emit_lock = threading.Lock()
+
+        def emit(final: bool = False) -> None:
+            with emit_lock:
+                done = state["done"]
+                pct = (done / total_bytes * 100.0) if total_bytes else 0.0
+                on_progress({
+                    "done": int(done),
+                    "total": int(total_bytes),
+                    "pct": round(pct, 2),
+                    "final": bool(final),
+                })
+
+        class ProgressTqdm(_BaseTqdm):
+            def __init__(self, *args, **kwargs):
+                self._is_bytes = kwargs.get("unit") == "B"
+                kwargs["disable"] = True
+                super().__init__(*args, **kwargs)
+
+            def update(self, n=1):
+                if cancel_event is not None and cancel_event.is_set():
+                    raise DownloadCancelled()
+                super().update(n)
+                if self._is_bytes and n:
+                    should_emit = False
+                    with emit_lock:
+                        state["done"] += n
+                        now = time.time()
+                        if (now - state["last_emit"]) >= 0.15:
+                            state["last_emit"] = now
+                            should_emit = True
+                    if should_emit:
+                        emit()
+
+        emit()
+        try:
+            kwargs: dict = {"repo_id": model_id, "tqdm_class": ProgressTqdm}
+            if ignore_patterns:
+                kwargs["ignore_patterns"] = ignore_patterns
+            path = snapshot_download(**kwargs)
+        finally:
+            emit(final=True)
+        return {"path": path, "bytes": state["done"], "total_bytes": total_bytes}
+
+def _default_log(msg: str) -> None:
+    print(f"[engine] {msg}", file=sys.stderr, flush=True)
+
+def _is_text_generation(info: dict, task: str | None, out: dict) -> bool:
+    """Best-effort: did this run produce LLM text from a causal model?"""
+    pt = (info.get("pipeline_tag") or task or "").lower()
+    if pt in ("text-generation", "conversational"):
+        return (out or {}).get("kind") == "text"
+    return False
+
+def _empty_torch_cache() -> None:
+    try:
+        import torch
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+    except Exception:
+        pass
+
+def actionable_error(e: Exception) -> str:
+    """Translate raw tracebacks into messages a user can act on.
+
+    Maps common failure modes (OOM, gated repos, missing deps, ...) to guidance
+    the UI and API can surface directly.
+    """
+    msg = str(e)
+    lower = msg.lower()
+    if "out of memory" in lower or "cuda out of memory" in lower:
+        return "Out of memory - try a smaller model or switch to CPU (disable CUDA) in settings."
+    if "cve-2025-32434" in lower or ("torch" in lower and "v2.6" in msg):
+        return ("Your torch version is too old - transformers requires torch ≥ 2.6 to load this model's weights. "
+                "Reinstall LocalML's inference extra with a torch ≥ 2.6 wheel.")
+    if "not a valid" in lower and "trust_remote_code" in lower:
+        return ("This model requires `trust_remote_code=True`. Add an entry for it in "
+                "python/model_overrides.json: { \"trust_remote_code\": true }.")
+    is_gated = (
+        "gatedrepoerror" in lower
+        or "gated repo" in lower
+        or "access to model" in lower and "restricted" in lower
+        or "401" in msg and ("huggingface" in lower or "unauthorized" in lower)
+        or "403" in msg and ("huggingface" in lower or "forbidden" in lower)
+        or "must be authenticated" in lower
+        or "you need to be logged in" in lower
+    )
+    if is_gated:
+        return ("This model is gated or private - it requires a Hugging Face access token. "
+                "Open Settings → HF Token, paste a token from "
+                "https://huggingface.co/settings/tokens (Read access is enough), then retry.")
+    if "no module named" in lower:
+        mod = msg.split("'")[1] if "'" in msg else "unknown"
+        return f"Missing Python package: `{mod}`. Install it into the LocalML environment and retry."
+    m = re.search(r"requires the (\S+) library", msg)
+    if m:
+        mod = m.group(1).strip("`'\".,")
+        return f"Missing Python package: `{mod}`. Install it into the LocalML environment and retry."
+    if "could not load model" in lower or "not a recognized model" in lower:
+        return (f"{msg}\n\nThis model doesn't fit any registered family. Add a folder "
+                "under python/models/ for it, pin it in python/model_overrides.json, "
+                "or drop a plugin file in python/plugins/.")
+    return msg
+
+ENGINE = Engine()

inferml-1.0.1.dist-info/METADATA

@@ -0,0 +1,137 @@
+Metadata-Version: 2.4
+Name: inferml
+Version: 1.0.1
+Summary: Any HuggingFace model. Local. Multi-modal. Served over an OpenAI-compatible API.
+Author: LocalML
+License: MIT License
+        
+        Copyright (c) 2026 LocalML, Gitesh Chawda
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+        THE SOFTWARE.
+        
+Project-URL: Homepage, https://github.com/IMvision12/InferML
+Keywords: huggingface,transformers,inference,openai,local,llm,diffusion
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: fastapi>=0.110
+Requires-Dist: uvicorn[standard]>=0.29
+Requires-Dist: huggingface_hub
+Requires-Dist: platformdirs>=4
+Requires-Dist: psutil>=5.9
+Provides-Extra: inference
+Requires-Dist: transformers>=5.7.0; extra == "inference"
+Requires-Dist: torch>=2.6; extra == "inference"
+Requires-Dist: torchvision; extra == "inference"
+Requires-Dist: torchaudio>=2.6; extra == "inference"
+Requires-Dist: diffusers; extra == "inference"
+Requires-Dist: accelerate; extra == "inference"
+Requires-Dist: timm; extra == "inference"
+Requires-Dist: pillow; extra == "inference"
+Requires-Dist: soundfile; extra == "inference"
+Requires-Dist: librosa; extra == "inference"
+Requires-Dist: numpy; extra == "inference"
+Requires-Dist: scipy; extra == "inference"
+Requires-Dist: sentencepiece; extra == "inference"
+Requires-Dist: protobuf; extra == "inference"
+Dynamic: license-file
+
+<p align="center">
+  <img src="assets/logo.png" alt="LocalML logo" width="140" />
+</p>
+
+# LocalML
+
+Any Hugging Face model. Local. Multi-modal. Now a **local web server** with an
+**OpenAI-compatible API** - no Electron, no native binary.
+
+Run 143+ model families fully on-device (LLMs, VLMs, diffusion, ASR, TTS,
+segmentation, detection) behind a browser UI, and point agent frameworks
+(LangChain, LangGraph, the OpenAI SDK) at it the way you point them at Ollama.
+
+## Install
+
+Requires **Python 3.10+** - the installer checks for it but won't install Python
+for you. One line in your terminal:
+
+```bash
+# macOS / Linux
+curl -fsSL https://www.localml.tech/install.sh | sh
+# Windows (PowerShell)
+irm https://www.localml.tech/install.ps1 | iex
+```
+
+The script bootstraps pipx and installs the LocalML server. On first launch the
+app walks you through installing the inference stack (PyTorch + transformers) for
+your hardware - pick **CPU** or **GPU** and it fetches the matching build.
+
+Prefer to do it by hand?
+
+```bash
+pipx install inferml                 # server only; the app installs torch on first run
+pipx install "inferml[inference]"    # or grab the whole stack up front (generic torch wheel)
+```
+
+## Run
+
+```bash
+localml                 # starts the server and opens http://localhost:11500
+localml --port 8080     # custom port
+localml --host 0.0.0.0 --no-browser   # expose on the LAN, headless
+```
+
+Open the printed URL, download a model from the Hub tab, and run it.
+
+## OpenAI-compatible API
+
+Point any OpenAI client at `http://localhost:11500/v1` (any api key). It routes
+to whichever LLM is currently loaded in LocalML.
+
+```python
+from openai import OpenAI
+client = OpenAI(base_url="http://localhost:11500/v1", api_key="not-needed")
+client.chat.completions.create(
+    model="Qwen/Qwen2.5-0.5B-Instruct",
+    messages=[{"role": "user", "content": "Hello!"}],
+)
+```
+
+Supports streaming (`stream=True`), `GET /v1/models`, and tool/function calling
+for the Qwen/Hermes, Llama, and Mistral families.
+
+## Docker
+
+```bash
+docker build -t localml .
+docker run --rm -p 11500:11500 localml            # CPU
+docker run --rm --gpus all -p 11500:11500 localml # GPU
+```
+
+## Development
+
+The React UI lives in `src/renderer/` (built with esbuild) and talks to the
+server via `window.localml` (see `src/renderer/web-bridge.js`). The Python
+server + inference engine live in `python/`.
+
+```bash
+npm install          # build deps (esbuild + the vendored UMD libs)
+npm run build        # compile the renderer and bundle it into the package
+pip install -e ".[inference]"
+localml
+```