inferml 1.0.1__tar.gz

inferml-1.0.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 LocalML, Gitesh Chawda
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

inferml-1.0.1/MANIFEST.in

@@ -0,0 +1,9 @@
+include LICENSE
+include README.md
+include python/supported_architectures.json
+include python/model_overrides.json
+recursive-include python *.json
+recursive-include python/server/webui *
+recursive-include python/server/_data *
+global-exclude __pycache__/*
+global-exclude *.py[cod]

inferml-1.0.1/PKG-INFO

@@ -0,0 +1,137 @@
+Metadata-Version: 2.4
+Name: inferml
+Version: 1.0.1
+Summary: Any HuggingFace model. Local. Multi-modal. Served over an OpenAI-compatible API.
+Author: LocalML
+License: MIT License
+        
+        Copyright (c) 2026 LocalML, Gitesh Chawda
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+        THE SOFTWARE.
+        
+Project-URL: Homepage, https://github.com/IMvision12/InferML
+Keywords: huggingface,transformers,inference,openai,local,llm,diffusion
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: fastapi>=0.110
+Requires-Dist: uvicorn[standard]>=0.29
+Requires-Dist: huggingface_hub
+Requires-Dist: platformdirs>=4
+Requires-Dist: psutil>=5.9
+Provides-Extra: inference
+Requires-Dist: transformers>=5.7.0; extra == "inference"
+Requires-Dist: torch>=2.6; extra == "inference"
+Requires-Dist: torchvision; extra == "inference"
+Requires-Dist: torchaudio>=2.6; extra == "inference"
+Requires-Dist: diffusers; extra == "inference"
+Requires-Dist: accelerate; extra == "inference"
+Requires-Dist: timm; extra == "inference"
+Requires-Dist: pillow; extra == "inference"
+Requires-Dist: soundfile; extra == "inference"
+Requires-Dist: librosa; extra == "inference"
+Requires-Dist: numpy; extra == "inference"
+Requires-Dist: scipy; extra == "inference"
+Requires-Dist: sentencepiece; extra == "inference"
+Requires-Dist: protobuf; extra == "inference"
+Dynamic: license-file
+
+<p align="center">
+  <img src="assets/logo.png" alt="LocalML logo" width="140" />
+</p>
+
+# LocalML
+
+Any Hugging Face model. Local. Multi-modal. Now a **local web server** with an
+**OpenAI-compatible API** - no Electron, no native binary.
+
+Run 143+ model families fully on-device (LLMs, VLMs, diffusion, ASR, TTS,
+segmentation, detection) behind a browser UI, and point agent frameworks
+(LangChain, LangGraph, the OpenAI SDK) at it the way you point them at Ollama.
+
+## Install
+
+Requires **Python 3.10+** - the installer checks for it but won't install Python
+for you. One line in your terminal:
+
+```bash
+# macOS / Linux
+curl -fsSL https://www.localml.tech/install.sh | sh
+# Windows (PowerShell)
+irm https://www.localml.tech/install.ps1 | iex
+```
+
+The script bootstraps pipx and installs the LocalML server. On first launch the
+app walks you through installing the inference stack (PyTorch + transformers) for
+your hardware - pick **CPU** or **GPU** and it fetches the matching build.
+
+Prefer to do it by hand?
+
+```bash
+pipx install inferml                 # server only; the app installs torch on first run
+pipx install "inferml[inference]"    # or grab the whole stack up front (generic torch wheel)
+```
+
+## Run
+
+```bash
+localml                 # starts the server and opens http://localhost:11500
+localml --port 8080     # custom port
+localml --host 0.0.0.0 --no-browser   # expose on the LAN, headless
+```
+
+Open the printed URL, download a model from the Hub tab, and run it.
+
+## OpenAI-compatible API
+
+Point any OpenAI client at `http://localhost:11500/v1` (any api key). It routes
+to whichever LLM is currently loaded in LocalML.
+
+```python
+from openai import OpenAI
+client = OpenAI(base_url="http://localhost:11500/v1", api_key="not-needed")
+client.chat.completions.create(
+    model="Qwen/Qwen2.5-0.5B-Instruct",
+    messages=[{"role": "user", "content": "Hello!"}],
+)
+```
+
+Supports streaming (`stream=True`), `GET /v1/models`, and tool/function calling
+for the Qwen/Hermes, Llama, and Mistral families.
+
+## Docker
+
+```bash
+docker build -t localml .
+docker run --rm -p 11500:11500 localml            # CPU
+docker run --rm --gpus all -p 11500:11500 localml # GPU
+```
+
+## Development
+
+The React UI lives in `src/renderer/` (built with esbuild) and talks to the
+server via `window.localml` (see `src/renderer/web-bridge.js`). The Python
+server + inference engine live in `python/`.
+
+```bash
+npm install          # build deps (esbuild + the vendored UMD libs)
+npm run build        # compile the renderer and bundle it into the package
+pip install -e ".[inference]"
+localml
+```

inferml-1.0.1/README.md

@@ -0,0 +1,83 @@
+<p align="center">
+  <img src="assets/logo.png" alt="LocalML logo" width="140" />
+</p>
+
+# LocalML
+
+Any Hugging Face model. Local. Multi-modal. Now a **local web server** with an
+**OpenAI-compatible API** - no Electron, no native binary.
+
+Run 143+ model families fully on-device (LLMs, VLMs, diffusion, ASR, TTS,
+segmentation, detection) behind a browser UI, and point agent frameworks
+(LangChain, LangGraph, the OpenAI SDK) at it the way you point them at Ollama.
+
+## Install
+
+Requires **Python 3.10+** - the installer checks for it but won't install Python
+for you. One line in your terminal:
+
+```bash
+# macOS / Linux
+curl -fsSL https://www.localml.tech/install.sh | sh
+# Windows (PowerShell)
+irm https://www.localml.tech/install.ps1 | iex
+```
+
+The script bootstraps pipx and installs the LocalML server. On first launch the
+app walks you through installing the inference stack (PyTorch + transformers) for
+your hardware - pick **CPU** or **GPU** and it fetches the matching build.
+
+Prefer to do it by hand?
+
+```bash
+pipx install inferml                 # server only; the app installs torch on first run
+pipx install "inferml[inference]"    # or grab the whole stack up front (generic torch wheel)
+```
+
+## Run
+
+```bash
+localml                 # starts the server and opens http://localhost:11500
+localml --port 8080     # custom port
+localml --host 0.0.0.0 --no-browser   # expose on the LAN, headless
+```
+
+Open the printed URL, download a model from the Hub tab, and run it.
+
+## OpenAI-compatible API
+
+Point any OpenAI client at `http://localhost:11500/v1` (any api key). It routes
+to whichever LLM is currently loaded in LocalML.
+
+```python
+from openai import OpenAI
+client = OpenAI(base_url="http://localhost:11500/v1", api_key="not-needed")
+client.chat.completions.create(
+    model="Qwen/Qwen2.5-0.5B-Instruct",
+    messages=[{"role": "user", "content": "Hello!"}],
+)
+```
+
+Supports streaming (`stream=True`), `GET /v1/models`, and tool/function calling
+for the Qwen/Hermes, Llama, and Mistral families.
+
+## Docker
+
+```bash
+docker build -t localml .
+docker run --rm -p 11500:11500 localml            # CPU
+docker run --rm --gpus all -p 11500:11500 localml # GPU
+```
+
+## Development
+
+The React UI lives in `src/renderer/` (built with esbuild) and talks to the
+server via `window.localml` (see `src/renderer/web-bridge.js`). The Python
+server + inference engine live in `python/`.
+
+```bash
+npm install          # build deps (esbuild + the vendored UMD libs)
+npm run build        # compile the renderer and bundle it into the package
+pip install -e ".[inference]"
+localml
+```

inferml-1.0.1/pyproject.toml

@@ -0,0 +1,67 @@
+[build-system]
+requires = ["setuptools>=68", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "inferml"
+version = "1.0.1"
+description = "Any HuggingFace model. Local. Multi-modal. Served over an OpenAI-compatible API."
+readme = "README.md"
+requires-python = ">=3.10"
+license = { file = "LICENSE" }
+authors = [{ name = "LocalML" }]
+keywords = ["huggingface", "transformers", "inference", "openai", "local", "llm", "diffusion"]
+
+# The web server layer. The inference stack (torch, transformers, diffusers, …)
+# is declared in the `inference` extra so `pipx install inferml` stays fast and
+# the heavy, hardware-specific torch wheels can be pinned to the right index by
+# the user. Phase 5 finalizes the CPU/GPU install story and bundles the built
+# frontend as package data.
+dependencies = [
+  "fastapi>=0.110",
+  "uvicorn[standard]>=0.29",
+  "huggingface_hub",
+  "platformdirs>=4",
+  "psutil>=5.9",
+]
+
+[project.optional-dependencies]
+inference = [
+  "transformers>=5.7.0",
+  "torch>=2.6",
+  "torchvision",
+  "torchaudio>=2.6",
+  "diffusers",
+  "accelerate",
+  "timm",
+  "pillow",
+  "soundfile",
+  "librosa",
+  "numpy",
+  "scipy",
+  "sentencepiece",
+  "protobuf",
+]
+
+[project.scripts]
+localml = "server.cli:main"
+
+[project.urls]
+Homepage = "https://github.com/IMvision12/InferML"
+
+# The Python sources live under python/ as flat modules + subpackages (the
+# inference engine imports `routing`, `adapters`, `tasks`, `models` at top
+# level). Map the package root to python/ and ship both shapes.
+[tool.setuptools]
+package-dir = { "" = "python" }
+py-modules = ["engine", "routing", "io_utils", "output_kinds", "_win_compat"]
+
+[tool.setuptools.packages.find]
+where = ["python"]
+include = ["server*", "adapters*", "tasks*", "models*", "plugins*"]
+
+[tool.setuptools.package-data]
+# Per-family JSON + the compiled frontend and routing tables bundled inside the
+# `server` package by `npm run build` (scripts/bundle-webui.js).
+"*" = ["*.json"]
+"server" = ["webui/**/*", "_data/**/*"]

inferml-1.0.1/python/_win_compat.py

@@ -0,0 +1,52 @@
+"""Windows compatibility patches applied process-wide at sidecar boot.
+
+Currently:
+- os.symlink → transparent copy fallback when the caller lacks
+  SeCreateSymbolicLinkPrivilege (the WinError 1314 case). HuggingFace's
+  cache layout uses symlinks to dedup blobs across snapshots, and a
+  standard non-admin user without Developer Mode hits this on every
+  download. POSIX users keep the real os.symlink (symlinks always work
+  there).
+
+Import this module before any other library that may call os.symlink.
+On non-Windows platforms the import is a no-op.
+"""
+from __future__ import annotations
+
+import os
+import shutil
+import sys
+
+
+def _install_symlink_copy_fallback() -> None:
+    if sys.platform != "win32":
+        return
+
+    _orig = os.symlink
+
+    def _resolve(src, dst):
+        if os.path.isabs(src):
+            return src
+        return os.path.normpath(os.path.join(os.path.dirname(dst), src))
+
+    def _symlink(src, dst, target_is_directory=False, *, dir_fd=None):
+        try:
+            return _orig(src, dst, target_is_directory=target_is_directory, dir_fd=dir_fd)
+        except OSError as e:
+            is_privilege_error = (
+                getattr(e, "winerror", None) == 1314
+                or "privilege" in str(e).lower()
+            )
+            if not is_privilege_error:
+                raise
+            real_src = _resolve(src, dst)
+            if target_is_directory or os.path.isdir(real_src):
+                shutil.copytree(real_src, dst, dirs_exist_ok=True)
+            else:
+                shutil.copyfile(real_src, dst)
+            return None
+
+    os.symlink = _symlink
+
+
+_install_symlink_copy_fallback()

inferml-1.0.1/python/adapters/__init__.py

@@ -0,0 +1,65 @@
+"""Adapter base classes + shared catch-all adapters.
+
+Per-family inference code lives in `python/models/<family>/`. This package
+holds only the cross-cutting pieces:
+
+  - `Adapter`                  the base class every family inherits from
+  - `StandardPipelineAdapter`  fallback for repos with no dedicated family
+  - `DiffusersAdapter`         library passthrough for diffusers checkpoints
+
+Routing strategy lives in `routing.py` and is:
+
+    1. Named override (via model_overrides.json `"adapter"` field)
+    2. Plugin adapters (python/plugins/*.py)
+    3. models/<family>/ registry (per-family folders)
+    4. DiffusersAdapter (library == "diffusers")
+    5. StandardPipelineAdapter (pipeline_tag in its task list)
+"""
+from __future__ import annotations
+
+from .base import Adapter  # noqa: F401
+from .standard_pipeline import StandardPipelineAdapter
+from .diffusers_pipeline import DiffusersAdapter
+
+def _named_adapters() -> dict[str, type]:
+    """Build the name→class map used by `model_overrides.json "adapter"` pins.
+
+    Includes the cross-cutting fallbacks plus every family folder in
+    `python/models/`. Built LAZILY (see `__getattr__` below) so we don't
+    capture a partially-loaded `models.FAMILIES` if some caller imports
+    `models` before `adapters` and the family-folder-discovery chain
+    re-enters this module mid-load.
+    """
+    out: dict[str, type] = {
+        "standard":  StandardPipelineAdapter,
+        "diffusers": DiffusersAdapter,
+    }
+    try:
+        from models import FAMILIES
+        for fam_name, entry in FAMILIES.items():
+            cls = entry.get("adapter")
+            if cls is None:
+                continue
+            out[fam_name] = cls
+            short = cls.__name__.replace("Adapter", "").lower()
+            out.setdefault(short, cls)
+    except Exception:
+        pass
+    return out
+
+_NAMED_ADAPTERS_CACHE: "dict[str, type] | None" = None
+
+def __getattr__(name: str):
+    global _NAMED_ADAPTERS_CACHE
+    if name == "NAMED_ADAPTERS":
+        if _NAMED_ADAPTERS_CACHE is None:
+            _NAMED_ADAPTERS_CACHE = _named_adapters()
+        return _NAMED_ADAPTERS_CACHE
+    raise AttributeError(f"module 'adapters' has no attribute {name!r}")
+
+__all__ = [
+    "Adapter",
+    "StandardPipelineAdapter",
+    "DiffusersAdapter",
+    "NAMED_ADAPTERS",
+]

inferml-1.0.1/python/adapters/base.py

@@ -0,0 +1,37 @@
+"""Adapter base class.
+
+One instance of an adapter = one loaded model. The router picks the adapter,
+calls `load(info, device)` once, then `run(inputs, params)` per request.
+Instances are cached by (adapter_class, model_id) in the engine.
+"""
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+
+class Adapter(ABC):
+    override: dict = {}
+
+    @classmethod
+    def can_handle(cls, info: dict) -> bool:
+        """Return True if this adapter can run the described model.
+
+        `info` is the dict from routing.inspect_model. Implementations should
+        inspect `model_id`, `model_type`, `architectures`, `tags`, etc. -
+        *not* download any weights."""
+        return False
+
+    @abstractmethod
+    def load(self, info: dict, device) -> None:
+        """Instantiate the underlying model + any helpers (processor, tokenizer)."""
+
+    @abstractmethod
+    def run(self, inputs: dict, params: dict) -> dict:
+        """Execute inference. Must return a dict matching one of the kinds in
+        `output_kinds.py` (`boxes`, `masks`, `labels`, `text`, `image`,
+        `audio`, `vector`)."""
+
+    def unload(self) -> None:
+        """Hook for freeing GPU memory - default: drop references."""
+        for attr in list(self.__dict__.keys()):
+            if attr not in ("override",):
+                setattr(self, attr, None)

inferml-1.0.1/python/adapters/diffusers_pipeline.py

@@ -0,0 +1,57 @@
+"""Diffusers fallback. text-to-image, img2img, inpainting."""
+from __future__ import annotations
+
+from .base import Adapter
+import output_kinds as ok
+from io_utils import decode_image, resolve_device, torch_dtype_for_device
+
+class DiffusersAdapter(Adapter):
+    @classmethod
+    def can_handle(cls, info):
+        if info.get("library") == "diffusers":
+            return info.get("pipeline_tag") != "text-to-video"
+        tag = info.get("pipeline_tag")
+        return tag in ("text-to-image", "image-to-image", "inpainting")
+
+    def load(self, info, device):
+        self.info = info
+        self.device = device
+        self.task = info.get("pipeline_tag") or "text-to-image"
+        dtype = torch_dtype_for_device()
+
+        from diffusers import (
+            AutoPipelineForText2Image,
+            AutoPipelineForImage2Image,
+            AutoPipelineForInpainting,
+        )
+        cls = {
+            "image-to-image": AutoPipelineForImage2Image,
+            "inpainting":     AutoPipelineForInpainting,
+        }.get(self.task, AutoPipelineForText2Image)
+
+        kwargs = {"torch_dtype": dtype}
+        if self.override.get("trust_remote_code"):
+            kwargs["trust_remote_code"] = True
+        self.pipe = cls.from_pretrained(info["model_id"], **kwargs)
+        resolved = resolve_device()
+        if resolved is not False:
+            self.pipe = self.pipe.to(resolved)
+
+    def run(self, inputs, params):
+        prompt = (inputs.get("text") or "").strip()
+        if not prompt:
+            raise ValueError("Prompt required")
+        kwargs = {k: params[k] for k in
+                  ("num_inference_steps", "guidance_scale", "negative_prompt", "strength")
+                  if k in params}
+        kwargs.setdefault("num_inference_steps", 20)
+        kwargs.setdefault("guidance_scale", 7.5)
+
+        if self.task == "image-to-image" and inputs.get("dataUrl"):
+            kwargs["image"] = decode_image(inputs["dataUrl"])
+        elif self.task == "inpainting" and inputs.get("dataUrl"):
+            kwargs["image"] = decode_image(inputs["dataUrl"])
+
+        result = self.pipe(prompt, **kwargs)
+        image = result.images[0]
+        return ok.image(image)

inferml-1.0.1/python/adapters/standard_pipeline.py

@@ -0,0 +1,31 @@
+"""Standard HF pipeline adapter. Fallback when no `models/<family>/` matches.
+
+Thin dispatcher: the actual per-task logic lives in `python/tasks/`. When a
+model breaks, add a folder under `python/models/` (preferred) or a Variant
+in the relevant task file.
+"""
+from __future__ import annotations
+
+from .base import Adapter
+from tasks import TASK_REGISTRY, get_task
+
+
+class StandardPipelineAdapter(Adapter):
+    SUPPORTED_TASKS = set(TASK_REGISTRY.keys())
+
+    @classmethod
+    def can_handle(cls, info):
+        return info.get("pipeline_tag") in cls.SUPPORTED_TASKS
+
+    def load(self, info, device):
+        self.info = info
+        self.device = device
+        self.task_name = info["pipeline_tag"]
+        self.handler = get_task(self.task_name)
+        if self.handler is None:
+            raise ValueError(f"No task handler registered for {self.task_name!r}")
+        extra = {"trust_remote_code": True} if self.override.get("trust_remote_code") else {}
+        self.state = self.handler.load_pipeline(info, device, extra_kwargs=extra)
+
+    def run(self, inputs, params):
+        return self.handler.handle(self.state, inputs, params)