llms-py 2.0.35__py3-none-any.whl → 3.0.0__py3-none-any.whl

llms/main.py

@@ -9,20 +9,27 @@
 import argparse
 import asyncio
 import base64
+import contextlib
+import hashlib
+import importlib.util
+import inspect
 import json
 import mimetypes
 import os
 import re
 import secrets
+import shutil
 import site
 import subprocess
 import sys
 import time
 import traceback
+from datetime import datetime
 from importlib import resources  # Py≥3.9  (pip install importlib_resources for 3.7/3.8)
 from io import BytesIO
 from pathlib import Path
-from urllib.parse import parse_qs, urlencode
+from typing import get_type_hints
+from urllib.parse import parse_qs, urlencode, urljoin
 
 import aiohttp
 from aiohttp import web
@@ -34,25 +41,40 @@ try:
 except ImportError:
     HAS_PIL = False
 
-VERSION = "2.0.35"
+VERSION = "3.0.0"
 _ROOT = None
+DEBUG = os.getenv("DEBUG") == "1"
+MOCK = os.getenv("MOCK") == "1"
+MOCK_DIR = os.getenv("MOCK_DIR")
+DISABLE_EXTENSIONS = (os.getenv("LLMS_DISABLE") or "").split(",")
 g_config_path = None
-g_ui_path = None
 g_config = None
+g_providers = None
 g_handlers = {}
 g_verbose = False
 g_logprefix = ""
 g_default_model = ""
 g_sessions = {}  # OAuth session storage: {session_token: {userId, userName, displayName, profileUrl, email, created}}
 g_oauth_states = {}  # CSRF protection: {state: {created, redirect_uri}}
+g_app = None  # ExtensionsContext Singleton
 
 
 def _log(message):
-    """Helper method for logging from the global polling task."""
     if g_verbose:
         print(f"{g_logprefix}{message}", flush=True)
 
 
+def _dbg(message):
+    if DEBUG:
+        print(f"DEBUG: {message}", flush=True)
+
+
+def _err(message, e):
+    print(f"ERROR: {message}: {e}", flush=True)
+    if g_verbose:
+        print(traceback.format_exc(), flush=True)
+
+
 def printdump(obj):
     args = obj.__dict__ if hasattr(obj, "__dict__") else obj
     print(json.dumps(args, indent=2))
@@ -85,17 +107,6 @@ def chat_summary(chat):
     return json.dumps(clone, indent=2)
 
 
-def gemini_chat_summary(gemini_chat):
-    """Summarize Gemini chat completion request for logging. Replace inline_data with size of content only"""
-    clone = json.loads(json.dumps(gemini_chat))
-    for content in clone["contents"]:
-        for part in content["parts"]:
-            if "inline_data" in part:
-                data = part["inline_data"]["data"]
-                part["inline_data"]["data"] = f"({len(data)})"
-    return json.dumps(clone, indent=2)
-
-
 image_exts = ["png", "webp", "jpg", "jpeg", "gif", "bmp", "svg", "tiff", "ico"]
 audio_exts = ["mp3", "wav", "ogg", "flac", "m4a", "opus", "webm"]
 
@@ -189,6 +200,16 @@ def is_base_64(data):
         return False
 
 
+def id_to_name(id):
+    return id.replace("-", " ").title()
+
+
+def pluralize(word, count):
+    if count == 1:
+        return word
+    return word + "s"
+
+
 def get_file_mime_type(filename):
     mime_type, _ = mimetypes.guess_type(filename)
     return mime_type or "application/octet-stream"
@@ -310,11 +331,52 @@ def convert_image_if_needed(image_bytes, mimetype="image/png"):
         return image_bytes, mimetype
 
 
-async def process_chat(chat):
+def to_content(result):
+    if isinstance(result, (str, int, float, bool)):
+        return str(result)
+    elif isinstance(result, (list, set, tuple, dict)):
+        return json.dumps(result)
+    else:
+        return str(result)
+
+
+def function_to_tool_definition(func):
+    type_hints = get_type_hints(func)
+    signature = inspect.signature(func)
+    parameters = {"type": "object", "properties": {}, "required": []}
+
+    for name, param in signature.parameters.items():
+        param_type = type_hints.get(name, str)
+        param_type_name = "string"
+        if param_type is int:
+            param_type_name = "integer"
+        elif param_type is float:
+            param_type_name = "number"
+        elif param_type is bool:
+            param_type_name = "boolean"
+
+        parameters["properties"][name] = {"type": param_type_name}
+        if param.default == inspect.Parameter.empty:
+            parameters["required"].append(name)
+
+    return {
+        "type": "function",
+        "function": {
+            "name": func.__name__,
+            "description": func.__doc__ or "",
+            "parameters": parameters,
+        },
+    }
+
+
+async def process_chat(chat, provider_id=None):
     if not chat:
         raise Exception("No chat provided")
     if "stream" not in chat:
         chat["stream"] = False
+    # Some providers don't support empty tools
+    if "tools" in chat and len(chat["tools"]) == 0:
+        del chat["tools"]
     if "messages" not in chat:
         return chat
 
@@ -331,6 +393,8 @@ async def process_chat(chat):
                         image_url = item["image_url"]
                         if "url" in image_url:
                             url = image_url["url"]
+                            if url.startswith("/~cache/"):
+                                url = get_cache_path(url[8:])
                             if is_url(url):
                                 _log(f"Downloading image: {url}")
                                 async with session.get(url, timeout=aiohttp.ClientTimeout(total=120)) as response:
@@ -377,6 +441,8 @@ async def process_chat(chat):
                         input_audio = item["input_audio"]
                         if "data" in input_audio:
                             url = input_audio["data"]
+                            if url.startswith("/~cache/"):
+                                url = get_cache_path(url[8:])
                             mimetype = get_file_mime_type(get_filename(url))
                             if is_url(url):
                                 _log(f"Downloading audio: {url}")
@@ -388,6 +454,8 @@ async def process_chat(chat):
                                         mimetype = response.headers["Content-Type"]
                                     # convert to base64
                                     input_audio["data"] = base64.b64encode(content).decode("utf-8")
+                                    if provider_id == "alibaba":
+                                        input_audio["data"] = f"data:{mimetype};base64,{input_audio['data']}"
                                     input_audio["format"] = mimetype.rsplit("/", 1)[1]
                             elif is_file_path(url):
                                 _log(f"Reading audio: {url}")
@@ -395,6 +463,8 @@ async def process_chat(chat):
                                     content = f.read()
                                     # convert to base64
                                     input_audio["data"] = base64.b64encode(content).decode("utf-8")
+                                    if provider_id == "alibaba":
+                                        input_audio["data"] = f"data:{mimetype};base64,{input_audio['data']}"
                                     input_audio["format"] = mimetype.rsplit("/", 1)[1]
                             elif is_base_64(url):
                                 pass  # use base64 data as-is
@@ -404,6 +474,8 @@ async def process_chat(chat):
                         file = item["file"]
                         if "file_data" in file:
                             url = file["file_data"]
+                            if url.startswith("/~cache/"):
+                                url = get_cache_path(url[8:])
                             mimetype = get_file_mime_type(get_filename(url))
                             if is_url(url):
                                 _log(f"Downloading file: {url}")
@@ -431,6 +503,92 @@ async def process_chat(chat):
     return chat
 
 
+def image_ext_from_mimetype(mimetype, default="png"):
+    if "/" in mimetype:
+        _ext = mimetypes.guess_extension(mimetype)
+        if _ext:
+            return _ext.lstrip(".")
+    return default
+
+
+def audio_ext_from_format(format, default="mp3"):
+    if format == "mpeg":
+        return "mp3"
+    return format or default
+
+
+def file_ext_from_mimetype(mimetype, default="pdf"):
+    if "/" in mimetype:
+        _ext = mimetypes.guess_extension(mimetype)
+        if _ext:
+            return _ext.lstrip(".")
+    return default
+
+
+def cache_message_inline_data(m):
+    """
+    Replaces and caches any inline data URIs in the message content.
+    """
+    if "content" not in m:
+        return
+
+    content = m["content"]
+    if isinstance(content, list):
+        for item in content:
+            if item.get("type") == "image_url":
+                image_url = item.get("image_url", {})
+                url = image_url.get("url")
+                if url and url.startswith("data:"):
+                    # Extract base64 and mimetype
+                    try:
+                        header, base64_data = url.split(";base64,")
+                        # header is like "data:image/png"
+                        ext = image_ext_from_mimetype(header.split(":")[1])
+                        filename = f"image.{ext}"  # Hash will handle uniqueness
+
+                        cache_url, _ = save_image_to_cache(base64_data, filename, {}, ignore_info=True)
+                        image_url["url"] = cache_url
+                    except Exception as e:
+                        _log(f"Error caching inline image: {e}")
+
+            elif item.get("type") == "input_audio":
+                input_audio = item.get("input_audio", {})
+                data = input_audio.get("data")
+                if data:
+                    # Handle data URI or raw base64
+                    base64_data = data
+                    if data.startswith("data:"):
+                        with contextlib.suppress(ValueError):
+                            header, base64_data = data.split(";base64,")
+
+                    fmt = audio_ext_from_format(input_audio.get("format"))
+                    filename = f"audio.{fmt}"
+
+                    try:
+                        cache_url, _ = save_bytes_to_cache(base64_data, filename, {}, ignore_info=True)
+                        input_audio["data"] = cache_url
+                    except Exception as e:
+                        _log(f"Error caching inline audio: {e}")
+
+            elif item.get("type") == "file":
+                file_info = item.get("file", {})
+                file_data = file_info.get("file_data")
+                if file_data and file_data.startswith("data:"):
+                    try:
+                        header, base64_data = file_data.split(";base64,")
+                        mimetype = header.split(":")[1]
+                        # Try to get extension from filename if available, else mimetype
+                        filename = file_info.get("filename", "file")
+                        if "." not in filename:
+                            ext = file_ext_from_mimetype(mimetype)
+                            filename = f"{filename}.{ext}"
+
+                        cache_url, _ = save_bytes_to_cache(base64_data, filename, {}, ignore_info=True)
+                        file_info["file_data"] = cache_url
+                    except Exception as e:
+                        _log(f"Error caching inline file: {e}")
+
+
 class HTTPError(Exception):
     def __init__(self, status, reason, body, headers=None):
         self.status = status
@@ -440,33 +598,323 @@ class HTTPError(Exception):
         super().__init__(f"HTTP {status} {reason}")
 
 
+def save_bytes_to_cache(base64_data, filename, file_info, ignore_info=False):
+    ext = filename.split(".")[-1]
+    mimetype = get_file_mime_type(filename)
+    content = base64.b64decode(base64_data) if isinstance(base64_data, str) else base64_data
+    sha256_hash = hashlib.sha256(content).hexdigest()
+
+    save_filename = f"{sha256_hash}.{ext}" if ext else sha256_hash
+
+    # Use first 2 chars for subdir to avoid too many files in one dir
+    subdir = sha256_hash[:2]
+    relative_path = f"{subdir}/{save_filename}"
+    full_path = get_cache_path(relative_path)
+    url = f"/~cache/{relative_path}"
+
+    # if file and its .info.json already exists, return it
+    info_path = os.path.splitext(full_path)[0] + ".info.json"
+    if os.path.exists(full_path) and os.path.exists(info_path):
+        _dbg(f"Cached bytes exists: {relative_path}")
+        if ignore_info:
+            return url, None
+        return url, json.load(open(info_path))
+
+    os.makedirs(os.path.dirname(full_path), exist_ok=True)
+
+    with open(full_path, "wb") as f:
+        f.write(content)
+    info = {
+        "date": int(time.time()),
+        "url": url,
+        "size": len(content),
+        "type": mimetype,
+        "name": filename,
+    }
+    info.update(file_info)
+
+    # Save metadata
+    info_path = os.path.splitext(full_path)[0] + ".info.json"
+    with open(info_path, "w") as f:
+        json.dump(info, f)
+
+    _dbg(f"Saved cached bytes and info: {relative_path}")
+
+    g_app.on_cache_saved_filters({"url": url, "info": info})
+
+    return url, info
+
+
+def save_image_to_cache(base64_data, filename, image_info, ignore_info=False):
+    ext = filename.split(".")[-1]
+    mimetype = get_file_mime_type(filename)
+    content = base64.b64decode(base64_data) if isinstance(base64_data, str) else base64_data
+    sha256_hash = hashlib.sha256(content).hexdigest()
+
+    save_filename = f"{sha256_hash}.{ext}" if ext else sha256_hash
+
+    # Use first 2 chars for subdir to avoid too many files in one dir
+    subdir = sha256_hash[:2]
+    relative_path = f"{subdir}/{save_filename}"
+    full_path = get_cache_path(relative_path)
+    url = f"/~cache/{relative_path}"
+
+    # if file and its .info.json already exists, return it
+    info_path = os.path.splitext(full_path)[0] + ".info.json"
+    if os.path.exists(full_path) and os.path.exists(info_path):
+        _dbg(f"Saved image exists: {relative_path}")
+        if ignore_info:
+            return url, None
+        return url, json.load(open(info_path))
+
+    os.makedirs(os.path.dirname(full_path), exist_ok=True)
+
+    with open(full_path, "wb") as f:
+        f.write(content)
+    info = {
+        "date": int(time.time()),
+        "url": url,
+        "size": len(content),
+        "type": mimetype,
+        "name": filename,
+    }
+    info.update(image_info)
+
+    # If image, get dimensions
+    if HAS_PIL and mimetype.startswith("image/"):
+        try:
+            with Image.open(BytesIO(content)) as img:
+                info["width"] = img.width
+                info["height"] = img.height
+        except Exception:
+            pass
+
+    if "width" in info and "height" in info:
+        _log(f"Saved image to cache: {full_path} ({len(content)} bytes) {info['width']}x{info['height']}")
+    else:
+        _log(f"Saved image to cache: {full_path} ({len(content)} bytes)")
+
+    # Save metadata
+    info_path = os.path.splitext(full_path)[0] + ".info.json"
+    with open(info_path, "w") as f:
+        json.dump(info, f)
+
+    _dbg(f"Saved image and info: {relative_path}")
+
+    g_app.on_cache_saved_filters({"url": url, "info": info})
+
+    return url, info
+
+
 async def response_json(response):
     text = await response.text()
     if response.status >= 400:
+        _dbg(f"HTTP {response.status} {response.reason}: {text}")
         raise HTTPError(response.status, reason=response.reason, body=text, headers=dict(response.headers))
     response.raise_for_status()
     body = json.loads(text)
     return body
 
 
-class OpenAiProvider:
-    def __init__(self, base_url, api_key=None, models=None, **kwargs):
-        if models is None:
-            models = {}
-        self.base_url = base_url.strip("/")
-        self.api_key = api_key
-        self.models = models
+def chat_to_prompt(chat):
+    prompt = ""
+    if "messages" in chat:
+        for message in chat["messages"]:
+            if message["role"] == "user":
+                # if content is string
+                if isinstance(message["content"], str):
+                    if prompt:
+                        prompt += "\n"
+                    prompt += message["content"]
+                elif isinstance(message["content"], list):
+                    # if content is array of objects
+                    for part in message["content"]:
+                        if part["type"] == "text":
+                            if prompt:
+                                prompt += "\n"
+                            prompt += part["text"]
+    return prompt
+
+
+def chat_to_system_prompt(chat):
+    if "messages" in chat:
+        for message in chat["messages"]:
+            if message["role"] == "system":
+                # if content is string
+                if isinstance(message["content"], str):
+                    return message["content"]
+                elif isinstance(message["content"], list):
+                    # if content is array of objects
+                    for part in message["content"]:
+                        if part["type"] == "text":
+                            return part["text"]
+    return None
 
-        # check if base_url ends with /v{\d} to handle providers with different versions (e.g. z.ai uses /v4)
-        last_segment = base_url.rsplit("/", 1)[1]
-        if last_segment.startswith("v") and last_segment[1:].isdigit():
-            self.chat_url = f"{base_url}/chat/completions"
-        else:
-            self.chat_url = f"{base_url}/v1/chat/completions"
+
+def chat_to_username(chat):
+    if "metadata" in chat and "user" in chat["metadata"]:
+        return chat["metadata"]["user"]
+    return None
+
+
+def last_user_prompt(chat):
+    prompt = ""
+    if "messages" in chat:
+        for message in chat["messages"]:
+            if message["role"] == "user":
+                # if content is string
+                if isinstance(message["content"], str):
+                    prompt = message["content"]
+                elif isinstance(message["content"], list):
+                    # if content is array of objects
+                    for part in message["content"]:
+                        if part["type"] == "text":
+                            prompt = part["text"]
+    return prompt
+
+
+def chat_response_to_message(openai_response):
+    """
+    Returns an assistant message from the OpenAI Response.
+    Handles normalizing text, image, and audio responses into the message content.
+    """
+    timestamp = int(time.time() * 1000)  # openai_response.get("created")
+    choices = openai_response
+    if isinstance(openai_response, dict) and "choices" in openai_response:
+        choices = openai_response["choices"]
+
+    choice = choices[0] if isinstance(choices, list) and choices else choices
+
+    if isinstance(choice, str):
+        return {"role": "assistant", "content": choice, "timestamp": timestamp}
+
+    if isinstance(choice, dict):
+        message = choice.get("message", choice)
+    else:
+        return {"role": "assistant", "content": str(choice), "timestamp": timestamp}
+
+    # Ensure message is a dict
+    if not isinstance(message, dict):
+        return {"role": "assistant", "content": message, "timestamp": timestamp}
+
+    message.update({"timestamp": timestamp})
+    return message
+
+
+def to_file_info(chat, info=None, response=None):
+    prompt = last_user_prompt(chat)
+    ret = info or {}
+    if chat["model"] and "model" not in ret:
+        ret["model"] = chat["model"]
+    if prompt and "prompt" not in ret:
+        ret["prompt"] = prompt
+    if "image_config" in chat:
+        ret.update(chat["image_config"])
+    user = chat_to_username(chat)
+    if user:
+        ret["user"] = user
+    return ret
+
+
+# Image Generator Providers
+class GeneratorBase:
+    def __init__(self, **kwargs):
+        self.id = kwargs.get("id")
+        self.api = kwargs.get("api")
+        self.api_key = kwargs.get("api_key")
+        self.headers = {
+            "Accept": "application/json",
+            "Content-Type": "application/json",
+        }
+        self.chat_url = f"{self.api}/chat/completions"
+        self.default_content = "I've generated the image for you."
+
+    def validate(self, **kwargs):
+        if not self.api_key:
+            api_keys = ", ".join(self.env)
+            return f"Provider '{self.name}' requires API Key {api_keys}"
+        return None
+
+    def test(self, **kwargs):
+        error_msg = self.validate(**kwargs)
+        if error_msg:
+            _log(error_msg)
+            return False
+        return True
+
+    async def load(self):
+        pass
+
+    def gen_summary(self, gen):
+        """Summarize gen response for logging."""
+        clone = json.loads(json.dumps(gen))
+        return json.dumps(clone, indent=2)
+
+    def chat_summary(self, chat):
+        return chat_summary(chat)
+
+    def process_chat(self, chat, provider_id=None):
+        return process_chat(chat, provider_id)
+
+    async def response_json(self, response):
+        return await response_json(response)
+
+    def get_headers(self, provider, chat):
+        headers = self.headers.copy()
+        if provider is not None:
+            headers["Authorization"] = f"Bearer {provider.api_key}"
+        elif self.api_key:
+            headers["Authorization"] = f"Bearer {self.api_key}"
+        return headers
+
+    def to_response(self, response, chat, started_at):
+        raise NotImplementedError
+
+    async def chat(self, chat, provider=None):
+        return {
+            "choices": [
+                {
+                    "message": {
+                        "role": "assistant",
+                        "content": "Not Implemented",
+                        "images": [
+                            {
+                                "type": "image_url",
+                                "image_url": {
+                                    "url": "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSIyNCIgaGVpZ2h0PSIyNCIgdmlld0JveD0iMCAwIDI0IDI0Ij48cGF0aCBmaWxsPSJjdXJyZW50Q29sb3IiIGQ9Ik0xMiAyMGE4IDggMCAxIDAgMC0xNmE4IDggMCAwIDAgMCAxNm0wIDJDNi40NzcgMjIgMiAxNy41MjMgMiAxMlM2LjQ3NyAyIDEyIDJzMTAgNC40NzcgMTAgMTBzLTQuNDc3IDEwLTEwIDEwbS0xLTZoMnYyaC0yem0wLTEwaDJ2OGgtMnoiLz48L3N2Zz4=",
+                                },
+                            }
+                        ],
+                    }
+                }
+            ]
+        }
+
+
+# OpenAI Providers
+
+
+class OpenAiCompatible:
+    sdk = "@ai-sdk/openai-compatible"
+
+    def __init__(self, **kwargs):
+        required_args = ["id", "api"]
+        for arg in required_args:
+            if arg not in kwargs:
+                raise ValueError(f"Missing required argument: {arg}")
+
+        self.id = kwargs.get("id")
+        self.api = kwargs.get("api").strip("/")
+        self.env = kwargs.get("env", [])
+        self.api_key = kwargs.get("api_key")
+        self.name = kwargs.get("name", id_to_name(self.id))
+        self.set_models(**kwargs)
+
+        self.chat_url = f"{self.api}/chat/completions"
 
         self.headers = kwargs.get("headers", {"Content-Type": "application/json"})
-        if api_key is not None:
-            self.headers["Authorization"] = f"Bearer {api_key}"
+        if self.api_key is not None:
+            self.headers["Authorization"] = f"Bearer {self.api_key}"
 
         self.frequency_penalty = float(kwargs["frequency_penalty"]) if "frequency_penalty" in kwargs else None
         self.max_completion_tokens = int(kwargs["max_completion_tokens"]) if "max_completion_tokens" in kwargs else None
@@ -486,44 +934,132 @@ class OpenAiProvider:
         self.verbosity = kwargs.get("verbosity")
         self.stream = bool(kwargs["stream"]) if "stream" in kwargs else None
         self.enable_thinking = bool(kwargs["enable_thinking"]) if "enable_thinking" in kwargs else None
-        self.pricing = kwargs.get("pricing")
-        self.default_pricing = kwargs.get("default_pricing")
         self.check = kwargs.get("check")
+        self.modalities = kwargs.get("modalities", {})
+
+    def set_models(self, **kwargs):
+        models = kwargs.get("models", {})
+        self.map_models = kwargs.get("map_models", {})
+        # if 'map_models' is provided, only include models in `map_models[model_id] = provider_model_id`
+        if self.map_models:
+            self.models = {}
+            for provider_model_id in self.map_models.values():
+                if provider_model_id in models:
+                    self.models[provider_model_id] = models[provider_model_id]
+        else:
+            self.models = models
+
+        include_models = kwargs.get("include_models")  # string regex pattern
+        # only include models that match the regex pattern
+        if include_models:
+            _log(f"Filtering {len(self.models)} models, only including models that match regex: {include_models}")
+            self.models = {k: v for k, v in self.models.items() if re.search(include_models, k)}
+
+        exclude_models = kwargs.get("exclude_models")  # string regex pattern
+        # exclude models that match the regex pattern
+        if exclude_models:
+            _log(f"Filtering {len(self.models)} models, excluding models that match regex: {exclude_models}")
+            self.models = {k: v for k, v in self.models.items() if not re.search(exclude_models, k)}
+
+    def validate(self, **kwargs):
+        if not self.api_key:
+            api_keys = ", ".join(self.env)
+            return f"Provider '{self.name}' requires API Key {api_keys}"
+        return None
 
-    @classmethod
-    def test(cls, base_url=None, api_key=None, models=None, **kwargs):
-        if models is None:
-            models = {}
-        return base_url and api_key and len(models) > 0
+    def test(self, **kwargs):
+        error_msg = self.validate(**kwargs)
+        if error_msg:
+            _log(error_msg)
+            return False
+        return True
 
     async def load(self):
-        pass
+        if not self.models:
+            await self.load_models()
 
-    def model_pricing(self, model):
+    def model_info(self, model):
         provider_model = self.provider_model(model) or model
-        if self.pricing and provider_model in self.pricing:
-            return self.pricing[provider_model]
-        return self.default_pricing or None
+        for model_id, model_info in self.models.items():
+            if model_id.lower() == provider_model.lower():
+                return model_info
+        return None
+
+    def model_cost(self, model):
+        model_info = self.model_info(model)
+        return model_info.get("cost") if model_info else None
 
     def provider_model(self, model):
-        if model in self.models:
-            return self.models[model]
+        # convert model to lowercase for case-insensitive comparison
+        model_lower = model.lower()
+
+        # if model is a map model id, return the provider model id
+        for model_id, provider_model in self.map_models.items():
+            if model_id.lower() == model_lower:
+                return provider_model
+
+        # if model is a provider model id, try again with just the model name
+        for provider_model in self.map_models.values():
+            if provider_model.lower() == model_lower:
+                return provider_model
+
+        # if model is a model id, try again with just the model id or name
+        for model_id, provider_model_info in self.models.items():
+            id = provider_model_info.get("id") or model_id
+            if model_id.lower() == model_lower or id.lower() == model_lower:
+                return id
+            name = provider_model_info.get("name")
+            if name and name.lower() == model_lower:
+                return id
+
+        # fallback to trying again with just the model short name
+        for model_id, provider_model_info in self.models.items():
+            id = provider_model_info.get("id") or model_id
+            if "/" in id:
+                model_name = id.split("/")[-1]
+                if model_name.lower() == model_lower:
+                    return id
+
+        # if model is a full provider model id, try again with just the model name
+        if "/" in model:
+            last_part = model.split("/")[-1]
+            return self.provider_model(last_part)
+
         return None
 
+    def response_json(self, response):
+        return response_json(response)
+
     def to_response(self, response, chat, started_at):
         if "metadata" not in response:
             response["metadata"] = {}
         response["metadata"]["duration"] = int((time.time() - started_at) * 1000)
         if chat is not None and "model" in chat:
-            pricing = self.model_pricing(chat["model"])
+            pricing = self.model_cost(chat["model"])
             if pricing and "input" in pricing and "output" in pricing:
                 response["metadata"]["pricing"] = f"{pricing['input']}/{pricing['output']}"
-        _log(json.dumps(response, indent=2))
         return response
 
+    def chat_summary(self, chat):
+        return chat_summary(chat)
+
+    def process_chat(self, chat, provider_id=None):
+        return process_chat(chat, provider_id)
+
     async def chat(self, chat):
         chat["model"] = self.provider_model(chat["model"]) or chat["model"]
 
+        if "modalities" in chat:
+            for modality in chat.get("modalities", []):
+                # use default implementation for text modalities
+                if modality == "text":
+                    continue
+                modality_provider = self.modalities.get(modality)
+                if modality_provider:
+                    return await modality_provider.chat(chat, self)
+                else:
+                    raise Exception(f"Provider {self.name} does not support '{modality}' modality")
+
         # with open(os.path.join(os.path.dirname(__file__), 'chat.wip.json'), "w") as f:
         #     f.write(json.dumps(chat, indent=2))
 
@@ -562,285 +1098,152 @@ class OpenAiProvider:
         if self.enable_thinking is not None:
             chat["enable_thinking"] = self.enable_thinking
 
-        chat = await process_chat(chat)
+        chat = await process_chat(chat, provider_id=self.id)
         _log(f"POST {self.chat_url}")
         _log(chat_summary(chat))
         # remove metadata if any (conflicts with some providers, e.g. Z.ai)
-        chat.pop("metadata", None)
+        metadata = chat.pop("metadata", None)
 
         async with aiohttp.ClientSession() as session:
             started_at = time.time()
             async with session.post(
                 self.chat_url, headers=self.headers, data=json.dumps(chat), timeout=aiohttp.ClientTimeout(total=120)
             ) as response:
+                chat["metadata"] = metadata
                 return self.to_response(await response_json(response), chat, started_at)
 
 
-class OllamaProvider(OpenAiProvider):
-    def __init__(self, base_url, models, all_models=False, **kwargs):
-        super().__init__(base_url=base_url, models=models, **kwargs)
-        self.all_models = all_models
+class MistralProvider(OpenAiCompatible):
+    sdk = "@ai-sdk/mistral"
+
+    def __init__(self, **kwargs):
+        if "api" not in kwargs:
+            kwargs["api"] = "https://api.mistral.ai/v1"
+        super().__init__(**kwargs)
+
+
+class GroqProvider(OpenAiCompatible):
+    sdk = "@ai-sdk/groq"
+
+    def __init__(self, **kwargs):
+        if "api" not in kwargs:
+            kwargs["api"] = "https://api.groq.com/openai/v1"
+        super().__init__(**kwargs)
+
+
+class XaiProvider(OpenAiCompatible):
+    sdk = "@ai-sdk/xai"
+
+    def __init__(self, **kwargs):
+        if "api" not in kwargs:
+            kwargs["api"] = "https://api.x.ai/v1"
+        super().__init__(**kwargs)
+
+
+class CodestralProvider(OpenAiCompatible):
+    sdk = "codestral"
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+
+class OllamaProvider(OpenAiCompatible):
+    sdk = "ollama"
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        # Ollama's OpenAI-compatible endpoint is at /v1/chat/completions
+        self.chat_url = f"{self.api}/v1/chat/completions"
 
     async def load(self):
-        if self.all_models:
-            await self.load_models(default_models=self.models)
+        if not self.models:
+            await self.load_models()
 
     async def get_models(self):
         ret = {}
         try:
             async with aiohttp.ClientSession() as session:
-                _log(f"GET {self.base_url}/api/tags")
+                _log(f"GET {self.api}/api/tags")
                 async with session.get(
-                    f"{self.base_url}/api/tags", headers=self.headers, timeout=aiohttp.ClientTimeout(total=120)
+                    f"{self.api}/api/tags", headers=self.headers, timeout=aiohttp.ClientTimeout(total=120)
                 ) as response:
                     data = await response_json(response)
                     for model in data.get("models", []):
-                        name = model["model"]
-                        if name.endswith(":latest"):
-                            name = name[:-7]
-                        ret[name] = name
+                        model_id = model["model"]
+                        if model_id.endswith(":latest"):
+                            model_id = model_id[:-7]
+                        ret[model_id] = model_id
                     _log(f"Loaded Ollama models: {ret}")
         except Exception as e:
             _log(f"Error getting Ollama models: {e}")
             # return empty dict if ollama is not available
         return ret
 
-    async def load_models(self, default_models):
+    async def load_models(self):
         """Load models if all_models was requested"""
-        if self.all_models:
-            self.models = await self.get_models()
-        if default_models:
-            self.models = {**default_models, **self.models}
-
-    @classmethod
-    def test(cls, base_url=None, models=None, all_models=False, **kwargs):
-        if models is None:
-            models = {}
-        return base_url and (len(models) > 0 or all_models)
-
-
-class GoogleOpenAiProvider(OpenAiProvider):
-    def __init__(self, api_key, models, **kwargs):
-        super().__init__(base_url="https://generativelanguage.googleapis.com", api_key=api_key, models=models, **kwargs)
-        self.chat_url = "https://generativelanguage.googleapis.com/v1beta/chat/completions"
-
-    @classmethod
-    def test(cls, api_key=None, models=None, **kwargs):
-        if models is None:
-            models = {}
-        return api_key and len(models) > 0
-
-
-class GoogleProvider(OpenAiProvider):
-    def __init__(self, models, api_key, safety_settings=None, thinking_config=None, curl=False, **kwargs):
-        super().__init__(base_url="https://generativelanguage.googleapis.com", api_key=api_key, models=models, **kwargs)
-        self.safety_settings = safety_settings
-        self.thinking_config = thinking_config
-        self.curl = curl
-        self.headers = kwargs.get("headers", {"Content-Type": "application/json"})
-        # Google fails when using Authorization header, use query string param instead
-        if "Authorization" in self.headers:
-            del self.headers["Authorization"]
-
-    @classmethod
-    def test(cls, api_key=None, models=None, **kwargs):
-        if models is None:
-            models = {}
-        return api_key is not None and len(models) > 0
 
-    async def chat(self, chat):
-        chat["model"] = self.provider_model(chat["model"]) or chat["model"]
+        # Map models to provider models {model_id:model_id}
+        model_map = await self.get_models()
+        if self.map_models:
+            map_model_values = set(self.map_models.values())
+            to = {}
+            for k, v in model_map.items():
+                if k in self.map_models:
+                    to[k] = v
+                if v in map_model_values:
+                    to[k] = v
+            model_map = to
+        else:
+            self.map_models = model_map
+        models = {}
+        for k, v in model_map.items():
+            models[k] = {
+                "id": k,
+                "name": v.replace(":", " "),
+                "modalities": {"input": ["text"], "output": ["text"]},
+                "cost": {
+                    "input": 0,
+                    "output": 0,
+                },
+            }
+        self.models = models
 
-        chat = await process_chat(chat)
-        generation_config = {}
+    def validate(self, **kwargs):
+        return None
 
-        # Filter out system messages and convert to proper Gemini format
-        contents = []
-        system_prompt = None
 
-        async with aiohttp.ClientSession() as session:
-            for message in chat["messages"]:
-                if message["role"] == "system":
-                    content = message["content"]
-                    if isinstance(content, list):
-                        for item in content:
-                            if "text" in item:
-                                system_prompt = item["text"]
-                                break
-                    elif isinstance(content, str):
-                        system_prompt = content
-                elif "content" in message:
-                    if isinstance(message["content"], list):
-                        parts = []
-                        for item in message["content"]:
-                            if "type" in item:
-                                if item["type"] == "image_url" and "image_url" in item:
-                                    image_url = item["image_url"]
-                                    if "url" not in image_url:
-                                        continue
-                                    url = image_url["url"]
-                                    if not url.startswith("data:"):
-                                        raise (Exception("Image was not downloaded: " + url))
-                                    # Extract mime type from data uri
-                                    mimetype = url.split(";", 1)[0].split(":", 1)[1] if ";" in url else "image/png"
-                                    base64_data = url.split(",", 1)[1]
-                                    parts.append({"inline_data": {"mime_type": mimetype, "data": base64_data}})
-                                elif item["type"] == "input_audio" and "input_audio" in item:
-                                    input_audio = item["input_audio"]
-                                    if "data" not in input_audio:
-                                        continue
-                                    data = input_audio["data"]
-                                    format = input_audio["format"]
-                                    mimetype = f"audio/{format}"
-                                    parts.append({"inline_data": {"mime_type": mimetype, "data": data}})
-                                elif item["type"] == "file" and "file" in item:
-                                    file = item["file"]
-                                    if "file_data" not in file:
-                                        continue
-                                    data = file["file_data"]
-                                    if not data.startswith("data:"):
-                                        raise (Exception("File was not downloaded: " + data))
-                                    # Extract mime type from data uri
-                                    mimetype = (
-                                        data.split(";", 1)[0].split(":", 1)[1]
-                                        if ";" in data
-                                        else "application/octet-stream"
-                                    )
-                                    base64_data = data.split(",", 1)[1]
-                                    parts.append({"inline_data": {"mime_type": mimetype, "data": base64_data}})
-                            if "text" in item:
-                                text = item["text"]
-                                parts.append({"text": text})
-                        if len(parts) > 0:
-                            contents.append(
-                                {
-                                    "role": message["role"]
-                                    if "role" in message and message["role"] == "user"
-                                    else "model",
-                                    "parts": parts,
-                                }
-                            )
-                    else:
-                        content = message["content"]
-                        contents.append(
-                            {
-                                "role": message["role"] if "role" in message and message["role"] == "user" else "model",
-                                "parts": [{"text": content}],
-                            }
-                        )
+class LMStudioProvider(OllamaProvider):
+    sdk = "lmstudio"
 
-            gemini_chat = {
-                "contents": contents,
-            }
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.chat_url = f"{self.api}/chat/completions"
 
-            if self.safety_settings:
-                gemini_chat["safetySettings"] = self.safety_settings
-
-            # Add system instruction if present
-            if system_prompt is not None:
-                gemini_chat["systemInstruction"] = {"parts": [{"text": system_prompt}]}
-
-            if "max_completion_tokens" in chat:
-                generation_config["maxOutputTokens"] = chat["max_completion_tokens"]
-            if "stop" in chat:
-                generation_config["stopSequences"] = [chat["stop"]]
-            if "temperature" in chat:
-                generation_config["temperature"] = chat["temperature"]
-            if "top_p" in chat:
-                generation_config["topP"] = chat["top_p"]
-            if "top_logprobs" in chat:
-                generation_config["topK"] = chat["top_logprobs"]
-
-            if "thinkingConfig" in chat:
-                generation_config["thinkingConfig"] = chat["thinkingConfig"]
-            elif self.thinking_config:
-                generation_config["thinkingConfig"] = self.thinking_config
-
-            if len(generation_config) > 0:
-                gemini_chat["generationConfig"] = generation_config
-
-            started_at = int(time.time() * 1000)
-            gemini_chat_url = f"https://generativelanguage.googleapis.com/v1beta/models/{chat['model']}:generateContent?key={self.api_key}"
-
-            _log(f"POST {gemini_chat_url}")
-            _log(gemini_chat_summary(gemini_chat))
-            started_at = time.time()
+    async def get_models(self):
+        ret = {}
+        try:
+            async with aiohttp.ClientSession() as session:
+                _log(f"GET {self.api}/models")
+                async with session.get(
+                    f"{self.api}/models", headers=self.headers, timeout=aiohttp.ClientTimeout(total=120)
+                ) as response:
+                    data = await response_json(response)
+                    for model in data.get("data", []):
+                        id = model["id"]
+                        ret[id] = id
+                    _log(f"Loaded LMStudio models: {ret}")
+        except Exception as e:
+            _log(f"Error getting LMStudio models: {e}")
+            # return empty dict if ollama is not available
+        return ret
 
-            if self.curl:
-                curl_args = [
-                    "curl",
-                    "-X",
-                    "POST",
-                    "-H",
-                    "Content-Type: application/json",
-                    "-d",
-                    json.dumps(gemini_chat),
-                    gemini_chat_url,
-                ]
-                try:
-                    o = subprocess.run(curl_args, check=True, capture_output=True, text=True, timeout=120)
-                    obj = json.loads(o.stdout)
-                except Exception as e:
-                    raise Exception(f"Error executing curl: {e}") from e
-            else:
-                async with session.post(
-                    gemini_chat_url,
-                    headers=self.headers,
-                    data=json.dumps(gemini_chat),
-                    timeout=aiohttp.ClientTimeout(total=120),
-                ) as res:
-                    obj = await response_json(res)
-                    _log(f"google response:\n{json.dumps(obj, indent=2)}")
-
-            response = {
-                "id": f"chatcmpl-{started_at}",
-                "created": started_at,
-                "model": obj.get("modelVersion", chat["model"]),
-            }
-            choices = []
-            if "error" in obj:
-                _log(f"Error: {obj['error']}")
-                raise Exception(obj["error"]["message"])
-            for i, candidate in enumerate(obj["candidates"]):
-                role = "assistant"
-                if "content" in candidate and "role" in candidate["content"]:
-                    role = "assistant" if candidate["content"]["role"] == "model" else candidate["content"]["role"]
-
-                # Safely extract content from all text parts
-                content = ""
-                reasoning = ""
-                if "content" in candidate and "parts" in candidate["content"]:
-                    text_parts = []
-                    reasoning_parts = []
-                    for part in candidate["content"]["parts"]:
-                        if "text" in part:
-                            if "thought" in part and part["thought"]:
-                                reasoning_parts.append(part["text"])
-                            else:
-                                text_parts.append(part["text"])
-                    content = " ".join(text_parts)
-                    reasoning = " ".join(reasoning_parts)
 
-                choice = {
-                    "index": i,
-                    "finish_reason": candidate.get("finishReason", "stop"),
-                    "message": {
-                        "role": role,
-                        "content": content,
-                    },
-                }
-                if reasoning:
-                    choice["message"]["reasoning"] = reasoning
-                choices.append(choice)
-            response["choices"] = choices
-            if "usageMetadata" in obj:
-                usage = obj["usageMetadata"]
-                response["usage"] = {
-                    "completion_tokens": usage["candidatesTokenCount"],
-                    "total_tokens": usage["totalTokenCount"],
-                    "prompt_tokens": usage["promptTokenCount"],
-                }
-            return self.to_response(response, chat, started_at)
+def get_provider_model(model_name):
+    for provider in g_handlers.values():
+        provider_model = provider.provider_model(model_name)
+        if provider_model:
+            return provider_model
+    return None
 
 
 def get_models():
@@ -856,42 +1259,259 @@ def get_models():
 def get_active_models():
     ret = []
     existing_models = set()
-    for id, provider in g_handlers.items():
-        for model in provider.models:
-            if model not in existing_models:
-                existing_models.add(model)
-                provider_model = provider.models[model]
-                pricing = provider.model_pricing(model)
-                ret.append({"id": model, "provider": id, "provider_model": provider_model, "pricing": pricing})
+    for provider_id, provider in g_handlers.items():
+        for model in provider.models.values():
+            name = model.get("name")
+            if not name:
+                _log(f"Provider {provider_id} model {model} has no name")
+                continue
+            if name not in existing_models:
+                existing_models.add(name)
+                item = model.copy()
+                item.update({"provider": provider_id})
+                ret.append(item)
     ret.sort(key=lambda x: x["id"])
     return ret
 
 
-async def chat_completion(chat):
-    model = chat["model"]
-    # get first provider that has the model
-    candidate_providers = [name for name, provider in g_handlers.items() if model in provider.models]
-    if len(candidate_providers) == 0:
-        raise (Exception(f"Model {model} not found"))
+def api_providers():
+    ret = []
+    for id, provider in g_handlers.items():
+        ret.append({"id": id, "name": provider.name, "models": provider.models})
+    return ret
+
+
+def to_error_message(e):
+    return str(e)
+
+
+def to_error_response(e, stacktrace=False):
+    status = {"errorCode": "Error", "message": to_error_message(e)}
+    if stacktrace:
+        status["stackTrace"] = traceback.format_exc()
+    return {"responseStatus": status}
+
+
+def create_error_response(message, error_code="Error", stack_trace=None):
+    ret = {"responseStatus": {"errorCode": error_code, "message": message}}
+    if stack_trace:
+        ret["responseStatus"]["stackTrace"] = stack_trace
+    return ret
+
+
+def should_cancel_thread(context):
+    ret = context.get("cancelled", False)
+    if ret:
+        thread_id = context.get("threadId")
+        _dbg(f"Thread cancelled {thread_id}")
+    return ret
+
+
+def g_chat_request(template=None, text=None, model=None, system_prompt=None):
+    chat_template = g_config["defaults"].get(template or "text")
+    if not chat_template:
+        raise Exception(f"Chat template '{template}' not found")
+
+    chat = chat_template.copy()
+    if model:
+        chat["model"] = model
+    if system_prompt is not None:
+        chat["messages"].insert(0, {"role": "system", "content": system_prompt})
+    if text is not None:
+        if not chat["messages"] or len(chat["messages"]) == 0:
+            chat["messages"] = [{"role": "user", "content": [{"type": "text", "text": ""}]}]
+
+        # replace content of last message if exists, else add
+        last_msg = chat["messages"][-1] if "messages" in chat else None
+        if last_msg and last_msg["role"] == "user":
+            if isinstance(last_msg["content"], list):
+                last_msg["content"][-1]["text"] = text
+            else:
+                last_msg["content"] = text
+        else:
+            chat["messages"].append({"role": "user", "content": text})
+
+    return chat
+
+
+async def g_chat_completion(chat, context=None):
+    try:
+        model = chat.get("model")
+        if not model:
+            raise Exception("Model not specified")
 
+        if context is None:
+            context = {"chat": chat, "tools": "all"}
+
+        # get first provider that has the model
+        candidate_providers = [name for name, provider in g_handlers.items() if provider.provider_model(model)]
+        if len(candidate_providers) == 0:
+            raise (Exception(f"Model {model} not found"))
+    except Exception as e:
+        await g_app.on_chat_error(e, context or {"chat": chat})
+        raise e
+
+    started_at = time.time()
     first_exception = None
+    provider_name = "Unknown"
     for name in candidate_providers:
-        provider = g_handlers[name]
-        _log(f"provider: {name} {type(provider).__name__}")
         try:
-            response = await provider.chat(chat.copy())
-            return response
+            provider_name = name
+            provider = g_handlers[name]
+            _log(f"provider: {name} {type(provider).__name__}")
+            started_at = time.time()
+            context["startedAt"] = datetime.now()
+            context["provider"] = name
+            model_info = provider.model_info(model)
+            context["modelCost"] = model_info.get("cost", provider.model_cost(model)) or {"input": 0, "output": 0}
+            context["modelInfo"] = model_info
+
+            # Accumulate usage across tool calls
+            total_usage = {
+                "prompt_tokens": 0,
+                "completion_tokens": 0,
+                "total_tokens": 0,
+            }
+            accumulated_cost = 0.0
+
+            # Inject global tools if present
+            current_chat = chat.copy()
+            if g_app.tool_definitions:
+                only_tools_str = context.get("tools", "all")
+                include_all_tools = only_tools_str == "all"
+                only_tools = only_tools_str.split(",")
+
+                if include_all_tools or len(only_tools) > 0:
+                    if "tools" not in current_chat:
+                        current_chat["tools"] = []
+
+                    existing_tools = {t["function"]["name"] for t in current_chat["tools"]}
+                    for tool_def in g_app.tool_definitions:
+                        name = tool_def["function"]["name"]
+                        if name not in existing_tools and (include_all_tools or name in only_tools):
+                            current_chat["tools"].append(tool_def)
+
+            # Apply pre-chat filters ONCE
+            context["chat"] = current_chat
+            for filter_func in g_app.chat_request_filters:
+                await filter_func(current_chat, context)
+
+            # Tool execution loop
+            max_iterations = 10
+            tool_history = []
+            final_response = None
+
+            for _ in range(max_iterations):
+                if should_cancel_thread(context):
+                    return
+
+                response = await provider.chat(current_chat)
+
+                if should_cancel_thread(context):
+                    return
+
+                # Aggregate usage
+                if "usage" in response:
+                    usage = response["usage"]
+                    total_usage["prompt_tokens"] += usage.get("prompt_tokens", 0)
+                    total_usage["completion_tokens"] += usage.get("completion_tokens", 0)
+                    total_usage["total_tokens"] += usage.get("total_tokens", 0)
+
+                    # Calculate cost for this step if available
+                    if "cost" in response and isinstance(response["cost"], (int, float)):
+                        accumulated_cost += response["cost"]
+                    elif "cost" in usage and isinstance(usage["cost"], (int, float)):
+                        accumulated_cost += usage["cost"]
+
+                # Check for tool_calls in the response
+                choice = response.get("choices", [])[0] if response.get("choices") else {}
+                message = choice.get("message", {})
+                tool_calls = message.get("tool_calls")
+
+                if tool_calls:
+                    # Append the assistant's message with tool calls to history
+                    if "messages" not in current_chat:
+                        current_chat["messages"] = []
+                    if "timestamp" not in message:
+                        message["timestamp"] = int(time.time() * 1000)
+                    current_chat["messages"].append(message)
+                    tool_history.append(message)
+
+                    await g_app.on_chat_tool(current_chat, context)
+
+                    for tool_call in tool_calls:
+                        function_name = tool_call["function"]["name"]
+                        try:
+                            function_args = json.loads(tool_call["function"]["arguments"])
+                        except Exception as e:
+                            tool_result = f"Error parsing JSON arguments for tool {function_name}: {e}"
+                        else:
+                            tool_result = f"Error: Tool {function_name} not found"
+                            if function_name in g_app.tools:
+                                try:
+                                    func = g_app.tools[function_name]
+                                    if inspect.iscoroutinefunction(func):
+                                        tool_result = await func(**function_args)
+                                    else:
+                                        tool_result = func(**function_args)
+                                except Exception as e:
+                                    tool_result = f"Error executing tool {function_name}: {e}"
+
+                        # Append tool result to history
+                        tool_msg = {"role": "tool", "tool_call_id": tool_call["id"], "content": to_content(tool_result)}
+                        current_chat["messages"].append(tool_msg)
+                        tool_history.append(tool_msg)
+
+                    await g_app.on_chat_tool(current_chat, context)
+
+                    if should_cancel_thread(context):
+                        return
+
+                    # Continue loop to send tool results back to LLM
+                    continue
+
+                # If no tool calls, this is the final response
+                if tool_history:
+                    response["tool_history"] = tool_history
+
+                # Update final response with aggregated usage
+                if "usage" not in response:
+                    response["usage"] = {}
+                # convert to int seconds
+                context["duration"] = duration = int(time.time() - started_at)
+                total_usage.update({"duration": duration})
+                response["usage"].update(total_usage)
+                # If we accumulated cost, set it on the response
+                if accumulated_cost > 0:
+                    response["cost"] = accumulated_cost
+
+                final_response = response
+                break  # Exit tool loop
+
+            if final_response:
+                # Apply post-chat filters ONCE on final response
+                for filter_func in g_app.chat_response_filters:
+                    await filter_func(final_response, context)
+
+                if DEBUG:
+                    _dbg(json.dumps(final_response, indent=2))
+
+                return final_response
+
         except Exception as e:
             if first_exception is None:
                 first_exception = e
-            _log(f"Provider {name} failed: {e}")
+                context["stackTrace"] = traceback.format_exc()
+            _err(f"Provider {provider_name} failed", first_exception)
+            await g_app.on_chat_error(e, context)
+
             continue
 
     # If we get here, all providers failed
     raise first_exception
 
 
-async def cli_chat(chat, image=None, audio=None, file=None, args=None, raw=False):
+async def cli_chat(chat, tools=None, image=None, audio=None, file=None, args=None, raw=False):
     if g_default_model:
         chat["model"] = g_default_model
 
@@ -966,73 +1586,161 @@ async def cli_chat(chat, image=None, audio=None, file=None, args=None, raw=False
         printdump(chat)
 
     try:
-        response = await chat_completion(chat)
+        context = {
+            "tools": tools or "all",
+        }
+        response = await g_app.chat_completion(chat, context=context)
+
         if raw:
             print(json.dumps(response, indent=2))
             exit(0)
         else:
-            answer = response["choices"][0]["message"]["content"]
-            print(answer)
+            msg = response["choices"][0]["message"]
+            if "content" in msg or "answer" in msg:
+                print(msg["content"])
+
+            generated_files = []
+            for choice in response["choices"]:
+                if "message" in choice:
+                    msg = choice["message"]
+                    if "images" in msg:
+                        for image in msg["images"]:
+                            image_url = image["image_url"]["url"]
+                            generated_files.append(image_url)
+                    if "audios" in msg:
+                        for audio in msg["audios"]:
+                            audio_url = audio["audio_url"]["url"]
+                            generated_files.append(audio_url)
+
+            if len(generated_files) > 0:
+                print("\nSaved files:")
+                for file in generated_files:
+                    if file.startswith("/~cache"):
+                        print(get_cache_path(file[8:]))
+                        print(urljoin("http://localhost:8000", file))
+                    else:
+                        print(file)
+
     except HTTPError as e:
         # HTTP error (4xx, 5xx)
         print(f"{e}:\n{e.body}")
-        exit(1)
+        g_app.exit(1)
     except aiohttp.ClientConnectionError as e:
         # Connection issues
         print(f"Connection error: {e}")
-        exit(1)
+        g_app.exit(1)
     except asyncio.TimeoutError as e:
         # Timeout
         print(f"Timeout error: {e}")
-        exit(1)
+        g_app.exit(1)
 
 
 def config_str(key):
     return key in g_config and g_config[key] or None
 
 
-def init_llms(config):
+def load_config(config, providers, verbose=None):
+    global g_config, g_providers, g_verbose
+    g_config = config
+    g_providers = providers
+    if verbose:
+        g_verbose = verbose
+
+
+def init_llms(config, providers):
     global g_config, g_handlers
 
-    g_config = config
+    load_config(config, providers)
     g_handlers = {}
     # iterate over config and replace $ENV with env value
     for key, value in g_config.items():
         if isinstance(value, str) and value.startswith("$"):
-            g_config[key] = os.environ.get(value[1:], "")
+            g_config[key] = os.getenv(value[1:], "")
 
     # if g_verbose:
     #     printdump(g_config)
     providers = g_config["providers"]
 
-    for name, orig in providers.items():
-        definition = orig.copy()
-        provider_type = definition["type"]
-        if "enabled" in definition and not definition["enabled"]:
+    for id, orig in providers.items():
+        if "enabled" in orig and not orig["enabled"]:
             continue
 
-        # Replace API keys with environment variables if they start with $
-        if "api_key" in definition:
-            value = definition["api_key"]
-            if isinstance(value, str) and value.startswith("$"):
-                definition["api_key"] = os.environ.get(value[1:], "")
-
-        # Create a copy of definition without the 'type' key for constructor kwargs
-        constructor_kwargs = {k: v for k, v in definition.items() if k != "type" and k != "enabled"}
-        constructor_kwargs["headers"] = g_config["defaults"]["headers"].copy()
-
-        if provider_type == "OpenAiProvider" and OpenAiProvider.test(**constructor_kwargs):
-            g_handlers[name] = OpenAiProvider(**constructor_kwargs)
-        elif provider_type == "OllamaProvider" and OllamaProvider.test(**constructor_kwargs):
-            g_handlers[name] = OllamaProvider(**constructor_kwargs)
-        elif provider_type == "GoogleProvider" and GoogleProvider.test(**constructor_kwargs):
-            g_handlers[name] = GoogleProvider(**constructor_kwargs)
-        elif provider_type == "GoogleOpenAiProvider" and GoogleOpenAiProvider.test(**constructor_kwargs):
-            g_handlers[name] = GoogleOpenAiProvider(**constructor_kwargs)
-
+        provider, constructor_kwargs = create_provider_from_definition(id, orig)
+        if provider and provider.test(**constructor_kwargs):
+            g_handlers[id] = provider
     return g_handlers
 
 
+def create_provider_from_definition(id, orig):
+    definition = orig.copy()
+    provider_id = definition.get("id", id)
+    if "id" not in definition:
+        definition["id"] = provider_id
+    provider = g_providers.get(provider_id)
+    constructor_kwargs = create_provider_kwargs(definition, provider)
+    provider = create_provider(constructor_kwargs)
+    return provider, constructor_kwargs
+
+
+def create_provider_kwargs(definition, provider=None):
+    if provider:
+        provider = provider.copy()
+        provider.update(definition)
+    else:
+        provider = definition.copy()
+
+    # Replace API keys with environment variables if they start with $
+    if "api_key" in provider:
+        value = provider["api_key"]
+        if isinstance(value, str) and value.startswith("$"):
+            provider["api_key"] = os.getenv(value[1:], "")
+
+    if "api_key" not in provider and "env" in provider:
+        for env_var in provider["env"]:
+            val = os.getenv(env_var)
+            if val:
+                provider["api_key"] = val
+                break
+
+    # Create a copy of provider
+    constructor_kwargs = dict(provider.items())
+    # Create a copy of all list and dict values
+    for key, value in constructor_kwargs.items():
+        if isinstance(value, (list, dict)):
+            constructor_kwargs[key] = value.copy()
+    constructor_kwargs["headers"] = g_config["defaults"]["headers"].copy()
+
+    if "modalities" in definition:
+        constructor_kwargs["modalities"] = {}
+        for modality, modality_definition in definition["modalities"].items():
+            modality_provider = create_provider(modality_definition)
+            if not modality_provider:
+                return None
+            constructor_kwargs["modalities"][modality] = modality_provider
+
+    return constructor_kwargs
+
+
+def create_provider(provider):
+    if not isinstance(provider, dict):
+        return None
+    provider_label = provider.get("id", provider.get("name", "unknown"))
+    npm_sdk = provider.get("npm")
+    if not npm_sdk:
+        _log(f"Provider {provider_label} is missing 'npm' sdk")
+        return None
+
+    for provider_type in g_app.all_providers:
+        if provider_type.sdk == npm_sdk:
+            kwargs = create_provider_kwargs(provider)
+            if kwargs is None:
+                kwargs = provider
+            return provider_type(**kwargs)
+
+    _log(f"Could not find provider {provider_label} with npm sdk {npm_sdk}")
+    return None
+
+
 async def load_llms():
     global g_handlers
     _log("Loading providers...")
@@ -1076,6 +1784,35 @@ async def save_default_config(config_path):
     g_config = json.loads(config_json)
 
 
+async def update_providers(home_providers_path):
+    global g_providers
+    text = await get_text("https://models.dev/api.json")
+    all_providers = json.loads(text)
+    extra_providers = {}
+    extra_providers_path = home_providers_path.replace("providers.json", "providers-extra.json")
+    if os.path.exists(extra_providers_path):
+        with open(extra_providers_path) as f:
+            extra_providers = json.load(f)
+
+    filtered_providers = {}
+    for id, provider in all_providers.items():
+        if id in g_config["providers"]:
+            filtered_providers[id] = provider
+            if id in extra_providers and "models" in extra_providers[id]:
+                for model_id, model in extra_providers[id]["models"].items():
+                    if "id" not in model:
+                        model["id"] = model_id
+                    if "name" not in model:
+                        model["name"] = id_to_name(model["id"])
+                    filtered_providers[id]["models"][model_id] = model
+
+    os.makedirs(os.path.dirname(home_providers_path), exist_ok=True)
+    with open(home_providers_path, "w", encoding="utf-8") as f:
+        json.dump(filtered_providers, f)
+
+    g_providers = filtered_providers
+
+
 def provider_status():
     enabled = list(g_handlers.keys())
     disabled = [provider for provider in g_config["providers"] if provider not in enabled]
@@ -1097,7 +1834,11 @@ def print_status():
 
 
 def home_llms_path(filename):
-    return f"{os.environ.get('HOME')}/.llms/{filename}"
+    return f"{os.getenv('HOME')}/.llms/{filename}"
+
+
+def get_cache_path(path=""):
+    return home_llms_path(f"cache/{path}") if path else home_llms_path("cache")
 
 
 def get_config_path():
@@ -1106,8 +1847,8 @@ def get_config_path():
         "./llms.json",
         home_config_path,
     ]
-    if os.environ.get("LLMS_CONFIG_PATH"):
-        check_paths.insert(0, os.environ.get("LLMS_CONFIG_PATH"))
+    if os.getenv("LLMS_CONFIG_PATH"):
+        check_paths.insert(0, os.getenv("LLMS_CONFIG_PATH"))
 
     for check_path in check_paths:
         g_config_path = os.path.normpath(os.path.join(os.path.dirname(__file__), check_path))
@@ -1116,28 +1857,20 @@ def get_config_path():
     return None
 
 
-def get_ui_path():
-    ui_paths = [home_llms_path("ui.json"), "ui.json"]
-    for ui_path in ui_paths:
-        if os.path.exists(ui_path):
-            return ui_path
-    return None
-
-
 def enable_provider(provider):
     msg = None
     provider_config = g_config["providers"][provider]
+    if not provider_config:
+        return None, f"Provider {provider} not found"
+
+    provider, constructor_kwargs = create_provider_from_definition(provider, provider_config)
+    msg = provider.validate(**constructor_kwargs)
+    if msg:
+        return None, msg
+
     provider_config["enabled"] = True
-    if "api_key" in provider_config:
-        api_key = provider_config["api_key"]
-        if isinstance(api_key, str):
-            if api_key.startswith("$"):
-                if not os.environ.get(api_key[1:], ""):
-                    msg = f"WARNING: {provider} requires missing API Key in Environment Variable {api_key}"
-            else:
-                msg = f"WARNING: {provider} is not configured with an API Key"
     save_config(g_config)
-    init_llms(g_config)
+    init_llms(g_config, g_providers)
     return provider_config, msg
 
 
@@ -1145,7 +1878,7 @@ def disable_provider(provider):
     provider_config = g_config["providers"][provider]
     provider_config["enabled"] = False
     save_config(g_config)
-    init_llms(g_config)
+    init_llms(g_config, g_providers)
 
 
 def resolve_root():
@@ -1340,7 +2073,8 @@ async def check_models(provider_name, model_names=None):
     else:
         # Check only specified models
         for model_name in model_names:
-            if model_name in provider.models:
+            provider_model = provider.provider_model(model_name)
+            if provider_model:
                 models_to_check.append(model_name)
             else:
                 print(f"Model '{model_name}' not found in provider '{provider_name}'")
@@ -1355,69 +2089,76 @@ async def check_models(provider_name, model_names=None):
 
     # Test each model
     for model in models_to_check:
-        # Create a simple ping chat request
-        chat = (provider.check or g_config["defaults"]["check"]).copy()
-        chat["model"] = model
+        await check_provider_model(provider, model)
 
-        started_at = time.time()
-        try:
-            # Try to get a response from the model
-            response = await provider.chat(chat)
-            duration_ms = int((time.time() - started_at) * 1000)
+    print()
 
-            # Check if we got a valid response
-            if response and "choices" in response and len(response["choices"]) > 0:
-                print(f"  ✓ {model:<40} ({duration_ms}ms)")
-            else:
-                print(f"  ✗ {model:<40} Invalid response format")
-        except HTTPError as e:
-            duration_ms = int((time.time() - started_at) * 1000)
-            error_msg = f"HTTP {e.status}"
-            try:
-                # Try to parse error body for more details
-                error_body = json.loads(e.body) if e.body else {}
-                if "error" in error_body:
-                    error = error_body["error"]
-                    if isinstance(error, dict):
-                        if "message" in error and isinstance(error["message"], str):
-                            # OpenRouter
-                            error_msg = error["message"]
-                            if "code" in error:
-                                error_msg = f"{error['code']} {error_msg}"
-                            if "metadata" in error and "raw" in error["metadata"]:
-                                error_msg += f" - {error['metadata']['raw']}"
-                            if "provider" in error:
-                                error_msg += f" ({error['provider']})"
-                    elif isinstance(error, str):
-                        error_msg = error
-                elif "message" in error_body:
-                    if isinstance(error_body["message"], str):
-                        error_msg = error_body["message"]
-                    elif (
-                        isinstance(error_body["message"], dict)
-                        and "detail" in error_body["message"]
-                        and isinstance(error_body["message"]["detail"], list)
-                    ):
-                        # codestral error format
-                        error_msg = error_body["message"]["detail"][0]["msg"]
-                        if (
-                            "loc" in error_body["message"]["detail"][0]
-                            and len(error_body["message"]["detail"][0]["loc"]) > 0
-                        ):
-                            error_msg += f" (in {' '.join(error_body['message']['detail'][0]['loc'])})"
-            except Exception as parse_error:
-                _log(f"Error parsing error body: {parse_error}")
-                error_msg = e.body[:100] if e.body else f"HTTP {e.status}"
-            print(f"  ✗ {model:<40} {error_msg}")
-        except asyncio.TimeoutError:
-            duration_ms = int((time.time() - started_at) * 1000)
-            print(f"  ✗ {model:<40} Timeout after {duration_ms}ms")
-        except Exception as e:
-            duration_ms = int((time.time() - started_at) * 1000)
-            error_msg = str(e)[:100]
-            print(f"  ✗ {model:<40} {error_msg}")
 
-    print()
+async def check_provider_model(provider, model):
+    # Create a simple ping chat request
+    chat = (provider.check or g_config["defaults"]["check"]).copy()
+    chat["model"] = model
+
+    success = False
+    started_at = time.time()
+    try:
+        # Try to get a response from the model
+        response = await provider.chat(chat)
+        duration_ms = int((time.time() - started_at) * 1000)
+
+        # Check if we got a valid response
+        if response and "choices" in response and len(response["choices"]) > 0:
+            success = True
+            print(f"  ✓ {model:<40} ({duration_ms}ms)")
+        else:
+            print(f"  ✗ {model:<40} Invalid response format")
+    except HTTPError as e:
+        duration_ms = int((time.time() - started_at) * 1000)
+        error_msg = f"HTTP {e.status}"
+        try:
+            # Try to parse error body for more details
+            error_body = json.loads(e.body) if e.body else {}
+            if "error" in error_body:
+                error = error_body["error"]
+                if isinstance(error, dict):
+                    if "message" in error and isinstance(error["message"], str):
+                        # OpenRouter
+                        error_msg = error["message"]
+                        if "code" in error:
+                            error_msg = f"{error['code']} {error_msg}"
+                        if "metadata" in error and "raw" in error["metadata"]:
+                            error_msg += f" - {error['metadata']['raw']}"
+                        if "provider" in error:
+                            error_msg += f" ({error['provider']})"
+                elif isinstance(error, str):
+                    error_msg = error
+            elif "message" in error_body:
+                if isinstance(error_body["message"], str):
+                    error_msg = error_body["message"]
+                elif (
+                    isinstance(error_body["message"], dict)
+                    and "detail" in error_body["message"]
+                    and isinstance(error_body["message"]["detail"], list)
+                ):
+                    # codestral error format
+                    error_msg = error_body["message"]["detail"][0]["msg"]
+                    if (
+                        "loc" in error_body["message"]["detail"][0]
+                        and len(error_body["message"]["detail"][0]["loc"]) > 0
+                    ):
+                        error_msg += f" (in {' '.join(error_body['message']['detail'][0]['loc'])})"
+        except Exception as parse_error:
+            _log(f"Error parsing error body: {parse_error}")
+            error_msg = e.body[:100] if e.body else f"HTTP {e.status}"
+        print(f"  ✗ {model:<40} {error_msg}")
+    except asyncio.TimeoutError:
+        duration_ms = int((time.time() - started_at) * 1000)
+        print(f"  ✗ {model:<40} Timeout after {duration_ms}ms")
+    except Exception as e:
+        duration_ms = int((time.time() - started_at) * 1000)
+        error_msg = str(e)[:100]
+        print(f"  ✗ {model:<40} {error_msg}")
+    return success
 
 
 def text_from_resource(filename):
@@ -1452,8 +2193,14 @@ async def text_from_resource_or_url(filename):
 
 async def save_home_configs():
     home_config_path = home_llms_path("llms.json")
-    home_ui_path = home_llms_path("ui.json")
-    if os.path.exists(home_config_path) and os.path.exists(home_ui_path):
+    home_providers_path = home_llms_path("providers.json")
+    home_providers_extra_path = home_llms_path("providers-extra.json")
+
+    if (
+        os.path.exists(home_config_path)
+        and os.path.exists(home_providers_path)
+        and os.path.exists(home_providers_extra_path)
+    ):
         return
 
     llms_home = os.path.dirname(home_config_path)
@@ -1465,92 +2212,650 @@ async def save_home_configs():
                 f.write(config_json)
             _log(f"Created default config at {home_config_path}")
 
-        if not os.path.exists(home_ui_path):
-            ui_json = await text_from_resource_or_url("ui.json")
-            with open(home_ui_path, "w", encoding="utf-8") as f:
-                f.write(ui_json)
-            _log(f"Created default ui config at {home_ui_path}")
+        if not os.path.exists(home_providers_path):
+            providers_json = await text_from_resource_or_url("providers.json")
+            with open(home_providers_path, "w", encoding="utf-8") as f:
+                f.write(providers_json)
+            _log(f"Created default providers config at {home_providers_path}")
+
+        if not os.path.exists(home_providers_extra_path):
+            extra_json = await text_from_resource_or_url("providers-extra.json")
+            with open(home_providers_extra_path, "w", encoding="utf-8") as f:
+                f.write(extra_json)
+            _log(f"Created default extra providers config at {home_providers_extra_path}")
     except Exception:
         print("Could not create llms.json. Create one with --init or use --config <path>")
         exit(1)
 
 
-async def reload_providers():
-    global g_config, g_handlers
-    g_handlers = init_llms(g_config)
-    await load_llms()
+def load_config_json(config_json):
+    if config_json is None:
+        return None
+    config = json.loads(config_json)
+    if not config or "version" not in config or config["version"] < 3:
+        preserve_keys = ["auth", "defaults", "limits", "convert"]
+        new_config = json.loads(text_from_resource("llms.json"))
+        if config:
+            for key in preserve_keys:
+                if key in config:
+                    new_config[key] = config[key]
+        config = new_config
+        # move old config to YYYY-MM-DD.bak
+        new_path = f"{g_config_path}.{datetime.now().strftime('%Y-%m-%d')}.bak"
+        if os.path.exists(new_path):
+            os.remove(new_path)
+        os.rename(g_config_path, new_path)
+        print(f"llms.json migrated. old config moved to {new_path}")
+        # save new config
+        save_config(g_config)
+    return config
+
+
+async def reload_providers():
+    global g_config, g_handlers
+    g_handlers = init_llms(g_config, g_providers)
+    await load_llms()
     _log(f"{len(g_handlers)} providers loaded")
     return g_handlers
 
 
-async def watch_config_files(config_path, ui_path, interval=1):
+async def watch_config_files(config_path, providers_path, interval=1):
     """Watch config files and reload providers when they change"""
     global g_config
 
     config_path = Path(config_path)
-    ui_path = Path(ui_path) if ui_path else None
+    providers_path = Path(providers_path)
+
+    _log(f"Watching config file: {config_path}")
+    _log(f"Watching providers file: {providers_path}")
 
-    file_mtimes = {}
+    def get_latest_mtime():
+        ret = 0
+        name = "llms.json"
+        if config_path.is_file():
+            ret = config_path.stat().st_mtime
+            name = config_path.name
+        if providers_path.is_file() and providers_path.stat().st_mtime > ret:
+            ret = providers_path.stat().st_mtime
+            name = providers_path.name
+        return ret, name
 
-    _log(f"Watching config files: {config_path}" + (f", {ui_path}" if ui_path else ""))
+    latest_mtime, name = get_latest_mtime()
 
     while True:
         await asyncio.sleep(interval)
 
         # Check llms.json
         try:
-            if config_path.is_file():
-                mtime = config_path.stat().st_mtime
+            new_mtime, name = get_latest_mtime()
+            if new_mtime > latest_mtime:
+                _log(f"Config file changed: {name}")
+                latest_mtime = new_mtime
 
-                if str(config_path) not in file_mtimes:
-                    file_mtimes[str(config_path)] = mtime
-                elif file_mtimes[str(config_path)] != mtime:
-                    _log(f"Config file changed: {config_path.name}")
-                    file_mtimes[str(config_path)] = mtime
-
-                    try:
-                        # Reload llms.json
-                        with open(config_path) as f:
-                            g_config = json.load(f)
+                try:
+                    # Reload llms.json
+                    with open(config_path) as f:
+                        g_config = json.load(f)
 
-                        # Reload providers
-                        await reload_providers()
-                        _log("Providers reloaded successfully")
-                    except Exception as e:
-                        _log(f"Error reloading config: {e}")
+                    # Reload providers
+                    await reload_providers()
+                    _log("Providers reloaded successfully")
+                except Exception as e:
+                    _log(f"Error reloading config: {e}")
         except FileNotFoundError:
             pass
 
-        # Check ui.json
-        if ui_path:
+
+def get_session_token(request):
+    return request.query.get("session") or request.headers.get("X-Session-Token") or request.cookies.get("llms-token")
+
+
+class AppExtensions:
+    """
+    APIs extensions can use to extend the app
+    """
+
+    def __init__(self, cli_args, extra_args):
+        self.cli_args = cli_args
+        self.extra_args = extra_args
+        self.config = None
+        self.error_auth_required = create_error_response("Authentication required", "Unauthorized")
+        self.ui_extensions = []
+        self.chat_request_filters = []
+        self.chat_tool_filters = []
+        self.chat_response_filters = []
+        self.chat_error_filters = []
+        self.server_add_get = []
+        self.server_add_post = []
+        self.server_add_put = []
+        self.server_add_delete = []
+        self.server_add_patch = []
+        self.cache_saved_filters = []
+        self.shutdown_handlers = []
+        self.tools = {}
+        self.tool_definitions = []
+        self.index_headers = []
+        self.index_footers = []
+        self.request_args = {
+            "image_config": dict,  # e.g. { "aspect_ratio": "1:1" }
+            "temperature": float,  # e.g: 0.7
+            "max_completion_tokens": int,  # e.g: 2048
+            "seed": int,  # e.g: 42
+            "top_p": float,  # e.g: 0.9
+            "frequency_penalty": float,  # e.g: 0.5
+            "presence_penalty": float,  # e.g: 0.5
+            "stop": list,  # e.g: ["Stop"]
+            "reasoning_effort": str,  # e.g: minimal, low, medium, high
+            "verbosity": str,  # e.g: low, medium, high
+            "service_tier": str,  # e.g: auto, default
+            "top_logprobs": int,
+            "safety_identifier": str,
+            "store": bool,
+            "enable_thinking": bool,
+        }
+        self.all_providers = [
+            OpenAiCompatible,
+            MistralProvider,
+            GroqProvider,
+            XaiProvider,
+            CodestralProvider,
+            OllamaProvider,
+            LMStudioProvider,
+        ]
+        self.aspect_ratios = {
+            "1:1": "1024×1024",
+            "2:3": "832×1248",
+            "3:2": "1248×832",
+            "3:4": "864×1184",
+            "4:3": "1184×864",
+            "4:5": "896×1152",
+            "5:4": "1152×896",
+            "9:16": "768×1344",
+            "16:9": "1344×768",
+            "21:9": "1536×672",
+        }
+        self.import_maps = {
+            "vue-prod": "/ui/lib/vue.min.mjs",
+            "vue": "/ui/lib/vue.mjs",
+            "vue-router": "/ui/lib/vue-router.min.mjs",
+            "@servicestack/client": "/ui/lib/servicestack-client.mjs",
+            "@servicestack/vue": "/ui/lib/servicestack-vue.mjs",
+            "idb": "/ui/lib/idb.min.mjs",
+            "marked": "/ui/lib/marked.min.mjs",
+            "highlight.js": "/ui/lib/highlight.min.mjs",
+            "chart.js": "/ui/lib/chart.js",
+            "color.js": "/ui/lib/color.js",
+            "ctx.mjs": "/ui/ctx.mjs",
+        }
+
+    def set_config(self, config):
+        self.config = config
+        self.auth_enabled = self.config.get("auth", {}).get("enabled", False)
+
+    # Authentication middleware helper
+    def check_auth(self, request):
+        """Check if request is authenticated. Returns (is_authenticated, user_data)"""
+        if not self.auth_enabled:
+            return True, None
+
+        # Check for OAuth session token
+        session_token = get_session_token(request)
+        if session_token and session_token in g_sessions:
+            return True, g_sessions[session_token]
+
+        # Check for API key
+        auth_header = request.headers.get("Authorization", "")
+        if auth_header.startswith("Bearer "):
+            api_key = auth_header[7:]
+            if api_key:
+                return True, {"authProvider": "apikey"}
+
+        return False, None
+
+    def get_session(self, request):
+        session_token = get_session_token(request)
+
+        if not session_token or session_token not in g_sessions:
+            return None
+
+        session_data = g_sessions[session_token]
+        return session_data
+
+    def get_username(self, request):
+        session = self.get_session(request)
+        if session:
+            return session.get("userName")
+        return None
+
+    def get_user_path(self, username=None):
+        if username:
+            return home_llms_path(os.path.join("user", username))
+        return home_llms_path(os.path.join("user", "default"))
+
+    def chat_request(self, template=None, text=None, model=None, system_prompt=None):
+        return g_chat_request(template=template, text=text, model=model, system_prompt=system_prompt)
+
+    async def chat_completion(self, chat, context=None):
+        response = await g_chat_completion(chat, context)
+        return response
+
+    def on_cache_saved_filters(self, context):
+        # _log(f"on_cache_saved_filters {len(self.cache_saved_filters)}: {context['url']}")
+        for filter_func in self.cache_saved_filters:
+            filter_func(context)
+
+    async def on_chat_error(self, e, context):
+        # Apply chat error filters
+        if "stackTrace" not in context:
+            context["stackTrace"] = traceback.format_exc()
+        for filter_func in self.chat_error_filters:
             try:
-                if ui_path.is_file():
-                    mtime = ui_path.stat().st_mtime
+                await filter_func(e, context)
+            except Exception as e:
+                _err("chat error filter failed", e)
+
+    async def on_chat_tool(self, chat, context):
+        m_len = len(chat.get("messages", []))
+        t_len = len(self.chat_tool_filters)
+        _dbg(
+            f"on_tool_call for thread {context.get('threadId', None)} with {m_len} {pluralize('message', m_len)}, invoking {t_len} {pluralize('filter', t_len)}:"
+        )
+        for filter_func in self.chat_tool_filters:
+            await filter_func(chat, context)
+
+    def exit(self, exit_code=0):
+        if len(self.shutdown_handlers) > 0:
+            _dbg(f"running {len(self.shutdown_handlers)} shutdown handlers...")
+            for handler in self.shutdown_handlers:
+                handler()
+
+        _dbg(f"exit({exit_code})")
+        sys.exit(exit_code)
+
+
+def handler_name(handler):
+    if hasattr(handler, "__name__"):
+        return handler.__name__
+    return "unknown"
+
+
+class ExtensionContext:
+    def __init__(self, app, path):
+        self.app = app
+        self.cli_args = app.cli_args
+        self.extra_args = app.extra_args
+        self.error_auth_required = app.error_auth_required
+        self.path = path
+        self.name = os.path.basename(path)
+        if self.name.endswith(".py"):
+            self.name = self.name[:-3]
+        self.ext_prefix = f"/ext/{self.name}"
+        self.MOCK = MOCK
+        self.MOCK_DIR = MOCK_DIR
+        self.debug = DEBUG
+        self.verbose = g_verbose
+        self.aspect_ratios = app.aspect_ratios
+        self.request_args = app.request_args
+
+    def chat_to_prompt(self, chat):
+        return chat_to_prompt(chat)
+
+    def chat_to_system_prompt(self, chat):
+        return chat_to_system_prompt(chat)
+
+    def chat_response_to_message(self, response):
+        return chat_response_to_message(response)
+
+    def last_user_prompt(self, chat):
+        return last_user_prompt(chat)
+
+    def to_file_info(self, chat, info=None, response=None):
+        return to_file_info(chat, info=info, response=response)
+
+    def save_image_to_cache(self, base64_data, filename, image_info):
+        return save_image_to_cache(base64_data, filename, image_info)
+
+    def save_bytes_to_cache(self, bytes_data, filename, file_info):
+        return save_bytes_to_cache(bytes_data, filename, file_info)
+
+    def text_from_file(self, path):
+        return text_from_file(path)
+
+    def log(self, message):
+        if self.verbose:
+            print(f"[{self.name}] {message}", flush=True)
+        return message
+
+    def log_json(self, obj):
+        if self.verbose:
+            print(f"[{self.name}] {json.dumps(obj, indent=2)}", flush=True)
+        return obj
+
+    def dbg(self, message):
+        if self.debug:
+            print(f"DEBUG [{self.name}]: {message}", flush=True)
 
-                    if str(ui_path) not in file_mtimes:
-                        file_mtimes[str(ui_path)] = mtime
-                    elif file_mtimes[str(ui_path)] != mtime:
-                        _log(f"Config file changed: {ui_path.name}")
-                        file_mtimes[str(ui_path)] = mtime
-                        _log("ui.json reloaded - reload page to update")
-            except FileNotFoundError:
-                pass
+    def err(self, message, e):
+        print(f"ERROR [{self.name}]: {message}", e)
+        if self.verbose:
+            print(traceback.format_exc(), flush=True)
+
+    def error_message(self, e):
+        return to_error_message(e)
+
+    def error_response(self, e, stacktrace=False):
+        return to_error_response(e, stacktrace=stacktrace)
+
+    def add_provider(self, provider):
+        self.log(f"Registered provider: {provider.__name__}")
+        self.app.all_providers.append(provider)
+
+    def register_ui_extension(self, index):
+        path = os.path.join(self.ext_prefix, index)
+        self.log(f"Registered UI extension: {path}")
+        self.app.ui_extensions.append({"id": self.name, "path": path})
+
+    def register_chat_request_filter(self, handler):
+        self.log(f"Registered chat request filter: {handler_name(handler)}")
+        self.app.chat_request_filters.append(handler)
+
+    def register_chat_tool_filter(self, handler):
+        self.log(f"Registered chat tool filter: {handler_name(handler)}")
+        self.app.chat_tool_filters.append(handler)
+
+    def register_chat_response_filter(self, handler):
+        self.log(f"Registered chat response filter: {handler_name(handler)}")
+        self.app.chat_response_filters.append(handler)
+
+    def register_chat_error_filter(self, handler):
+        self.log(f"Registered chat error filter: {handler_name(handler)}")
+        self.app.chat_error_filters.append(handler)
+
+    def register_cache_saved_filter(self, handler):
+        self.log(f"Registered cache saved filter: {handler_name(handler)}")
+        self.app.cache_saved_filters.append(handler)
+
+    def register_shutdown_handler(self, handler):
+        self.log(f"Registered shutdown handler: {handler_name(handler)}")
+        self.app.shutdown_handlers.append(handler)
+
+    def add_static_files(self, ext_dir):
+        self.log(f"Registered static files: {ext_dir}")
+
+        async def serve_static(request):
+            path = request.match_info["path"]
+            file_path = os.path.join(ext_dir, path)
+            if os.path.exists(file_path):
+                return web.FileResponse(file_path)
+            return web.Response(status=404)
+
+        self.app.server_add_get.append((os.path.join(self.ext_prefix, "{path:.*}"), serve_static, {}))
+
+    def add_get(self, path, handler, **kwargs):
+        self.dbg(f"Registered GET: {os.path.join(self.ext_prefix, path)}")
+        self.app.server_add_get.append((os.path.join(self.ext_prefix, path), handler, kwargs))
+
+    def add_post(self, path, handler, **kwargs):
+        self.dbg(f"Registered POST: {os.path.join(self.ext_prefix, path)}")
+        self.app.server_add_post.append((os.path.join(self.ext_prefix, path), handler, kwargs))
+
+    def add_put(self, path, handler, **kwargs):
+        self.dbg(f"Registered PUT: {os.path.join(self.ext_prefix, path)}")
+        self.app.server_add_put.append((os.path.join(self.ext_prefix, path), handler, kwargs))
+
+    def add_delete(self, path, handler, **kwargs):
+        self.dbg(f"Registered DELETE: {os.path.join(self.ext_prefix, path)}")
+        self.app.server_add_delete.append((os.path.join(self.ext_prefix, path), handler, kwargs))
+
+    def add_patch(self, path, handler, **kwargs):
+        self.dbg(f"Registered PATCH: {os.path.join(self.ext_prefix, path)}")
+        self.app.server_add_patch.append((os.path.join(self.ext_prefix, path), handler, kwargs))
+
+    def add_importmaps(self, dict):
+        self.app.import_maps.update(dict)
+
+    def add_index_header(self, html):
+        self.app.index_headers.append(html)
+
+    def add_index_footer(self, html):
+        self.app.index_footers.append(html)
+
+    def get_config(self):
+        return g_config
+
+    def get_cache_path(self, path=""):
+        return get_cache_path(path)
+
+    def chat_request(self, template=None, text=None, model=None, system_prompt=None):
+        return self.app.chat_request(template=template, text=text, model=model, system_prompt=system_prompt)
+
+    def chat_completion(self, chat, context=None):
+        return self.app.chat_completion(chat, context=context)
+
+    def get_providers(self):
+        return g_handlers
+
+    def get_provider(self, name):
+        return g_handlers.get(name)
+
+    def register_tool(self, func, tool_def=None):
+        if tool_def is None:
+            tool_def = function_to_tool_definition(func)
+
+        name = tool_def["function"]["name"]
+        self.log(f"Registered tool: {name}")
+        self.app.tools[name] = func
+        self.app.tool_definitions.append(tool_def)
+
+    def check_auth(self, request):
+        return self.app.check_auth(request)
+
+    def get_session(self, request):
+        return self.app.get_session(request)
+
+    def get_username(self, request):
+        return self.app.get_username(request)
+
+    def get_user_path(self, username=None):
+        return self.app.get_user_path(username)
+
+    def should_cancel_thread(self, context):
+        return should_cancel_thread(context)
+
+    def cache_message_inline_data(self, message):
+        return cache_message_inline_data(message)
+
+    def to_content(self, result):
+        return to_content(result)
+
+
+def get_extensions_path():
+    return os.getenv("LLMS_EXTENSIONS_DIR", os.path.join(Path.home(), ".llms", "extensions"))
+
+
+def get_disabled_extensions():
+    ret = DISABLE_EXTENSIONS.copy()
+    if g_config:
+        for ext in g_config.get("disable_extensions", []):
+            if ext not in ret:
+                ret.append(ext)
+    return ret
+
+
+def get_extensions_dirs():
+    """
+    Returns a list of extension directories.
+    """
+    extensions_path = get_extensions_path()
+    os.makedirs(extensions_path, exist_ok=True)
+
+    # allow overriding builtin extensions
+    override_extensions = []
+    if os.path.exists(extensions_path):
+        override_extensions = os.listdir(extensions_path)
+
+    ret = []
+    disabled_extensions = get_disabled_extensions()
+
+    builtin_extensions_dir = _ROOT / "extensions"
+    if os.path.exists(builtin_extensions_dir):
+        for item in os.listdir(builtin_extensions_dir):
+            if os.path.isdir(os.path.join(builtin_extensions_dir, item)):
+                if item in override_extensions:
+                    continue
+                if item in disabled_extensions:
+                    continue
+                ret.append(os.path.join(builtin_extensions_dir, item))
+
+    if os.path.exists(extensions_path):
+        for item in os.listdir(extensions_path):
+            if os.path.isdir(os.path.join(extensions_path, item)):
+                if item in disabled_extensions:
+                    continue
+                ret.append(os.path.join(extensions_path, item))
+
+    return ret
+
+
+def init_extensions(parser):
+    """
+    Initializes extensions by loading their __init__.py files and calling the __parser__ function if it exists.
+    """
+    for item_path in get_extensions_dirs():
+        item = os.path.basename(item_path)
+
+        if os.path.isdir(item_path):
+            try:
+                # check for __parser__ function if exists in __init.__.py and call it with parser
+                init_file = os.path.join(item_path, "__init__.py")
+                if os.path.exists(init_file):
+                    spec = importlib.util.spec_from_file_location(item, init_file)
+                    if spec and spec.loader:
+                        module = importlib.util.module_from_spec(spec)
+                        sys.modules[item] = module
+                        spec.loader.exec_module(module)
+
+                        parser_func = getattr(module, "__parser__", None)
+                        if callable(parser_func):
+                            parser_func(parser)
+                            _log(f"Extension {item} parser loaded")
+            except Exception as e:
+                _err(f"Failed to load extension {item} parser", e)
+
+
+def install_extensions():
+    """
+    Scans ensure ~/.llms/extensions/ for directories with __init__.py and loads them as extensions.
+    Calls the `__install__(ctx)` function in the extension module.
+    """
+
+    extension_dirs = get_extensions_dirs()
+    ext_count = len(list(extension_dirs))
+    if ext_count == 0:
+        _log("No extensions found")
+        return
+
+    disabled_extensions = get_disabled_extensions()
+    if len(disabled_extensions) > 0:
+        _log(f"Disabled extensions: {', '.join(disabled_extensions)}")
+
+    _log(f"Installing {ext_count} extension{'' if ext_count == 1 else 's'}...")
+
+    for item_path in extension_dirs:
+        item = os.path.basename(item_path)
+
+        if os.path.isdir(item_path):
+            sys.path.append(item_path)
+            try:
+                ctx = ExtensionContext(g_app, item_path)
+                init_file = os.path.join(item_path, "__init__.py")
+                if os.path.exists(init_file):
+                    spec = importlib.util.spec_from_file_location(item, init_file)
+                    if spec and spec.loader:
+                        module = importlib.util.module_from_spec(spec)
+                        sys.modules[item] = module
+                        spec.loader.exec_module(module)
+
+                        install_func = getattr(module, "__install__", None)
+                        if callable(install_func):
+                            install_func(ctx)
+                            _log(f"Extension {item} installed")
+                        else:
+                            _dbg(f"Extension {item} has no __install__ function")
+                    else:
+                        _dbg(f"Extension {item} has no __init__.py")
+                else:
+                    _dbg(f"Extension {init_file} not found")
+
+                # if ui folder exists, serve as static files at /ext/{item}/
+                ui_path = os.path.join(item_path, "ui")
+                if os.path.exists(ui_path):
+                    ctx.add_static_files(ui_path)
+
+                # Register UI extension if index.mjs exists (/ext/{item}/index.mjs)
+                if os.path.exists(os.path.join(ui_path, "index.mjs")):
+                    ctx.register_ui_extension("index.mjs")
+
+            except Exception as e:
+                _err(f"Failed to install extension {item}", e)
+        else:
+            _dbg(f"Extension {item} not found: {item_path} is not a directory {os.path.exists(item_path)}")
+
+
+def run_extension_cli():
+    """
+    Run the CLI for an extension.
+    """
+    for item_path in get_extensions_dirs():
+        item = os.path.basename(item_path)
+
+        if os.path.isdir(item_path):
+            init_file = os.path.join(item_path, "__init__.py")
+            if os.path.exists(init_file):
+                ctx = ExtensionContext(g_app, item_path)
+                try:
+                    spec = importlib.util.spec_from_file_location(item, init_file)
+                    if spec and spec.loader:
+                        module = importlib.util.module_from_spec(spec)
+                        sys.modules[item] = module
+                        spec.loader.exec_module(module)
+
+                    # Check for __run__ function if exists in __init__.py and call it with ctx
+                    run_func = getattr(module, "__run__", None)
+                    if callable(run_func):
+                        _log(f"Running extension {item}...")
+                        handled = run_func(ctx)
+                        return handled
+
+                except Exception as e:
+                    _err(f"Failed to run extension {item}", e)
+                    return False
 
 
 def main():
-    global _ROOT, g_verbose, g_default_model, g_logprefix, g_config, g_config_path, g_ui_path
+    global _ROOT, g_verbose, g_default_model, g_logprefix, g_providers, g_config, g_config_path, g_app
+
+    _ROOT = os.getenv("LLMS_ROOT", resolve_root())
+    if not _ROOT:
+        print("Resource root not found")
+        exit(1)
 
     parser = argparse.ArgumentParser(description=f"llms v{VERSION}")
     parser.add_argument("--config", default=None, help="Path to config file", metavar="FILE")
+    parser.add_argument("--providers", default=None, help="Path to models.dev providers file", metavar="FILE")
     parser.add_argument("-m", "--model", default=None, help="Model to use")
 
     parser.add_argument("--chat", default=None, help="OpenAI Chat Completion Request to send", metavar="REQUEST")
     parser.add_argument(
         "-s", "--system", default=None, help="System prompt to use for chat completion", metavar="PROMPT"
     )
+    parser.add_argument(
+        "--tools", default=None, help="Tools to use for chat completion (all|none|<tool>,<tool>...)", metavar="TOOLS"
+    )
     parser.add_argument("--image", default=None, help="Image input to use in chat completion")
     parser.add_argument("--audio", default=None, help="Audio input to use in chat completion")
     parser.add_argument("--file", default=None, help="File input to use in chat completion")
+    parser.add_argument("--out", default=None, help="Image or Video Generation Request", metavar="MODALITY")
     parser.add_argument(
         "--args",
         default=None,
@@ -1573,15 +2878,46 @@ def main():
     parser.add_argument("--default", default=None, help="Configure the default model to use", metavar="MODEL")
 
     parser.add_argument("--init", action="store_true", help="Create a default llms.json")
+    parser.add_argument("--update-providers", action="store_true", help="Update local models.dev providers.json")
 
-    parser.add_argument("--root", default=None, help="Change root directory for UI files", metavar="PATH")
     parser.add_argument("--logprefix", default="", help="Prefix used in log messages", metavar="PREFIX")
     parser.add_argument("--verbose", action="store_true", help="Verbose output")
 
+    parser.add_argument(
+        "--add",
+        nargs="?",
+        const="ls",
+        default=None,
+        help="Install an extension (lists available extensions if no name provided)",
+        metavar="EXTENSION",
+    )
+    parser.add_argument(
+        "--remove",
+        nargs="?",
+        const="ls",
+        default=None,
+        help="Remove an extension (lists installed extensions if no name provided)",
+        metavar="EXTENSION",
+    )
+
+    parser.add_argument(
+        "--update",
+        nargs="?",
+        const="ls",
+        default=None,
+        help="Update an extension (use 'all' to update all extensions)",
+        metavar="EXTENSION",
+    )
+
+    # Load parser extensions, go through all extensions and load their parser arguments
+    init_extensions(parser)
+
     cli_args, extra_args = parser.parse_known_args()
 
+    g_app = AppExtensions(cli_args, extra_args)
+
     # Check for verbose mode from CLI argument or environment variables
-    verbose_env = os.environ.get("VERBOSE", "").lower()
+    verbose_env = os.getenv("VERBOSE", "").lower()
     if cli_args.verbose or verbose_env in ("1", "true"):
         g_verbose = True
         # printdump(cli_args)
@@ -1590,13 +2926,9 @@ def main():
     if cli_args.logprefix:
         g_logprefix = cli_args.logprefix
 
-    _ROOT = Path(cli_args.root) if cli_args.root else resolve_root()
-    if not _ROOT:
-        print("Resource root not found")
-        exit(1)
-
     home_config_path = home_llms_path("llms.json")
-    home_ui_path = home_llms_path("ui.json")
+    home_providers_path = home_llms_path("providers.json")
+    home_providers_extra_path = home_llms_path("providers-extra.json")
 
     if cli_args.init:
         if os.path.exists(home_config_path):
@@ -1605,38 +2937,215 @@ def main():
             asyncio.run(save_default_config(home_config_path))
             print(f"Created default config at {home_config_path}")
 
-        if os.path.exists(home_ui_path):
-            print(f"ui.json already exists at {home_ui_path}")
+        if os.path.exists(home_providers_path):
+            print(f"providers.json already exists at {home_providers_path}")
+        else:
+            asyncio.run(save_text_url(github_url("providers.json"), home_providers_path))
+            print(f"Created default providers config at {home_providers_path}")
+
+        if os.path.exists(home_providers_extra_path):
+            print(f"providers-extra.json already exists at {home_providers_extra_path}")
         else:
-            asyncio.run(save_text_url(github_url("ui.json"), home_ui_path))
-            print(f"Created default ui config at {home_ui_path}")
+            asyncio.run(save_text_url(github_url("providers-extra.json"), home_providers_extra_path))
+            print(f"Created default extra providers config at {home_providers_extra_path}")
         exit(0)
 
+    if cli_args.providers:
+        if not os.path.exists(cli_args.providers):
+            print(f"providers.json not found at {cli_args.providers}")
+            exit(1)
+        g_providers = json.loads(text_from_file(cli_args.providers))
+
     if cli_args.config:
         # read contents
         g_config_path = cli_args.config
         with open(g_config_path, encoding="utf-8") as f:
             config_json = f.read()
-            g_config = json.loads(config_json)
+            g_config = load_config_json(config_json)
 
         config_dir = os.path.dirname(g_config_path)
-        # look for ui.json in same directory as config
-        ui_path = os.path.join(config_dir, "ui.json")
-        if os.path.exists(ui_path):
-            g_ui_path = ui_path
-        else:
-            if not os.path.exists(home_ui_path):
-                ui_json = text_from_resource("ui.json")
-                with open(home_ui_path, "w", encoding="utf-8") as f:
-                    f.write(ui_json)
-                _log(f"Created default ui config at {home_ui_path}")
-            g_ui_path = home_ui_path
+
+        if not g_providers and os.path.exists(os.path.join(config_dir, "providers.json")):
+            g_providers = json.loads(text_from_file(os.path.join(config_dir, "providers.json")))
+
     else:
-        # ensure llms.json and ui.json exist in home directory
+        # ensure llms.json and providers.json exist in home directory
         asyncio.run(save_home_configs())
         g_config_path = home_config_path
-        g_ui_path = home_ui_path
-        g_config = json.loads(text_from_file(g_config_path))
+        g_config = load_config_json(text_from_file(g_config_path))
+
+    g_app.set_config(g_config)
+
+    if not g_providers:
+        g_providers = json.loads(text_from_file(home_providers_path))
+
+    if cli_args.update_providers:
+        asyncio.run(update_providers(home_providers_path))
+        print(f"Updated {home_providers_path}")
+        exit(0)
+
+    # if home_providers_path is older than 1 day, update providers list
+    if (
+        os.path.exists(home_providers_path)
+        and (time.time() - os.path.getmtime(home_providers_path)) > 86400
+        and os.getenv("LLMS_DISABLE_UPDATE", "") != "1"
+    ):
+        try:
+            asyncio.run(update_providers(home_providers_path))
+            _log(f"Updated {home_providers_path}")
+        except Exception as e:
+            _err("Failed to update providers", e)
+
+    if cli_args.add is not None:
+        if cli_args.add == "ls":
+
+            async def list_extensions():
+                print("\nAvailable extensions:")
+                text = await get_text("https://api.github.com/orgs/llmspy/repos?per_page=100&sort=updated")
+                repos = json.loads(text)
+                max_name_length = 0
+                for repo in repos:
+                    max_name_length = max(max_name_length, len(repo["name"]))
+
+                for repo in repos:
+                    print(f"  {repo['name']:<{max_name_length + 2}} {repo['description']}")
+
+                print("\nUsage:")
+                print("  llms --add <extension>")
+                print("  llms --add <github-user>/<repo>")
+
+            asyncio.run(list_extensions())
+            exit(0)
+
+        async def install_extension(name):
+            # Determine git URL and target directory name
+            if "/" in name:
+                git_url = f"https://github.com/{name}"
+                target_name = name.split("/")[-1]
+            else:
+                git_url = f"https://github.com/llmspy/{name}"
+                target_name = name
+
+            # check extension is not already installed
+            extensions_path = get_extensions_path()
+            target_path = os.path.join(extensions_path, target_name)
+
+            if os.path.exists(target_path):
+                print(f"Extension {target_name} is already installed at {target_path}")
+                return
+
+            print(f"Installing extension: {name}")
+            print(f"Cloning from {git_url} to {target_path}...")
+
+            try:
+                subprocess.run(["git", "clone", git_url, target_path], check=True)
+
+                # Check for requirements.txt
+                requirements_path = os.path.join(target_path, "requirements.txt")
+                if os.path.exists(requirements_path):
+                    print(f"Installing dependencies from {requirements_path}...")
+
+                    # Check if uv is installed
+                    has_uv = False
+                    try:
+                        subprocess.run(
+                            ["uv", "--version"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=True
+                        )
+                        has_uv = True
+                    except (subprocess.CalledProcessError, FileNotFoundError):
+                        pass
+
+                    if has_uv:
+                        subprocess.run(
+                            ["uv", "pip", "install", "-p", sys.executable, "-r", "requirements.txt"],
+                            cwd=target_path,
+                            check=True,
+                        )
+                    else:
+                        subprocess.run(
+                            [sys.executable, "-m", "pip", "install", "-r", "requirements.txt"],
+                            cwd=target_path,
+                            check=True,
+                        )
+                    print("Dependencies installed successfully.")
+
+                print(f"Extension {target_name} installed successfully.")
+
+            except subprocess.CalledProcessError as e:
+                print(f"Failed to install extension: {e}")
+                # cleanup if clone failed but directory was created (unlikely with simple git clone but good practice)
+                if os.path.exists(target_path) and not os.listdir(target_path):
+                    os.rmdir(target_path)
+
+        asyncio.run(install_extension(cli_args.add))
+        exit(0)
+
+    if cli_args.remove is not None:
+        if cli_args.remove == "ls":
+            # List installed extensions
+            extensions_path = get_extensions_path()
+            extensions = os.listdir(extensions_path)
+            if len(extensions) == 0:
+                print("No extensions installed.")
+                exit(0)
+            print("Installed extensions:")
+            for extension in extensions:
+                print(f"  {extension}")
+            exit(0)
+        # Remove an extension
+        extension_name = cli_args.remove
+        extensions_path = get_extensions_path()
+        target_path = os.path.join(extensions_path, extension_name)
+
+        if not os.path.exists(target_path):
+            print(f"Extension {extension_name} not found at {target_path}")
+            exit(1)
+
+        print(f"Removing extension: {extension_name}...")
+        try:
+            shutil.rmtree(target_path)
+            print(f"Extension {extension_name} removed successfully.")
+        except Exception as e:
+            print(f"Failed to remove extension: {e}")
+            exit(1)
+
+        exit(0)
+
+    if cli_args.update:
+        if cli_args.update == "ls":
+            # List installed extensions
+            extensions_path = get_extensions_path()
+            extensions = os.listdir(extensions_path)
+            if len(extensions) == 0:
+                print("No extensions installed.")
+                exit(0)
+            print("Installed extensions:")
+            for extension in extensions:
+                print(f"  {extension}")
+
+            print("\nUsage:")
+            print("  llms --update <extension>")
+            print("  llms --update all")
+            exit(0)
+
+        async def update_extensions(extension_name):
+            extensions_path = get_extensions_path()
+            for extension in os.listdir(extensions_path):
+                extension_path = os.path.join(extensions_path, extension)
+                if os.path.isdir(extension_path):
+                    if extension_name != "all" and extension != extension_name:
+                        continue
+                    result = subprocess.run(["git", "pull"], cwd=extension_path, capture_output=True)
+                    if result.returncode != 0:
+                        print(f"Failed to update extension {extension}: {result.stderr.decode('utf-8')}")
+                        continue
+                    print(f"Updated extension {extension}")
+                    _log(result.stdout.decode("utf-8"))
+
+        asyncio.run(update_extensions(cli_args.update))
+        exit(0)
+
+    install_extensions()
 
     asyncio.run(reload_providers())
 
@@ -1654,23 +3163,45 @@ def main():
     if cli_args.list:
         # Show list of enabled providers and their models
         enabled = []
+        provider_count = 0
+        model_count = 0
+
+        max_model_length = 0
+        for name, provider in g_handlers.items():
+            if len(filter_list) > 0 and name not in filter_list:
+                continue
+            for model in provider.models:
+                max_model_length = max(max_model_length, len(model))
+
         for name, provider in g_handlers.items():
             if len(filter_list) > 0 and name not in filter_list:
                 continue
+            provider_count += 1
             print(f"{name}:")
             enabled.append(name)
             for model in provider.models:
-                print(f"  {model}")
+                model_count += 1
+                model_cost_info = None
+                if "cost" in provider.models[model]:
+                    model_cost = provider.models[model]["cost"]
+                    if "input" in model_cost and "output" in model_cost:
+                        if model_cost["input"] == 0 and model_cost["output"] == 0:
+                            model_cost_info = "      0"
+                        else:
+                            model_cost_info = f"{model_cost['input']:5} / {model_cost['output']}"
+                print(f"  {model:{max_model_length}} {model_cost_info or ''}")
+
+        print(f"\n{model_count} models available from {provider_count} providers")
 
         print_status()
-        exit(0)
+        g_app.exit(0)
 
     if cli_args.check is not None:
         # Check validity of models for a provider
         provider_name = cli_args.check
         model_names = extra_args if len(extra_args) > 0 else None
         asyncio.run(check_models(provider_name, model_names))
-        exit(0)
+        g_app.exit(0)
 
     if cli_args.serve is not None:
         # Disable inactive providers and save to config before starting server
@@ -1690,10 +3221,6 @@ def main():
         # Start server
         port = int(cli_args.serve)
 
-        if not os.path.exists(g_ui_path):
-            print(f"UI not found at {g_ui_path}")
-            exit(1)
-
         # Validate auth configuration if enabled
         auth_enabled = g_config.get("auth", {}).get("enabled", False)
         if auth_enabled:
@@ -1703,11 +3230,19 @@ def main():
 
             # Expand environment variables
             if client_id.startswith("$"):
-                client_id = os.environ.get(client_id[1:], "")
+                client_id = client_id[1:]
             if client_secret.startswith("$"):
-                client_secret = os.environ.get(client_secret[1:], "")
+                client_secret = client_secret[1:]
 
-            if not client_id or not client_secret:
+            client_id = os.getenv(client_id, client_id)
+            client_secret = os.getenv(client_secret, client_secret)
+
+            if (
+                not client_id
+                or not client_secret
+                or client_id == "GITHUB_CLIENT_ID"
+                or client_secret == "GITHUB_CLIENT_SECRET"
+            ):
                 print("ERROR: Authentication is enabled but GitHub OAuth is not properly configured.")
                 print("Please set GITHUB_CLIENT_ID and GITHUB_CLIENT_SECRET environment variables,")
                 print("or disable authentication by setting 'auth.enabled' to false in llms.json")
@@ -1721,60 +3256,35 @@ def main():
         _log(f"client_max_size set to {client_max_size} bytes ({client_max_size / 1024 / 1024:.1f}MB)")
         app = web.Application(client_max_size=client_max_size)
 
-        # Authentication middleware helper
-        def check_auth(request):
-            """Check if request is authenticated. Returns (is_authenticated, user_data)"""
-            if not auth_enabled:
-                return True, None
-
-            # Check for OAuth session token
-            session_token = request.query.get("session") or request.headers.get("X-Session-Token")
-            if session_token and session_token in g_sessions:
-                return True, g_sessions[session_token]
-
-            # Check for API key
-            auth_header = request.headers.get("Authorization", "")
-            if auth_header.startswith("Bearer "):
-                api_key = auth_header[7:]
-                if api_key:
-                    return True, {"authProvider": "apikey"}
-
-            return False, None
-
         async def chat_handler(request):
             # Check authentication if enabled
-            is_authenticated, user_data = check_auth(request)
+            is_authenticated, user_data = g_app.check_auth(request)
             if not is_authenticated:
-                return web.json_response(
-                    {
-                        "error": {
-                            "message": "Authentication required",
-                            "type": "authentication_error",
-                            "code": "unauthorized",
-                        }
-                    },
-                    status=401,
-                )
+                return web.json_response(g_app.error_auth_required, status=401)
 
             try:
                 chat = await request.json()
-                response = await chat_completion(chat)
+                context = {"chat": chat, "request": request, "user": g_app.get_username(request)}
+                metadata = chat.get("metadata", {})
+                context["threadId"] = metadata.get("threadId", None)
+                context["tools"] = metadata.get("tools", "all")
+                response = await g_app.chat_completion(chat, context)
                 return web.json_response(response)
             except Exception as e:
-                return web.json_response({"error": str(e)}, status=500)
+                return web.json_response(to_error_response(e), status=500)
 
         app.router.add_post("/v1/chat/completions", chat_handler)
 
-        async def models_handler(request):
-            return web.json_response(get_models())
-
-        app.router.add_get("/models/list", models_handler)
-
         async def active_models_handler(request):
             return web.json_response(get_active_models())
 
         app.router.add_get("/models", active_models_handler)
 
+        async def active_providers_handler(request):
+            return web.json_response(api_providers())
+
+        app.router.add_get("/providers", active_providers_handler)
+
         async def status_handler(request):
             enabled, disabled = provider_status()
             return web.json_response(
@@ -1794,8 +3304,9 @@ def main():
             if provider:
                 if data.get("enable", False):
                     provider_config, msg = enable_provider(provider)
-                    _log(f"Enabled provider {provider}")
-                    await load_llms()
+                    _log(f"Enabled provider {provider} {msg}")
+                    if not msg:
+                        await load_llms()
                 elif data.get("disable", False):
                     disable_provider(provider)
                     _log(f"Disabled provider {provider}")
@@ -1810,11 +3321,144 @@ def main():
 
         app.router.add_post("/providers/{provider}", provider_handler)
 
+        async def upload_handler(request):
+            # Check authentication if enabled
+            is_authenticated, user_data = g_app.check_auth(request)
+            if not is_authenticated:
+                return web.json_response(g_app.error_auth_required, status=401)
+
+            reader = await request.multipart()
+
+            # Read first file field
+            field = await reader.next()
+            while field and field.name != "file":
+                field = await reader.next()
+
+            if not field:
+                return web.json_response(create_error_response("No file provided"), status=400)
+
+            filename = field.filename or "file"
+            content = await field.read()
+            mimetype = get_file_mime_type(filename)
+
+            # If image, resize if needed
+            if mimetype.startswith("image/"):
+                content, mimetype = convert_image_if_needed(content, mimetype)
+
+            # Calculate SHA256
+            sha256_hash = hashlib.sha256(content).hexdigest()
+            ext = filename.rsplit(".", 1)[1] if "." in filename else ""
+            if not ext:
+                ext = mimetypes.guess_extension(mimetype) or ""
+                if ext.startswith("."):
+                    ext = ext[1:]
+
+            if not ext:
+                ext = "bin"
+
+            save_filename = f"{sha256_hash}.{ext}" if ext else sha256_hash
+
+            # Use first 2 chars for subdir to avoid too many files in one dir
+            subdir = sha256_hash[:2]
+            relative_path = f"{subdir}/{save_filename}"
+            full_path = get_cache_path(relative_path)
+
+            # if file and its .info.json already exists, return it
+            info_path = os.path.splitext(full_path)[0] + ".info.json"
+            if os.path.exists(full_path) and os.path.exists(info_path):
+                return web.json_response(json.load(open(info_path)))
+
+            os.makedirs(os.path.dirname(full_path), exist_ok=True)
+
+            with open(full_path, "wb") as f:
+                f.write(content)
+
+            url = f"/~cache/{relative_path}"
+            response_data = {
+                "date": int(time.time()),
+                "url": url,
+                "size": len(content),
+                "type": mimetype,
+                "name": filename,
+            }
+
+            # If image, get dimensions
+            if HAS_PIL and mimetype.startswith("image/"):
+                try:
+                    with Image.open(BytesIO(content)) as img:
+                        response_data["width"] = img.width
+                        response_data["height"] = img.height
+                except Exception:
+                    pass
+
+            # Save metadata
+            info_path = os.path.splitext(full_path)[0] + ".info.json"
+            with open(info_path, "w") as f:
+                json.dump(response_data, f)
+
+            g_app.on_cache_saved_filters({"url": url, "info": response_data})
+
+            return web.json_response(response_data)
+
+        app.router.add_post("/upload", upload_handler)
+
+        async def extensions_handler(request):
+            return web.json_response(g_app.ui_extensions)
+
+        app.router.add_get("/ext", extensions_handler)
+
+        async def tools_handler(request):
+            return web.json_response(g_app.tool_definitions)
+
+        app.router.add_get("/ext/tools", tools_handler)
+
+        async def cache_handler(request):
+            path = request.match_info["tail"]
+            full_path = get_cache_path(path)
+
+            if "info" in request.query:
+                info_path = os.path.splitext(full_path)[0] + ".info.json"
+                if not os.path.exists(info_path):
+                    return web.Response(text="404: Not Found", status=404)
+
+                # Check for directory traversal for info path
+                try:
+                    cache_root = Path(get_cache_path())
+                    requested_path = Path(info_path).resolve()
+                    if not str(requested_path).startswith(str(cache_root)):
+                        return web.Response(text="403: Forbidden", status=403)
+                except Exception:
+                    return web.Response(text="403: Forbidden", status=403)
+
+                with open(info_path) as f:
+                    content = f.read()
+                return web.Response(text=content, content_type="application/json")
+
+            if not os.path.exists(full_path):
+                return web.Response(text="404: Not Found", status=404)
+
+            # Check for directory traversal
+            try:
+                cache_root = Path(get_cache_path())
+                requested_path = Path(full_path).resolve()
+                if not str(requested_path).startswith(str(cache_root)):
+                    return web.Response(text="403: Forbidden", status=403)
+            except Exception:
+                return web.Response(text="403: Forbidden", status=403)
+
+            with open(full_path, "rb") as f:
+                content = f.read()
+
+            mimetype = get_file_mime_type(full_path)
+            return web.Response(body=content, content_type=mimetype)
+
+        app.router.add_get("/~cache/{tail:.*}", cache_handler)
+
         # OAuth handlers
         async def github_auth_handler(request):
             """Initiate GitHub OAuth flow"""
             if "auth" not in g_config or "github" not in g_config["auth"]:
-                return web.json_response({"error": "GitHub OAuth not configured"}, status=500)
+                return web.json_response(create_error_response("GitHub OAuth not configured"), status=500)
 
             auth_config = g_config["auth"]["github"]
             client_id = auth_config.get("client_id", "")
@@ -1822,12 +3466,15 @@ def main():
 
             # Expand environment variables
             if client_id.startswith("$"):
-                client_id = os.environ.get(client_id[1:], "")
+                client_id = client_id[1:]
             if redirect_uri.startswith("$"):
-                redirect_uri = os.environ.get(redirect_uri[1:], "")
+                redirect_uri = redirect_uri[1:]
+
+            client_id = os.getenv(client_id, client_id)
+            redirect_uri = os.getenv(redirect_uri, redirect_uri)
 
             if not client_id:
-                return web.json_response({"error": "GitHub client_id not configured"}, status=500)
+                return web.json_response(create_error_response("GitHub client_id not configured"), status=500)
 
             # Generate CSRF state token
             state = secrets.token_urlsafe(32)
@@ -1857,7 +3504,9 @@ def main():
 
             # Expand environment variables
             if restrict_to.startswith("$"):
-                restrict_to = os.environ.get(restrict_to[1:], "")
+                restrict_to = restrict_to[1:]
+
+            restrict_to = os.getenv(restrict_to, None if restrict_to == "GITHUB_USERS" else restrict_to)
 
             # If restrict_to is configured, validate the user
             if restrict_to:
@@ -1878,6 +3527,14 @@ def main():
             code = request.query.get("code")
             state = request.query.get("state")
 
+            # Handle malformed URLs where query params are appended with & instead of ?
+            if not code and "tail" in request.match_info:
+                tail = request.match_info["tail"]
+                if tail.startswith("&"):
+                    params = parse_qs(tail[1:])
+                    code = params.get("code", [None])[0]
+                    state = params.get("state", [None])[0]
+
             if not code or not state:
                 return web.Response(text="Missing code or state parameter", status=400)
 
@@ -1888,7 +3545,7 @@ def main():
             g_oauth_states.pop(state)
 
             if "auth" not in g_config or "github" not in g_config["auth"]:
-                return web.json_response({"error": "GitHub OAuth not configured"}, status=500)
+                return web.json_response(create_error_response("GitHub OAuth not configured"), status=500)
 
             auth_config = g_config["auth"]["github"]
             client_id = auth_config.get("client_id", "")
@@ -1897,14 +3554,18 @@ def main():
 
             # Expand environment variables
             if client_id.startswith("$"):
-                client_id = os.environ.get(client_id[1:], "")
+                client_id = client_id[1:]
             if client_secret.startswith("$"):
-                client_secret = os.environ.get(client_secret[1:], "")
+                client_secret = client_secret[1:]
             if redirect_uri.startswith("$"):
-                redirect_uri = os.environ.get(redirect_uri[1:], "")
+                redirect_uri = redirect_uri[1:]
+
+            client_id = os.getenv(client_id, client_id)
+            client_secret = os.getenv(client_secret, client_secret)
+            redirect_uri = os.getenv(redirect_uri, redirect_uri)
 
             if not client_id or not client_secret:
-                return web.json_response({"error": "GitHub OAuth credentials not configured"}, status=500)
+                return web.json_response(create_error_response("GitHub OAuth credentials not configured"), status=500)
 
             # Exchange code for access token
             async with aiohttp.ClientSession() as session:
@@ -1923,7 +3584,7 @@ def main():
 
                     if not access_token:
                         error = token_response.get("error_description", "Failed to get access token")
-                        return web.Response(text=f"OAuth error: {error}", status=400)
+                        return web.json_response(create_error_response(f"OAuth error: {error}"), status=400)
 
                 # Fetch user info
                 user_url = "https://api.github.com/user"
@@ -1949,14 +3610,16 @@ def main():
             }
 
             # Redirect to UI with session token
-            return web.HTTPFound(f"/?session={session_token}")
+            response = web.HTTPFound(f"/?session={session_token}")
+            response.set_cookie("llms-token", session_token, httponly=True, path="/", max_age=86400)
+            return response
 
         async def session_handler(request):
             """Validate and return session info"""
-            session_token = request.query.get("session") or request.headers.get("X-Session-Token")
+            session_token = get_session_token(request)
 
             if not session_token or session_token not in g_sessions:
-                return web.json_response({"error": "Invalid or expired session"}, status=401)
+                return web.json_response(create_error_response("Invalid or expired session"), status=401)
 
             session_data = g_sessions[session_token]
 
@@ -1970,17 +3633,19 @@ def main():
 
         async def logout_handler(request):
             """End OAuth session"""
-            session_token = request.query.get("session") or request.headers.get("X-Session-Token")
+            session_token = get_session_token(request)
 
             if session_token and session_token in g_sessions:
                 del g_sessions[session_token]
 
-            return web.json_response({"success": True})
+            response = web.json_response({"success": True})
+            response.del_cookie("llms-token")
+            return response
 
         async def auth_handler(request):
             """Check authentication status and return user info"""
             # Check for OAuth session token
-            session_token = request.query.get("session") or request.headers.get("X-Session-Token")
+            session_token = get_session_token(request)
 
             if session_token and session_token in g_sessions:
                 session_data = g_sessions[session_token]
@@ -2010,13 +3675,12 @@ def main():
             #         })
 
             # Not authenticated - return error in expected format
-            return web.json_response(
-                {"responseStatus": {"errorCode": "Unauthorized", "message": "Not authenticated"}}, status=401
-            )
+            return web.json_response(g_app.error_auth_required, status=401)
 
         app.router.add_get("/auth", auth_handler)
         app.router.add_get("/auth/github", github_auth_handler)
         app.router.add_get("/auth/github/callback", github_callback_handler)
+        app.router.add_get("/auth/github/callback{tail:.*}", github_callback_handler)
         app.router.add_get("/auth/session", session_handler)
         app.router.add_post("/auth/logout", logout_handler)
 
@@ -2051,30 +3715,101 @@ def main():
 
         app.router.add_get("/ui/{path:.*}", ui_static, name="ui_static")
 
-        async def ui_config_handler(request):
-            with open(g_ui_path, encoding="utf-8") as f:
-                ui = json.load(f)
-                if "defaults" not in ui:
-                    ui["defaults"] = g_config["defaults"]
-                enabled, disabled = provider_status()
-                ui["status"] = {"all": list(g_config["providers"].keys()), "enabled": enabled, "disabled": disabled}
-                # Add auth configuration
-                ui["requiresAuth"] = auth_enabled
-                ui["authType"] = "oauth" if auth_enabled else "apikey"
-                return web.json_response(ui)
+        async def config_handler(request):
+            ret = {}
+            if "defaults" not in ret:
+                ret["defaults"] = g_config["defaults"]
+            enabled, disabled = provider_status()
+            ret["status"] = {"all": list(g_config["providers"].keys()), "enabled": enabled, "disabled": disabled}
+            # Add auth configuration
+            ret["requiresAuth"] = auth_enabled
+            ret["authType"] = "oauth" if auth_enabled else "apikey"
+            return web.json_response(ret)
 
-        app.router.add_get("/config", ui_config_handler)
+        app.router.add_get("/config", config_handler)
 
         async def not_found_handler(request):
             return web.Response(text="404: Not Found", status=404)
 
         app.router.add_get("/favicon.ico", not_found_handler)
 
+        # go through and register all g_app extensions
+        for handler in g_app.server_add_get:
+            handler_fn = handler[1]
+
+            async def managed_handler(request, handler_fn=handler_fn):
+                try:
+                    return await handler_fn(request)
+                except Exception as e:
+                    return web.json_response(to_error_response(e, stacktrace=g_verbose), status=500)
+
+            app.router.add_get(handler[0], managed_handler, **handler[2])
+        for handler in g_app.server_add_post:
+            handler_fn = handler[1]
+
+            async def managed_handler(request, handler_fn=handler_fn):
+                try:
+                    return await handler_fn(request)
+                except Exception as e:
+                    return web.json_response(to_error_response(e, stacktrace=g_verbose), status=500)
+
+            app.router.add_post(handler[0], managed_handler, **handler[2])
+        for handler in g_app.server_add_put:
+            handler_fn = handler[1]
+
+            async def managed_handler(request, handler_fn=handler_fn):
+                try:
+                    return await handler_fn(request)
+                except Exception as e:
+                    return web.json_response(to_error_response(e, stacktrace=g_verbose), status=500)
+
+            app.router.add_put(handler[0], managed_handler, **handler[2])
+        for handler in g_app.server_add_delete:
+            handler_fn = handler[1]
+
+            async def managed_handler(request, handler_fn=handler_fn):
+                try:
+                    return await handler_fn(request)
+                except Exception as e:
+                    return web.json_response(to_error_response(e, stacktrace=g_verbose), status=500)
+
+            app.router.add_delete(handler[0], managed_handler, **handler[2])
+        for handler in g_app.server_add_patch:
+            handler_fn = handler[1]
+
+            async def managed_handler(request, handler_fn=handler_fn):
+                try:
+                    return await handler_fn(request)
+                except Exception as e:
+                    return web.json_response(to_error_response(e, stacktrace=g_verbose), status=500)
+
+            app.router.add_patch(handler[0], managed_handler, **handler[2])
+
         # Serve index.html from root
         async def index_handler(request):
             index_content = read_resource_file_bytes("index.html")
-            if index_content is None:
-                raise web.HTTPNotFound
+
+            importmaps = {"imports": g_app.import_maps}
+            importmaps_script = '<script type="importmap">\n' + json.dumps(importmaps, indent=4) + "\n</script>"
+            index_content = index_content.replace(
+                b'<script type="importmap"></script>',
+                importmaps_script.encode("utf-8"),
+            )
+
+            if len(g_app.index_headers) > 0:
+                html_header = ""
+                for header in g_app.index_headers:
+                    html_header += header
+                # replace </head> with html_header
+                index_content = index_content.replace(b"</head>", html_header.encode("utf-8") + b"\n</head>")
+
+            if len(g_app.index_footers) > 0:
+                html_footer = ""
+                for footer in g_app.index_footers:
+                    html_footer += footer
+                # replace </body> with html_footer
+                index_content = index_content.replace(b"</body>", html_footer.encode("utf-8") + b"\n</body>")
+
             return web.Response(body=index_content, content_type="text/html")
 
         app.router.add_get("/", index_handler)
@@ -2086,13 +3821,15 @@ def main():
         async def start_background_tasks(app):
             """Start background tasks when the app starts"""
             # Start watching config files in the background
-            asyncio.create_task(watch_config_files(g_config_path, g_ui_path))
+            asyncio.create_task(watch_config_files(g_config_path, home_providers_path))
 
         app.on_startup.append(start_background_tasks)
 
+        # go through and register all g_app extensions
+
         print(f"Starting server on port {port}...")
         web.run_app(app, host="0.0.0.0", port=port, print=_log)
-        exit(0)
+        g_app.exit(0)
 
     if cli_args.enable is not None:
         if cli_args.enable.endswith(","):
@@ -2109,7 +3846,7 @@ def main():
 
         for provider in enable_providers:
             if provider not in g_config["providers"]:
-                print(f"Provider {provider} not found")
+                print(f"Provider '{provider}' not found")
                 print(f"Available providers: {', '.join(g_config['providers'].keys())}")
                 exit(1)
             if provider in g_config["providers"]:
@@ -2122,7 +3859,7 @@ def main():
         print_status()
         if len(msgs) > 0:
             print("\n" + "\n".join(msgs))
-        exit(0)
+        g_app.exit(0)
 
     if cli_args.disable is not None:
         if cli_args.disable.endswith(","):
@@ -2145,26 +3882,26 @@ def main():
             print(f"\nDisabled provider {provider}")
 
         print_status()
-        exit(0)
+        g_app.exit(0)
 
     if cli_args.default is not None:
         default_model = cli_args.default
-        all_models = get_models()
-        if default_model not in all_models:
+        provider_model = get_provider_model(default_model)
+        if provider_model is None:
             print(f"Model {default_model} not found")
-            print(f"Available models: {', '.join(all_models)}")
             exit(1)
         default_text = g_config["defaults"]["text"]
         default_text["model"] = default_model
         save_config(g_config)
         print(f"\nDefault model set to: {default_model}")
-        exit(0)
+        g_app.exit(0)
 
     if (
         cli_args.chat is not None
         or cli_args.image is not None
         or cli_args.audio is not None
         or cli_args.file is not None
+        or cli_args.out is not None
         or len(extra_args) > 0
     ):
         try:
@@ -2175,6 +3912,12 @@ def main():
                 chat = g_config["defaults"]["audio"]
             elif cli_args.file is not None:
                 chat = g_config["defaults"]["file"]
+            elif cli_args.out is not None:
+                template = f"out:{cli_args.out}"
+                if template not in g_config["defaults"]:
+                    print(f"Template for output modality '{cli_args.out}' not found")
+                    exit(1)
+                chat = g_config["defaults"][template]
             if cli_args.chat is not None:
                 chat_path = os.path.join(os.path.dirname(__file__), cli_args.chat)
                 if not os.path.exists(chat_path):
@@ -2191,6 +3934,9 @@ def main():
 
             if len(extra_args) > 0:
                 prompt = " ".join(extra_args)
+                if not chat["messages"] or len(chat["messages"]) == 0:
+                    chat["messages"] = [{"role": "user", "content": [{"type": "text", "text": ""}]}]
+
                 # replace content of last message if exists, else add
                 last_msg = chat["messages"][-1] if "messages" in chat else None
                 if last_msg and last_msg["role"] == "user":
@@ -2208,19 +3954,31 @@ def main():
 
             asyncio.run(
                 cli_chat(
-                    chat, image=cli_args.image, audio=cli_args.audio, file=cli_args.file, args=args, raw=cli_args.raw
+                    chat,
+                    tools=cli_args.tools,
+                    image=cli_args.image,
+                    audio=cli_args.audio,
+                    file=cli_args.file,
+                    args=args,
+                    raw=cli_args.raw,
                 )
             )
-            exit(0)
+            g_app.exit(0)
         except Exception as e:
             print(f"{cli_args.logprefix}Error: {e}")
             if cli_args.verbose:
                 traceback.print_exc()
-            exit(1)
+            g_app.exit(1)
+
+    handled = run_extension_cli()
 
-    # show usage from ArgumentParser
-    parser.print_help()
+    if not handled:
+        # show usage from ArgumentParser
+        parser.print_help()
+        g_app.exit(0)
 
 
 if __name__ == "__main__":
+    if MOCK or DEBUG:
+        print(f"MOCK={MOCK} or DEBUG={DEBUG}")
     main()