llms-py 3.0.0b6__py3-none-any.whl → 3.0.0b8__py3-none-any.whl

llms/extensions/providers/__init__.py

@@ -0,0 +1,18 @@
+from .anthropic import install_anthropic
+from .chutes import install_chutes
+from .google import install_google
+from .nvidia import install_nvidia
+from .openai import install_openai
+from .openrouter import install_openrouter
+
+
+def install(ctx):
+    install_anthropic(ctx)
+    install_chutes(ctx)
+    install_google(ctx)
+    install_openai(ctx)
+    install_openrouter(ctx)
+    install_nvidia(ctx)
+
+
+__install__ = install

llms/{providers → extensions/providers}/anthropic.py

@@ -4,7 +4,7 @@ import time
 import aiohttp
 
 
-def install(ctx):
+def install_anthropic(ctx):
     from llms.main import OpenAiCompatible
 
     class AnthropicProvider(OpenAiCompatible):
@@ -58,6 +58,23 @@ def install(ctx):
                 if message.get("role") == "system":
                     continue
 
+                if message.get("role") == "tool":
+                    # Convert OpenAI tool response to Anthropic tool_result
+                    tool_call_id = message.get("tool_call_id")
+                    content = ctx.to_content(message.get("content", ""))
+                    if not isinstance(content, (str, list)):
+                        content = str(content)
+
+                    tool_result = {"type": "tool_result", "tool_use_id": tool_call_id, "content": content}
+
+                    # Anthropic requires tool results to be in a user message
+                    # Check if the last message was a user message, if so append to it
+                    if anthropic_request["messages"] and anthropic_request["messages"][-1]["role"] == "user":
+                        anthropic_request["messages"][-1]["content"].append(tool_result)
+                    else:
+                        anthropic_request["messages"].append({"role": "user", "content": [tool_result]})
+                    continue
+
                 anthropic_message = {"role": message.get("role"), "content": []}
 
                 content = message.get("content", "")
@@ -106,7 +123,18 @@ def install(ctx):
             if "stream" in chat:
                 anthropic_request["stream"] = chat["stream"]
             if "tools" in chat:
-                anthropic_request["tools"] = chat["tools"]
+                anthropic_tools = []
+                for tool in chat["tools"]:
+                    if tool.get("type") == "function":
+                        function = tool.get("function", {})
+                        anthropic_tool = {
+                            "name": function.get("name"),
+                            "description": function.get("description"),
+                            "input_schema": function.get("parameters"),
+                        }
+                        anthropic_tools.append(anthropic_tool)
+                if anthropic_tools:
+                    anthropic_request["tools"] = anthropic_tools
             if "tool_choice" in chat:
                 anthropic_request["tool_choice"] = chat["tool_choice"]
 
@@ -138,6 +166,7 @@ def install(ctx):
             # Transform content blocks to message content
             content_parts = []
             thinking_parts = []
+            tool_calls = []
 
             for block in response.get("content", []):
                 if block.get("type") == "text":
@@ -145,6 +174,16 @@ def install(ctx):
                 elif block.get("type") == "thinking":
                     # Store thinking blocks separately (some models include reasoning)
                     thinking_parts.append(block.get("thinking", ""))
+                elif block.get("type") == "tool_use":
+                    tool_call = {
+                        "id": block.get("id"),
+                        "type": "function",
+                        "function": {
+                            "name": block.get("name"),
+                            "arguments": json.dumps(block.get("input", {})),
+                        },
+                    }
+                    tool_calls.append(tool_call)
 
             # Combine all text content
             message_content = "\n".join(content_parts) if content_parts else ""
@@ -160,6 +199,10 @@ def install(ctx):
             if thinking_parts:
                 choice["message"]["thinking"] = "\n".join(thinking_parts)
 
+            # Add tool_calls if present
+            if tool_calls:
+                choice["message"]["tool_calls"] = tool_calls
+
             ret["choices"].append(choice)
 
             # Transform usage
@@ -184,6 +227,3 @@ def install(ctx):
             return ret
 
     ctx.add_provider(AnthropicProvider)
-
-
-__install__ = install

llms/{providers → extensions/providers}/chutes.py

@@ -5,7 +5,7 @@ import time
 import aiohttp
 
 
-def install(ctx):
+def install_chutes(ctx):
     from llms.main import GeneratorBase
 
     class ChutesImage(GeneratorBase):
@@ -50,15 +50,18 @@ def install(ctx):
             if "messages" in chat and len(chat["messages"]) > 0:
                 aspect_ratio = chat["messages"][0].get("aspect_ratio", "1:1")
             cfg_scale = self.cfg_scale
+            steps = self.steps
+            width = self.width
+            height = self.height
             if chat["model"] == "chutes-z-image-turbo":
                 cfg_scale = min(self.cfg_scale, 5)
             payload = {
                 "model": chat["model"],
                 "prompt": ctx.last_user_prompt(chat),
                 "guidance_scale": cfg_scale,
-                "width": self.width,
-                "height": self.height,
-                "num_inference_steps": self.steps,
+                "width": width,
+                "height": height,
+                "num_inference_steps": steps,
             }
             if chat["model"] in self.model_negative_prompt:
                 payload["negative_prompt"] = self.negative_prompt
@@ -68,9 +71,10 @@ def install(ctx):
             if aspect_ratio:
                 dimension = ctx.app.aspect_ratios.get(aspect_ratio)
                 if dimension:
-                    width, height = dimension.split("×")
-                    payload["width"] = int(width)
-                    payload["height"] = int(height)
+                    w, h = dimension.split("×")
+                    width, height = int(w), int(h)
+                    payload["width"] = width
+                    payload["height"] = height
 
             if chat["model"] in self.model_resolutions:
                 # if models use resolution, remove width and height
@@ -107,14 +111,16 @@ def install(ctx):
                     relative_url, info = ctx.save_image_to_cache(
                         image_data,
                         f"{chat['model']}.{ext}",
-                        {
-                            "model": chat["model"],
-                            "prompt": ctx.last_user_prompt(chat),
-                            "width": self.width,
-                            "height": self.height,
-                            "cfg_scale": self.cfg_scale,
-                            "steps": self.steps,
-                        },
+                        ctx.to_file_info(
+                            chat,
+                            {
+                                "aspect_ratio": aspect_ratio,
+                                "width": width,
+                                "height": height,
+                                "cfg_scale": cfg_scale,
+                                "steps": steps,
+                            },
+                        ),
                     )
                     return {
                         "choices": [
@@ -147,6 +153,3 @@ def install(ctx):
                     raise Exception(f"Failed to generate image {response.status}")
 
     ctx.add_provider(ChutesImage)
-
-
-__install__ = install

llms/{providers → extensions/providers}/google.py

@@ -1,5 +1,8 @@
+import base64
+import io
 import json
 import time
+import wave
 
 import aiohttp
 
@@ -11,7 +14,7 @@ import aiohttp
 #         self.chat_url = "https://generativelanguage.googleapis.com/v1beta/chat/completions"
 
 
-def install(ctx):
+def install_google(ctx):
     from llms.main import OpenAiCompatible
 
     def gemini_chat_summary(gemini_chat):
@@ -68,6 +71,7 @@ def install(ctx):
             super().__init__(**new_kwargs)
             self.safety_settings = kwargs.get("safety_settings")
             self.thinking_config = kwargs.get("thinking_config")
+            self.speech_config = kwargs.get("speech_config")
             self.tools = kwargs.get("tools")
             self.curl = kwargs.get("curl")
             self.headers = kwargs.get("headers", {"Content-Type": "application/json"})
@@ -189,13 +193,15 @@ def install(ctx):
                     gemini_chat["generationConfig"] = generation_config
 
                 if "tools" in chat:
-                    gemini_chat["tools"] = chat["tools"]
+                    # gemini_chat["tools"] = chat["tools"]
+                    ctx.log("Error: tools not supported in Gemini")
                 elif self.tools:
-                    gemini_chat["tools"] = self.tools.copy()
+                    # gemini_chat["tools"] = self.tools.copy()
+                    ctx.log("Error: tools not supported in Gemini")
 
                 if "modalities" in chat:
                     generation_config["responseModalities"] = [modality.upper() for modality in chat["modalities"]]
-                    if "image_config" in chat:
+                    if "image" in chat["modalities"] and "image_config" in chat:
                         # delete thinkingConfig
                         del generation_config["thinkingConfig"]
                         config_map = {
@@ -205,6 +211,11 @@ def install(ctx):
                         generation_config["imageConfig"] = {
                             config_map[k]: v for k, v in chat["image_config"].items() if k in config_map
                         }
+                    if "audio" in chat["modalities"] and self.speech_config:
+                        del generation_config["thinkingConfig"]
+                        generation_config["speechConfig"] = self.speech_config.copy()
+                        # Currently Google Audio Models only accept AUDIO
+                        generation_config["responseModalities"] = ["AUDIO"]
 
                 started_at = int(time.time() * 1000)
                 gemini_chat_url = f"https://generativelanguage.googleapis.com/v1beta/models/{chat['model']}:generateContent?key={self.api_key}"
@@ -218,13 +229,22 @@ def install(ctx):
                     with open(f"{ctx.MOCK_DIR}/gemini-image.json") as f:
                         obj = json.load(f)
                 else:
-                    async with session.post(
-                        gemini_chat_url,
-                        headers=self.headers,
-                        data=json.dumps(gemini_chat),
-                        timeout=aiohttp.ClientTimeout(total=120),
-                    ) as res:
-                        obj = await self.response_json(res)
+                    try:
+                        async with session.post(
+                            gemini_chat_url,
+                            headers=self.headers,
+                            data=json.dumps(gemini_chat),
+                            timeout=aiohttp.ClientTimeout(total=120),
+                        ) as res:
+                            obj = await self.response_json(res)
+                    except Exception as e:
+                        ctx.log(f"Error: {res.status} {res.reason}: {e}")
+                        text = await res.text()
+                        try:
+                            obj = json.loads(text)
+                        except:
+                            ctx.log(text)
+                            raise e
 
                 if "error" in obj:
                     ctx.log(f"Error: {obj['error']}")
@@ -233,6 +253,18 @@ def install(ctx):
                 if ctx.debug:
                     ctx.dbg(json.dumps(gemini_response_summary(obj), indent=2))
 
+                # calculate cost per generation
+                cost = None
+                token_costs = obj.get("metadata", {}).get("pricing", "")
+                if token_costs:
+                    input_price, output_price = token_costs.split("/")
+                    input_per_token = float(input_price) / 1000000
+                    output_per_token = float(output_price) / 1000000
+                    if "usageMetadata" in obj:
+                        input_tokens = obj["usageMetadata"].get("promptTokenCount", 0)
+                        output_tokens = obj["usageMetadata"].get("candidatesTokenCount", 0)
+                        cost = (input_per_token * input_tokens) + (output_per_token * output_tokens)
+
                 response = {
                     "id": f"chatcmpl-{started_at}",
                     "created": started_at,
@@ -248,6 +280,7 @@ def install(ctx):
                     content = ""
                     reasoning = ""
                     images = []
+                    audios = []
                     if "content" in candidate and "parts" in candidate["content"]:
                         text_parts = []
                         reasoning_parts = []
@@ -260,19 +293,59 @@ def install(ctx):
                             if "inlineData" in part:
                                 inline_data = part["inlineData"]
                                 mime_type = inline_data.get("mimeType", "image/png")
-                                ext = mime_type.split("/")[1]
-                                base64_data = inline_data["data"]
-                                filename = f"{chat['model'].split('/')[-1]}-{len(images)}.{ext}"
-                                relative_url, info = ctx.save_image_to_cache(base64_data, filename, {})
-                                images.append(
-                                    {
-                                        "type": "image_url",
-                                        "index": len(images),
-                                        "image_url": {
-                                            "url": relative_url,
-                                        },
-                                    }
-                                )
+                                if mime_type.startswith("image"):
+                                    ext = mime_type.split("/")[1]
+                                    base64_data = inline_data["data"]
+                                    filename = f"{chat['model'].split('/')[-1]}-{len(images)}.{ext}"
+                                    ctx.log(f"inlineData {len(base64_data)} {mime_type} {filename}")
+                                    relative_url, info = ctx.save_image_to_cache(
+                                        base64_data,
+                                        filename,
+                                        ctx.to_file_info(chat, {"cost": cost}),
+                                    )
+                                    images.append(
+                                        {
+                                            "type": "image_url",
+                                            "index": len(images),
+                                            "image_url": {
+                                                "url": relative_url,
+                                            },
+                                        }
+                                    )
+                                elif mime_type.startswith("audio"):
+                                    # mime_type audio/L16;codec=pcm;rate=24000
+                                    base64_data = inline_data["data"]
+
+                                    pcm = base64.b64decode(base64_data)
+                                    # Convert PCM to WAV
+                                    wav_io = io.BytesIO()
+                                    with wave.open(wav_io, "wb") as wf:
+                                        wf.setnchannels(1)
+                                        wf.setsampwidth(2)
+                                        wf.setframerate(24000)
+                                        wf.writeframes(pcm)
+                                    wav_data = wav_io.getvalue()
+
+                                    ext = mime_type.split("/")[1].split(";")[0]
+                                    pcm_filename = f"{chat['model'].split('/')[-1]}-{len(audios)}.{ext}"
+                                    filename = pcm_filename.replace(f".{ext}", ".wav")
+                                    ctx.log(f"inlineData {len(base64_data)} {mime_type} {filename}")
+
+                                    relative_url, info = ctx.save_bytes_to_cache(
+                                        wav_data,
+                                        filename,
+                                        ctx.to_file_info(chat, {"cost": cost}),
+                                    )
+
+                                    audios.append(
+                                        {
+                                            "type": "audio_url",
+                                            "index": len(audios),
+                                            "audio_url": {
+                                                "url": relative_url,
+                                            },
+                                        }
+                                    )
                         content = " ".join(text_parts)
                         reasoning = " ".join(reasoning_parts)
 
@@ -288,6 +361,8 @@ def install(ctx):
                         choice["message"]["reasoning"] = reasoning
                     if len(images) > 0:
                         choice["message"]["images"] = images
+                    if len(audios) > 0:
+                        choice["message"]["audios"] = audios
                     choices.append(choice)
                 response["choices"] = choices
                 if "usageMetadata" in obj:
@@ -301,6 +376,3 @@ def install(ctx):
                 return ctx.log_json(self.to_response(response, chat, started_at))
 
     ctx.add_provider(GoogleProvider)
-
-
-__install__ = install

llms/{providers → extensions/providers}/nvidia.py

@@ -4,7 +4,7 @@ import time
 import aiohttp
 
 
-def install(ctx):
+def install_nvidia(ctx):
     from llms.main import GeneratorBase
 
     class NvidiaGenAi(GeneratorBase):
@@ -29,10 +29,11 @@ def install(ctx):
                         last_model = "/" in chat["model"] and chat["model"].split("/")[-1] or chat["model"]
                         filename = f"{last_model}_{seed}.png"
 
-                    image_info = {
-                        "seed": seed,
-                    }
-                    relative_url, info = ctx.save_image_to_cache(base64, filename, image_info)
+                    relative_url, info = ctx.save_image_to_cache(
+                        base64,
+                        filename,
+                        ctx.to_file_info(chat, {"seed": seed}),
+                    )
                     return {
                         "choices": [
                             {
@@ -102,6 +103,3 @@ def install(ctx):
                     return self.to_response(await self.response_json(response), chat, started_at)
 
     ctx.add_provider(NvidiaGenAi)
-
-
-__install__ = install

llms/{providers → extensions/providers}/openai.py

@@ -6,7 +6,7 @@ import time
 import aiohttp
 
 
-def install(ctx):
+def install_openai(ctx):
     from llms.main import GeneratorBase, OpenAiCompatible
 
     class OpenAiProvider(OpenAiCompatible):
@@ -83,10 +83,7 @@ def install(ctx):
                         relative_url, info = ctx.save_image_to_cache(
                             image_data,
                             f"{chat['model']}-{i}.{ext}",
-                            {
-                                "model": chat["model"],
-                                "prompt": ctx.last_user_prompt(chat),
-                            },
+                            ctx.to_file_info(chat),
                         )
                         images.append(
                             {
@@ -156,4 +153,4 @@ def install(ctx):
     ctx.add_provider(OpenAiGenerator)
 
 
-__install__ = install
+__install__ = install_openai

llms/{providers → extensions/providers}/openrouter.py

@@ -4,7 +4,7 @@ import time
 import aiohttp
 
 
-def install(ctx):
+def install_openrouter(ctx):
     from llms.main import GeneratorBase
 
     # https://openrouter.ai/docs/guides/overview/multimodal/image-generation
@@ -16,6 +16,9 @@ def install(ctx):
 
         def to_response(self, response, chat, started_at):
             # go through all image responses and save them to cache
+            cost = None
+            if "usage" in response and "cost" in response["usage"]:
+                cost = response["usage"]["cost"]
             for choice in response["choices"]:
                 if "message" in choice and "images" in choice["message"]:
                     for image in choice["message"]["images"]:
@@ -29,11 +32,9 @@ def install(ctx):
                                 base64_data = parts[1]
                                 model = chat["model"].split("/")[-1]
                                 filename = f"{model}-{choice['index']}.{ext}"
-                                info = {
-                                    "model": model,
-                                    "prompt": ctx.last_user_prompt(chat),
-                                }
-                                relative_url, info = ctx.save_image_to_cache(base64_data, filename, info)
+                                relative_url, info = ctx.save_image_to_cache(
+                                    base64_data, filename, ctx.to_file_info(chat, {"cost": cost})
+                                )
                                 image["image_url"]["url"] = relative_url
 
             return response
@@ -50,11 +51,13 @@ def install(ctx):
                 return ctx.log_json(self.to_response(json.loads(text), chat, started_at))
             else:
                 chat_url = provider.chat_url
+                # remove tools
+                chat.pop("tools", None)
                 chat = await self.process_chat(chat, provider_id=self.id)
                 ctx.log(f"POST {chat_url}")
                 ctx.log(provider.chat_summary(chat))
                 # remove metadata if any (conflicts with some providers, e.g. Z.ai)
-                chat.pop("metadata", None)
+                metadata = chat.pop("metadata", None)
 
                 async with aiohttp.ClientSession() as session, session.post(
                     chat_url,
@@ -62,9 +65,8 @@ def install(ctx):
                     data=json.dumps(chat),
                     timeout=aiohttp.ClientTimeout(total=300),
                 ) as response:
+                    if metadata:
+                        chat["metadata"] = metadata
                     return ctx.log_json(self.to_response(await self.response_json(response), chat, started_at))
 
     ctx.add_provider(OpenRouterGenerator)
-
-
-__install__ = install

llms/extensions/system_prompts/__init__.py

@@ -0,0 +1,45 @@
+import json
+import os
+
+from aiohttp import web
+
+default_prompts = [
+    {"name": "Helpful Assistant", "prompt": "You are a helpful assistant."},
+]
+
+
+# runs after providers are configured but before server is run
+def install(ctx):
+    # helper to get user or default prompts
+    def get_user_prompts(request):
+        candidate_paths = []
+        # check if user is signed in
+        username = ctx.get_username(request)
+        if username:
+            # if signed in (Github OAuth), return the prompts for this user if exists
+            candidate_paths.append(os.path.join(ctx.get_user_path(username), "system_prompts", "prompts.json"))
+        # return default prompts for all users if exists
+        candidate_paths.append(os.path.join(ctx.get_user_path(), "system_prompts", "prompts.json"))
+        # otherwise return the default prompts from this repo
+        candidate_paths.append(os.path.join(ctx.path, "ui", "prompts.json"))
+
+        # iterate all candidate paths and when exists return its json
+        for path in candidate_paths:
+            if os.path.exists(path):
+                with open(path, encoding="utf-8") as f:
+                    txt = f.read()
+                    return json.loads(txt)
+        return default_prompts
+
+    # API Handler to get prompts
+    async def get_prompts(request):
+        prompts_json = get_user_prompts(request)
+        return web.json_response(prompts_json)
+
+    ctx.add_get("prompts.json", get_prompts)
+
+
+# register install extension handler
+__install__ = install
+
+__order__ = -10