llms-py 2.0.20__py3-none-any.whl → 3.0.18__py3-none-any.whl

llms/extensions/providers/__init__.py

@@ -0,0 +1,22 @@
+from .anthropic import install_anthropic
+from .cerebras import install_cerebras
+from .chutes import install_chutes
+from .google import install_google
+from .nvidia import install_nvidia
+from .openai import install_openai
+from .openrouter import install_openrouter
+from .zai import install_zai
+
+
+def install(ctx):
+    install_anthropic(ctx)
+    install_cerebras(ctx)
+    install_chutes(ctx)
+    install_google(ctx)
+    install_nvidia(ctx)
+    install_openai(ctx)
+    install_openrouter(ctx)
+    install_zai(ctx)
+
+
+__install__ = install

llms/extensions/providers/anthropic.py

@@ -0,0 +1,260 @@
+import json
+import time
+
+import aiohttp
+
+
+def install_anthropic(ctx):
+    from llms.main import OpenAiCompatible
+
+    class AnthropicProvider(OpenAiCompatible):
+        sdk = "@ai-sdk/anthropic"
+
+        def __init__(self, **kwargs):
+            if "api" not in kwargs:
+                kwargs["api"] = "https://api.anthropic.com/v1"
+            super().__init__(**kwargs)
+
+            # Anthropic uses x-api-key header instead of Authorization
+            if self.api_key:
+                self.headers = self.headers.copy()
+                if "Authorization" in self.headers:
+                    del self.headers["Authorization"]
+                self.headers["x-api-key"] = self.api_key
+
+            if "anthropic-version" not in self.headers:
+                self.headers = self.headers.copy()
+                self.headers["anthropic-version"] = "2023-06-01"
+            self.chat_url = f"{self.api}/messages"
+
+        async def chat(self, chat, context=None):
+            chat["model"] = self.provider_model(chat["model"]) or chat["model"]
+
+            chat = await self.process_chat(chat, provider_id=self.id)
+
+            # Transform OpenAI format to Anthropic format
+            anthropic_request = {
+                "model": chat["model"],
+                "messages": [],
+            }
+
+            # Extract system message (Anthropic uses top-level 'system' parameter)
+            system_messages = []
+            for message in chat.get("messages", []):
+                if message.get("role") == "system":
+                    content = message.get("content", "")
+                    if isinstance(content, str):
+                        system_messages.append(content)
+                    elif isinstance(content, list):
+                        for item in content:
+                            if item.get("type") == "text":
+                                system_messages.append(item.get("text", ""))
+
+            if system_messages:
+                anthropic_request["system"] = "\n".join(system_messages)
+
+            # Transform messages (exclude system messages)
+            for message in chat.get("messages", []):
+                if message.get("role") == "system":
+                    continue
+
+                if message.get("role") == "tool":
+                    # Convert OpenAI tool response to Anthropic tool_result
+                    tool_call_id = message.get("tool_call_id")
+                    content = ctx.to_content(message.get("content", ""))
+                    if not isinstance(content, (str, list)):
+                        content = str(content)
+
+                    tool_result = {"type": "tool_result", "tool_use_id": tool_call_id, "content": content}
+
+                    # Anthropic requires tool results to be in a user message
+                    # Check if the last message was a user message, if so append to it
+                    if anthropic_request["messages"] and anthropic_request["messages"][-1]["role"] == "user":
+                        anthropic_request["messages"][-1]["content"].append(tool_result)
+                    else:
+                        anthropic_request["messages"].append({"role": "user", "content": [tool_result]})
+                    continue
+
+                anthropic_message = {"role": message.get("role"), "content": []}
+
+                # Handle interleaved thinking (must always be a list if present)
+                if "thinking" in message and message["thinking"]:
+                    anthropic_message["content"].append({"type": "thinking", "thinking": message["thinking"]})
+
+                content = message.get("content", "")
+                if isinstance(content, str):
+                    if anthropic_message["content"] or message.get("tool_calls"):
+                        # If we have thinking or tools, we must use blocks for text
+                        if content:
+                            anthropic_message["content"].append({"type": "text", "text": content})
+                    else:
+                        anthropic_message["content"] = content
+                elif isinstance(content, list):
+                    for item in content:
+                        if item.get("type") == "text":
+                            anthropic_message["content"].append({"type": "text", "text": item.get("text", "")})
+                        elif item.get("type") == "image_url" and "image_url" in item:
+                            # Transform OpenAI image_url format to Anthropic format
+                            image_url = item["image_url"].get("url", "")
+                            if image_url.startswith("data:"):
+                                # Extract media type and base64 data
+                                parts = image_url.split(";base64,", 1)
+                                if len(parts) == 2:
+                                    media_type = parts[0].replace("data:", "")
+                                    base64_data = parts[1]
+                                    anthropic_message["content"].append(
+                                        {
+                                            "type": "image",
+                                            "source": {"type": "base64", "media_type": media_type, "data": base64_data},
+                                        }
+                                    )
+
+                # Handle tool_calls
+                if "tool_calls" in message and message["tool_calls"]:
+                    # specific check for content being a string and not empty, because we might have converted it above
+                    if isinstance(anthropic_message["content"], str):
+                        anthropic_message["content"] = []
+                        if content:
+                            anthropic_message["content"].append({"type": "text", "text": content})
+
+                    for tool_call in message["tool_calls"]:
+                        function = tool_call.get("function", {})
+                        tool_use = {
+                            "type": "tool_use",
+                            "id": tool_call.get("id"),
+                            "name": function.get("name"),
+                            "input": json.loads(function.get("arguments", "{}")),
+                        }
+                        anthropic_message["content"].append(tool_use)
+
+                anthropic_request["messages"].append(anthropic_message)
+
+            # Handle max_tokens (required by Anthropic, uses max_tokens not max_completion_tokens)
+            if "max_completion_tokens" in chat:
+                anthropic_request["max_tokens"] = chat["max_completion_tokens"]
+            elif "max_tokens" in chat:
+                anthropic_request["max_tokens"] = chat["max_tokens"]
+            else:
+                # Anthropic requires max_tokens, set a default
+                anthropic_request["max_tokens"] = 4096
+
+            # Copy other supported parameters
+            if "temperature" in chat:
+                anthropic_request["temperature"] = chat["temperature"]
+            if "top_p" in chat:
+                anthropic_request["top_p"] = chat["top_p"]
+            if "top_k" in chat:
+                anthropic_request["top_k"] = chat["top_k"]
+            if "stop" in chat:
+                anthropic_request["stop_sequences"] = chat["stop"] if isinstance(chat["stop"], list) else [chat["stop"]]
+            if "stream" in chat:
+                anthropic_request["stream"] = chat["stream"]
+            if "tools" in chat:
+                anthropic_tools = []
+                for tool in chat["tools"]:
+                    if tool.get("type") == "function":
+                        function = tool.get("function", {})
+                        anthropic_tool = {
+                            "name": function.get("name"),
+                            "description": function.get("description"),
+                            "input_schema": function.get("parameters"),
+                        }
+                        anthropic_tools.append(anthropic_tool)
+                if anthropic_tools:
+                    anthropic_request["tools"] = anthropic_tools
+            if "tool_choice" in chat:
+                anthropic_request["tool_choice"] = chat["tool_choice"]
+
+            ctx.log(f"POST {self.chat_url}")
+            ctx.log(json.dumps(anthropic_request, indent=2))
+
+            async with aiohttp.ClientSession() as session:
+                started_at = time.time()
+                async with session.post(
+                    self.chat_url,
+                    headers=self.headers,
+                    data=json.dumps(anthropic_request),
+                    timeout=aiohttp.ClientTimeout(total=120),
+                ) as response:
+                    return ctx.log_json(
+                        self.to_response(await self.response_json(response), chat, started_at, context=context)
+                    )
+
+        def to_response(self, response, chat, started_at, context=None):
+            """Convert Anthropic response format to OpenAI-compatible format."""
+            if context is not None:
+                context["providerResponse"] = response
+            # Transform Anthropic response to OpenAI format
+            ret = {
+                "id": response.get("id", ""),
+                "object": "chat.completion",
+                "created": int(started_at),
+                "model": response.get("model", ""),
+                "choices": [],
+                "usage": {},
+            }
+
+            # Transform content blocks to message content
+            content_parts = []
+            thinking_parts = []
+            tool_calls = []
+
+            for block in response.get("content", []):
+                if block.get("type") == "text":
+                    content_parts.append(block.get("text", ""))
+                elif block.get("type") == "thinking":
+                    # Store thinking blocks separately (some models include reasoning)
+                    thinking_parts.append(block.get("thinking", ""))
+                elif block.get("type") == "tool_use":
+                    tool_call = {
+                        "id": block.get("id"),
+                        "type": "function",
+                        "function": {
+                            "name": block.get("name"),
+                            "arguments": json.dumps(block.get("input", {})),
+                        },
+                    }
+                    tool_calls.append(tool_call)
+
+            # Combine all text content
+            message_content = "\n".join(content_parts) if content_parts else ""
+
+            # Create the choice object
+            choice = {
+                "index": 0,
+                "message": {"role": "assistant", "content": message_content},
+                "finish_reason": response.get("stop_reason", "stop"),
+            }
+
+            # Add thinking as metadata if present
+            if thinking_parts:
+                choice["message"]["thinking"] = "\n".join(thinking_parts)
+
+            # Add tool_calls if present
+            if tool_calls:
+                choice["message"]["tool_calls"] = tool_calls
+
+            ret["choices"].append(choice)
+
+            # Transform usage
+            if "usage" in response:
+                usage = response["usage"]
+                ret["usage"] = {
+                    "prompt_tokens": usage.get("input_tokens", 0),
+                    "completion_tokens": usage.get("output_tokens", 0),
+                    "total_tokens": usage.get("input_tokens", 0) + usage.get("output_tokens", 0),
+                }
+
+            # Add metadata
+            if "metadata" not in ret:
+                ret["metadata"] = {}
+            ret["metadata"]["duration"] = int(time.time() - started_at)
+
+            if chat is not None and "model" in chat:
+                cost = self.model_cost(chat["model"])
+                if cost and "input" in cost and "output" in cost:
+                    ret["metadata"]["pricing"] = f"{cost['input']}/{cost['output']}"
+
+            return ret
+
+    ctx.add_provider(AnthropicProvider)

llms/extensions/providers/cerebras.py

@@ -0,0 +1,36 @@
+def install_cerebras(ctx):
+    from llms.main import OpenAiCompatible
+
+    class CerebrasProvider(OpenAiCompatible):
+        sdk = "@ai-sdk/cerebras"
+
+        def __init__(self, **kwargs):
+            if "api" not in kwargs:
+                kwargs["api"] = "https://api.cerebras.ai/v1"
+            super().__init__(**kwargs)
+
+        async def chat(self, chat, context=None):
+            # Cerebras only supports string content for text-only models
+            clean_chat = chat.copy()
+            clean_chat["messages"] = []
+            for msg in chat.get("messages", []):
+                new_msg = msg.copy()
+                content = msg.get("content")
+                if isinstance(content, list):
+                    # Check if text only
+                    is_text_only = True
+                    text_parts = []
+                    for part in content:
+                        if part.get("type") != "text":
+                            is_text_only = False
+                            break
+                        text_parts.append(part.get("text", ""))
+
+                    if is_text_only:
+                        new_msg["content"] = "".join(text_parts)
+                clean_chat["messages"].append(new_msg)
+
+            clean_chat.pop("modalities", None)
+            return await super().chat(clean_chat, context)
+
+    ctx.add_provider(CerebrasProvider)

llms/extensions/providers/chutes.py

@@ -0,0 +1,153 @@
+import json
+import mimetypes
+import time
+
+import aiohttp
+
+
+def install_chutes(ctx):
+    from llms.main import GeneratorBase
+
+    class ChutesImage(GeneratorBase):
+        sdk = "chutes/image"
+
+        def __init__(self, **kwargs):
+            super().__init__(**kwargs)
+            self.width = int(kwargs.get("width", 1024))
+            self.height = int(kwargs.get("height", 1024))
+            self.cfg_scale = float(kwargs.get("cfg_scale", 7.5))
+            self.steps = int(kwargs.get("steps", 50))
+            self.negative_prompt = kwargs.get("negative_prompt", "blur, distortion, low quality")
+            self.gen_url = kwargs.get("api", "https://image.chutes.ai/generate")
+            self.model_resolutions = {
+                "chutes-hidream": {
+                    "1:1": "1024x1024",
+                    "9:16": "768x1360",
+                    "16:9": "1360x768",
+                    "3:4": "880x1168",
+                    "4:3": "1168x880",
+                    "2:3": "832x1248",
+                    "3:2": "1248x832",
+                }
+            }
+            self.model_sizes = ["chutes-hunyuan-image-3"]
+            self.model_negative_prompt = [
+                "chroma",
+                "qwen-image-edit-2509",
+                "JuggernautXL-Ragnarok",
+                "JuggernautXL",
+                "Animij",
+                "iLustMix",
+            ]
+
+        async def chat(self, chat, provider=None, context=None):
+            headers = {"Authorization": f"Bearer {self.api_key}"}
+            if provider is not None:
+                headers["Authorization"] = f"Bearer {provider.api_key}"
+                chat["model"] = provider.provider_model(chat["model"]) or chat["model"]
+
+            aspect_ratio = "1:1"
+            if "messages" in chat and len(chat["messages"]) > 0:
+                aspect_ratio = chat["messages"][0].get("aspect_ratio", "1:1")
+            cfg_scale = self.cfg_scale
+            steps = self.steps
+            width = self.width
+            height = self.height
+            if chat["model"] == "chutes-z-image-turbo":
+                cfg_scale = min(self.cfg_scale, 5)
+            payload = {
+                "model": chat["model"],
+                "prompt": ctx.last_user_prompt(chat),
+                "guidance_scale": cfg_scale,
+                "width": width,
+                "height": height,
+                "num_inference_steps": steps,
+            }
+            if chat["model"] in self.model_negative_prompt:
+                payload["negative_prompt"] = self.negative_prompt
+
+            aspect_ratio = ctx.chat_to_aspect_ratio(chat) or "1:1"
+            dimension = ctx.app.aspect_ratios.get(aspect_ratio)
+            if dimension:
+                w, h = dimension.split("×")
+                width, height = int(w), int(h)
+                payload["width"] = width
+                payload["height"] = height
+
+            if chat["model"] in self.model_resolutions:
+                # if models use resolution, remove width and height
+                del payload["width"]
+                del payload["height"]
+                resolution = self.model_resolutions[chat["model"]][aspect_ratio]
+                payload["resolution"] = resolution
+            elif chat["model"] in self.model_sizes:
+                del payload["width"]
+                del payload["height"]
+                payload["size"] = aspect_ratio
+
+            gen_url = self.gen_url
+            if chat["model"].startswith("chutes-"):
+                model = payload["model"]
+                gen_url = f"https://{model}.chutes.ai/generate"
+                del payload["model"]
+
+            ctx.log(f"POST {gen_url}")
+            ctx.log(json.dumps(payload, indent=2))
+            async with aiohttp.ClientSession() as session, session.post(
+                gen_url, headers=headers, json=payload
+            ) as response:
+                if response.status < 300:
+                    image_data = await response.read()
+                    content_type = response.headers.get("Content-Type")
+                    if content_type:
+                        ext = mimetypes.guess_extension(content_type)
+                        if ext:
+                            ext = ext.lstrip(".")  # remove leading dot
+                    if not ext:
+                        ext = "png"
+
+                    relative_url, info = ctx.save_image_to_cache(
+                        image_data,
+                        f"{chat['model']}.{ext}",
+                        ctx.to_file_info(
+                            chat,
+                            {
+                                "aspect_ratio": aspect_ratio,
+                                "width": width,
+                                "height": height,
+                                "cfg_scale": cfg_scale,
+                                "steps": steps,
+                            },
+                        ),
+                    )
+                    return {
+                        "choices": [
+                            {
+                                "message": {
+                                    "role": "assistant",
+                                    "content": self.default_content,
+                                    "images": [
+                                        {
+                                            "type": "image_url",
+                                            "image_url": {
+                                                "url": relative_url,
+                                            },
+                                        }
+                                    ],
+                                }
+                            }
+                        ],
+                        "created": int(time.time()),
+                    }
+                else:
+                    text = await response.text()
+                    try:
+                        data = json.loads(text)
+                        ctx.log(data)
+                        if "detail" in data:
+                            raise Exception(data["detail"])
+                    except json.JSONDecodeError:
+                        pass
+                    raise Exception(f"Failed to generate image {response.status}")
+
+    ctx.add_provider(ChutesImage)