llms-py 3.0.0b2__py3-none-any.whl → 3.0.0b4__py3-none-any.whl

llms/providers/anthropic.py

@@ -0,0 +1,189 @@
+import json
+import time
+
+import aiohttp
+
+
+def install(ctx):
+    from llms.main import OpenAiCompatible
+
+    class AnthropicProvider(OpenAiCompatible):
+        sdk = "@ai-sdk/anthropic"
+
+        def __init__(self, **kwargs):
+            if "api" not in kwargs:
+                kwargs["api"] = "https://api.anthropic.com/v1"
+            super().__init__(**kwargs)
+
+            # Anthropic uses x-api-key header instead of Authorization
+            if self.api_key:
+                self.headers = self.headers.copy()
+                if "Authorization" in self.headers:
+                    del self.headers["Authorization"]
+                self.headers["x-api-key"] = self.api_key
+
+            if "anthropic-version" not in self.headers:
+                self.headers = self.headers.copy()
+                self.headers["anthropic-version"] = "2023-06-01"
+            self.chat_url = f"{self.api}/messages"
+
+        async def chat(self, chat):
+            chat["model"] = self.provider_model(chat["model"]) or chat["model"]
+
+            chat = await self.process_chat(chat, provider_id=self.id)
+
+            # Transform OpenAI format to Anthropic format
+            anthropic_request = {
+                "model": chat["model"],
+                "messages": [],
+            }
+
+            # Extract system message (Anthropic uses top-level 'system' parameter)
+            system_messages = []
+            for message in chat.get("messages", []):
+                if message.get("role") == "system":
+                    content = message.get("content", "")
+                    if isinstance(content, str):
+                        system_messages.append(content)
+                    elif isinstance(content, list):
+                        for item in content:
+                            if item.get("type") == "text":
+                                system_messages.append(item.get("text", ""))
+
+            if system_messages:
+                anthropic_request["system"] = "\n".join(system_messages)
+
+            # Transform messages (exclude system messages)
+            for message in chat.get("messages", []):
+                if message.get("role") == "system":
+                    continue
+
+                anthropic_message = {"role": message.get("role"), "content": []}
+
+                content = message.get("content", "")
+                if isinstance(content, str):
+                    anthropic_message["content"] = content
+                elif isinstance(content, list):
+                    for item in content:
+                        if item.get("type") == "text":
+                            anthropic_message["content"].append({"type": "text", "text": item.get("text", "")})
+                        elif item.get("type") == "image_url" and "image_url" in item:
+                            # Transform OpenAI image_url format to Anthropic format
+                            image_url = item["image_url"].get("url", "")
+                            if image_url.startswith("data:"):
+                                # Extract media type and base64 data
+                                parts = image_url.split(";base64,", 1)
+                                if len(parts) == 2:
+                                    media_type = parts[0].replace("data:", "")
+                                    base64_data = parts[1]
+                                    anthropic_message["content"].append(
+                                        {
+                                            "type": "image",
+                                            "source": {"type": "base64", "media_type": media_type, "data": base64_data},
+                                        }
+                                    )
+
+                anthropic_request["messages"].append(anthropic_message)
+
+            # Handle max_tokens (required by Anthropic, uses max_tokens not max_completion_tokens)
+            if "max_completion_tokens" in chat:
+                anthropic_request["max_tokens"] = chat["max_completion_tokens"]
+            elif "max_tokens" in chat:
+                anthropic_request["max_tokens"] = chat["max_tokens"]
+            else:
+                # Anthropic requires max_tokens, set a default
+                anthropic_request["max_tokens"] = 4096
+
+            # Copy other supported parameters
+            if "temperature" in chat:
+                anthropic_request["temperature"] = chat["temperature"]
+            if "top_p" in chat:
+                anthropic_request["top_p"] = chat["top_p"]
+            if "top_k" in chat:
+                anthropic_request["top_k"] = chat["top_k"]
+            if "stop" in chat:
+                anthropic_request["stop_sequences"] = chat["stop"] if isinstance(chat["stop"], list) else [chat["stop"]]
+            if "stream" in chat:
+                anthropic_request["stream"] = chat["stream"]
+            if "tools" in chat:
+                anthropic_request["tools"] = chat["tools"]
+            if "tool_choice" in chat:
+                anthropic_request["tool_choice"] = chat["tool_choice"]
+
+            ctx.log(f"POST {self.chat_url}")
+            ctx.log(json.dumps(anthropic_request, indent=2))
+
+            async with aiohttp.ClientSession() as session:
+                started_at = time.time()
+                async with session.post(
+                    self.chat_url,
+                    headers=self.headers,
+                    data=json.dumps(anthropic_request),
+                    timeout=aiohttp.ClientTimeout(total=120),
+                ) as response:
+                    return ctx.log_json(self.to_response(await self.response_json(response), chat, started_at))
+
+        def to_response(self, response, chat, started_at):
+            """Convert Anthropic response format to OpenAI-compatible format."""
+            # Transform Anthropic response to OpenAI format
+            ret = {
+                "id": response.get("id", ""),
+                "object": "chat.completion",
+                "created": int(started_at),
+                "model": response.get("model", ""),
+                "choices": [],
+                "usage": {},
+            }
+
+            # Transform content blocks to message content
+            content_parts = []
+            thinking_parts = []
+
+            for block in response.get("content", []):
+                if block.get("type") == "text":
+                    content_parts.append(block.get("text", ""))
+                elif block.get("type") == "thinking":
+                    # Store thinking blocks separately (some models include reasoning)
+                    thinking_parts.append(block.get("thinking", ""))
+
+            # Combine all text content
+            message_content = "\n".join(content_parts) if content_parts else ""
+
+            # Create the choice object
+            choice = {
+                "index": 0,
+                "message": {"role": "assistant", "content": message_content},
+                "finish_reason": response.get("stop_reason", "stop"),
+            }
+
+            # Add thinking as metadata if present
+            if thinking_parts:
+                choice["message"]["thinking"] = "\n".join(thinking_parts)
+
+            ret["choices"].append(choice)
+
+            # Transform usage
+            if "usage" in response:
+                usage = response["usage"]
+                ret["usage"] = {
+                    "prompt_tokens": usage.get("input_tokens", 0),
+                    "completion_tokens": usage.get("output_tokens", 0),
+                    "total_tokens": usage.get("input_tokens", 0) + usage.get("output_tokens", 0),
+                }
+
+            # Add metadata
+            if "metadata" not in ret:
+                ret["metadata"] = {}
+            ret["metadata"]["duration"] = int((time.time() - started_at) * 1000)
+
+            if chat is not None and "model" in chat:
+                cost = self.model_cost(chat["model"])
+                if cost and "input" in cost and "output" in cost:
+                    ret["metadata"]["pricing"] = f"{cost['input']}/{cost['output']}"
+
+            return ret
+
+    ctx.add_provider(AnthropicProvider)
+
+
+__install__ = install

llms/providers/chutes.py

@@ -0,0 +1,152 @@
+import json
+import mimetypes
+import time
+
+import aiohttp
+
+
+def install(ctx):
+    from llms.main import GeneratorBase
+
+    class ChutesImage(GeneratorBase):
+        sdk = "chutes/image"
+
+        def __init__(self, **kwargs):
+            super().__init__(**kwargs)
+            self.width = int(kwargs.get("width", 1024))
+            self.height = int(kwargs.get("height", 1024))
+            self.cfg_scale = float(kwargs.get("cfg_scale", 7.5))
+            self.steps = int(kwargs.get("steps", 50))
+            self.negative_prompt = kwargs.get("negative_prompt", "blur, distortion, low quality")
+            self.gen_url = kwargs.get("api", "https://image.chutes.ai/generate")
+            self.model_resolutions = {
+                "chutes-hidream": {
+                    "1:1": "1024x1024",
+                    "9:16": "768x1360",
+                    "16:9": "1360x768",
+                    "3:4": "880x1168",
+                    "4:3": "1168x880",
+                    "2:3": "832x1248",
+                    "3:2": "1248x832",
+                }
+            }
+            self.model_sizes = ["chutes-hunyuan-image-3"]
+            self.model_negative_prompt = [
+                "chroma",
+                "qwen-image-edit-2509",
+                "JuggernautXL-Ragnarok",
+                "JuggernautXL",
+                "Animij",
+                "iLustMix",
+            ]
+
+        async def chat(self, chat, provider=None):
+            headers = {"Authorization": f"Bearer {self.api_key}"}
+            if provider is not None:
+                headers["Authorization"] = f"Bearer {provider.api_key}"
+                chat["model"] = provider.provider_model(chat["model"]) or chat["model"]
+
+            aspect_ratio = "1:1"
+            if "messages" in chat and len(chat["messages"]) > 0:
+                aspect_ratio = chat["messages"][0].get("aspect_ratio", "1:1")
+            cfg_scale = self.cfg_scale
+            if chat["model"] == "chutes-z-image-turbo":
+                cfg_scale = min(self.cfg_scale, 5)
+            payload = {
+                "model": chat["model"],
+                "prompt": ctx.last_user_prompt(chat),
+                "guidance_scale": cfg_scale,
+                "width": self.width,
+                "height": self.height,
+                "num_inference_steps": self.steps,
+            }
+            if chat["model"] in self.model_negative_prompt:
+                payload["negative_prompt"] = self.negative_prompt
+
+            image_config = chat.get("image_config", {})
+            aspect_ratio = image_config.get("aspect_ratio")
+            if aspect_ratio:
+                dimension = ctx.app.aspect_ratios.get(aspect_ratio)
+                if dimension:
+                    width, height = dimension.split("×")
+                    payload["width"] = int(width)
+                    payload["height"] = int(height)
+
+            if chat["model"] in self.model_resolutions:
+                # if models use resolution, remove width and height
+                del payload["width"]
+                del payload["height"]
+                resolution = self.model_resolutions[chat["model"]][aspect_ratio]
+                payload["resolution"] = resolution
+            elif chat["model"] in self.model_sizes:
+                del payload["width"]
+                del payload["height"]
+                payload["size"] = aspect_ratio
+
+            gen_url = self.gen_url
+            if chat["model"].startswith("chutes-"):
+                model = payload["model"]
+                gen_url = f"https://{model}.chutes.ai/generate"
+                del payload["model"]
+
+            ctx.log(f"POST {gen_url}")
+            ctx.log(json.dumps(payload, indent=2))
+            async with aiohttp.ClientSession() as session, session.post(
+                gen_url, headers=headers, json=payload
+            ) as response:
+                if response.status < 300:
+                    image_data = await response.read()
+                    content_type = response.headers.get("Content-Type")
+                    if content_type:
+                        ext = mimetypes.guess_extension(content_type)
+                        if ext:
+                            ext = ext.lstrip(".")  # remove leading dot
+                    if not ext:
+                        ext = "png"
+
+                    relative_url, info = ctx.save_image_to_cache(
+                        image_data,
+                        f"{chat['model']}.{ext}",
+                        {
+                            "model": chat["model"],
+                            "prompt": ctx.last_user_prompt(chat),
+                            "width": self.width,
+                            "height": self.height,
+                            "cfg_scale": self.cfg_scale,
+                            "steps": self.steps,
+                        },
+                    )
+                    return {
+                        "choices": [
+                            {
+                                "message": {
+                                    "role": "assistant",
+                                    "content": self.default_content,
+                                    "images": [
+                                        {
+                                            "type": "image_url",
+                                            "image_url": {
+                                                "url": relative_url,
+                                            },
+                                        }
+                                    ],
+                                }
+                            }
+                        ],
+                        "created": int(time.time()),
+                    }
+                else:
+                    text = await response.text()
+                    try:
+                        data = json.loads(text)
+                        ctx.log(data)
+                        if "detail" in data:
+                            raise Exception(data["detail"])
+                    except json.JSONDecodeError:
+                        pass
+                    raise Exception(f"Failed to generate image {response.status}")
+
+    ctx.add_provider(ChutesImage)
+
+
+__install__ = install

llms/providers/google.py

@@ -0,0 +1,306 @@
+import json
+import time
+
+import aiohttp
+
+# class GoogleOpenAiProvider(OpenAiCompatible):
+#     sdk = "google-openai-compatible"
+
+#     def __init__(self, api_key, **kwargs):
+#         super().__init__(api="https://generativelanguage.googleapis.com", api_key=api_key, **kwargs)
+#         self.chat_url = "https://generativelanguage.googleapis.com/v1beta/chat/completions"
+
+
+def install(ctx):
+    from llms.main import OpenAiCompatible
+
+    def gemini_chat_summary(gemini_chat):
+        """Summarize Gemini chat completion request for logging. Replace inline_data with size of content only"""
+        clone = json.loads(json.dumps(gemini_chat))
+        for content in clone["contents"]:
+            for part in content["parts"]:
+                if "inline_data" in part:
+                    data = part["inline_data"]["data"]
+                    part["inline_data"]["data"] = f"({len(data)})"
+        return json.dumps(clone, indent=2)
+
+    def gemini_response_summary(obj):
+        to = {}
+        for k, v in obj.items():
+            if k == "candidates":
+                candidates = []
+                for candidate in v:
+                    c = {}
+                    for ck, cv in candidate.items():
+                        if ck == "content":
+                            content = {}
+                            for content_k, content_v in cv.items():
+                                if content_k == "parts":
+                                    parts = []
+                                    for part in content_v:
+                                        p = {}
+                                        for pk, pv in part.items():
+                                            if pk == "inlineData":
+                                                p[pk] = {
+                                                    "mimeType": pv.get("mimeType"),
+                                                    "data": f"({len(pv.get('data'))})",
+                                                }
+                                            else:
+                                                p[pk] = pv
+                                        parts.append(p)
+                                    content[content_k] = parts
+                                else:
+                                    content[content_k] = content_v
+                            c[ck] = content
+                        else:
+                            c[ck] = cv
+                    candidates.append(c)
+                to[k] = candidates
+            else:
+                to[k] = v
+        return to
+
+    class GoogleProvider(OpenAiCompatible):
+        sdk = "@ai-sdk/google"
+
+        def __init__(self, **kwargs):
+            new_kwargs = {"api": "https://generativelanguage.googleapis.com", **kwargs}
+            super().__init__(**new_kwargs)
+            self.safety_settings = kwargs.get("safety_settings")
+            self.thinking_config = kwargs.get("thinking_config")
+            self.tools = kwargs.get("tools")
+            self.curl = kwargs.get("curl")
+            self.headers = kwargs.get("headers", {"Content-Type": "application/json"})
+            # Google fails when using Authorization header, use query string param instead
+            if "Authorization" in self.headers:
+                del self.headers["Authorization"]
+
+        async def chat(self, chat):
+            chat["model"] = self.provider_model(chat["model"]) or chat["model"]
+
+            chat = await self.process_chat(chat)
+            generation_config = {}
+
+            # Filter out system messages and convert to proper Gemini format
+            contents = []
+            system_prompt = None
+
+            async with aiohttp.ClientSession() as session:
+                for message in chat["messages"]:
+                    if message["role"] == "system":
+                        content = message["content"]
+                        if isinstance(content, list):
+                            for item in content:
+                                if "text" in item:
+                                    system_prompt = item["text"]
+                                    break
+                        elif isinstance(content, str):
+                            system_prompt = content
+                    elif "content" in message:
+                        if isinstance(message["content"], list):
+                            parts = []
+                            for item in message["content"]:
+                                if "type" in item:
+                                    if item["type"] == "image_url" and "image_url" in item:
+                                        image_url = item["image_url"]
+                                        if "url" not in image_url:
+                                            continue
+                                        url = image_url["url"]
+                                        if not url.startswith("data:"):
+                                            raise Exception("Image was not downloaded: " + url)
+                                        # Extract mime type from data uri
+                                        mimetype = url.split(";", 1)[0].split(":", 1)[1] if ";" in url else "image/png"
+                                        base64_data = url.split(",", 1)[1]
+                                        parts.append({"inline_data": {"mime_type": mimetype, "data": base64_data}})
+                                    elif item["type"] == "input_audio" and "input_audio" in item:
+                                        input_audio = item["input_audio"]
+                                        if "data" not in input_audio:
+                                            continue
+                                        data = input_audio["data"]
+                                        format = input_audio["format"]
+                                        mimetype = f"audio/{format}"
+                                        parts.append({"inline_data": {"mime_type": mimetype, "data": data}})
+                                    elif item["type"] == "file" and "file" in item:
+                                        file = item["file"]
+                                        if "file_data" not in file:
+                                            continue
+                                        data = file["file_data"]
+                                        if not data.startswith("data:"):
+                                            raise (Exception("File was not downloaded: " + data))
+                                        # Extract mime type from data uri
+                                        mimetype = (
+                                            data.split(";", 1)[0].split(":", 1)[1]
+                                            if ";" in data
+                                            else "application/octet-stream"
+                                        )
+                                        base64_data = data.split(",", 1)[1]
+                                        parts.append({"inline_data": {"mime_type": mimetype, "data": base64_data}})
+                                if "text" in item:
+                                    text = item["text"]
+                                    parts.append({"text": text})
+                            if len(parts) > 0:
+                                contents.append(
+                                    {
+                                        "role": message["role"]
+                                        if "role" in message and message["role"] == "user"
+                                        else "model",
+                                        "parts": parts,
+                                    }
+                                )
+                        else:
+                            content = message["content"]
+                            contents.append(
+                                {
+                                    "role": message["role"]
+                                    if "role" in message and message["role"] == "user"
+                                    else "model",
+                                    "parts": [{"text": content}],
+                                }
+                            )
+
+                gemini_chat = {
+                    "contents": contents,
+                }
+
+                if self.safety_settings:
+                    gemini_chat["safetySettings"] = self.safety_settings
+
+                # Add system instruction if present
+                if system_prompt is not None:
+                    gemini_chat["systemInstruction"] = {"parts": [{"text": system_prompt}]}
+
+                if "max_completion_tokens" in chat:
+                    generation_config["maxOutputTokens"] = chat["max_completion_tokens"]
+                if "stop" in chat:
+                    generation_config["stopSequences"] = [chat["stop"]]
+                if "temperature" in chat:
+                    generation_config["temperature"] = chat["temperature"]
+                if "top_p" in chat:
+                    generation_config["topP"] = chat["top_p"]
+                if "top_logprobs" in chat:
+                    generation_config["topK"] = chat["top_logprobs"]
+
+                if "thinkingConfig" in chat:
+                    generation_config["thinkingConfig"] = chat["thinkingConfig"]
+                elif self.thinking_config:
+                    generation_config["thinkingConfig"] = self.thinking_config
+
+                if len(generation_config) > 0:
+                    gemini_chat["generationConfig"] = generation_config
+
+                if "tools" in chat:
+                    gemini_chat["tools"] = chat["tools"]
+                elif self.tools:
+                    gemini_chat["tools"] = self.tools.copy()
+
+                if "modalities" in chat:
+                    generation_config["responseModalities"] = [modality.upper() for modality in chat["modalities"]]
+                    if "image_config" in chat:
+                        # delete thinkingConfig
+                        del generation_config["thinkingConfig"]
+                        config_map = {
+                            "aspect_ratio": "aspectRatio",
+                            "image_size": "imageSize",
+                        }
+                        generation_config["imageConfig"] = {
+                            config_map[k]: v for k, v in chat["image_config"].items() if k in config_map
+                        }
+
+                started_at = int(time.time() * 1000)
+                gemini_chat_url = f"https://generativelanguage.googleapis.com/v1beta/models/{chat['model']}:generateContent?key={self.api_key}"
+
+                ctx.log(f"POST {gemini_chat_url}")
+                ctx.log(gemini_chat_summary(gemini_chat))
+                started_at = time.time()
+
+                if ctx.MOCK and "modalities" in chat:
+                    print("Mocking Google Gemini Image")
+                    with open(f"{ctx.MOCK_DIR}/gemini-image.json") as f:
+                        obj = json.load(f)
+                else:
+                    async with session.post(
+                        gemini_chat_url,
+                        headers=self.headers,
+                        data=json.dumps(gemini_chat),
+                        timeout=aiohttp.ClientTimeout(total=120),
+                    ) as res:
+                        obj = await self.response_json(res)
+
+                if "error" in obj:
+                    ctx.log(f"Error: {obj['error']}")
+                    raise Exception(obj["error"]["message"])
+
+                if ctx.debug:
+                    ctx.dbg(json.dumps(gemini_response_summary(obj), indent=2))
+
+                response = {
+                    "id": f"chatcmpl-{started_at}",
+                    "created": started_at,
+                    "model": obj.get("modelVersion", chat["model"]),
+                }
+                choices = []
+                for i, candidate in enumerate(obj["candidates"]):
+                    role = "assistant"
+                    if "content" in candidate and "role" in candidate["content"]:
+                        role = "assistant" if candidate["content"]["role"] == "model" else candidate["content"]["role"]
+
+                    # Safely extract content from all text parts
+                    content = ""
+                    reasoning = ""
+                    images = []
+                    if "content" in candidate and "parts" in candidate["content"]:
+                        text_parts = []
+                        reasoning_parts = []
+                        for part in candidate["content"]["parts"]:
+                            if "text" in part:
+                                if "thought" in part and part["thought"]:
+                                    reasoning_parts.append(part["text"])
+                                else:
+                                    text_parts.append(part["text"])
+                            if "inlineData" in part:
+                                inline_data = part["inlineData"]
+                                mime_type = inline_data.get("mimeType", "image/png")
+                                ext = mime_type.split("/")[1]
+                                base64_data = inline_data["data"]
+                                filename = f"{chat['model'].split('/')[-1]}-{len(images)}.{ext}"
+                                relative_url, info = ctx.save_image_to_cache(base64_data, filename, {})
+                                images.append(
+                                    {
+                                        "type": "image_url",
+                                        "index": len(images),
+                                        "image_url": {
+                                            "url": relative_url,
+                                        },
+                                    }
+                                )
+                        content = " ".join(text_parts)
+                        reasoning = " ".join(reasoning_parts)
+
+                    choice = {
+                        "index": i,
+                        "finish_reason": candidate.get("finishReason", "stop"),
+                        "message": {
+                            "role": role,
+                            "content": content,
+                        },
+                    }
+                    if reasoning:
+                        choice["message"]["reasoning"] = reasoning
+                    if len(images) > 0:
+                        choice["message"]["images"] = images
+                    choices.append(choice)
+                response["choices"] = choices
+                if "usageMetadata" in obj:
+                    usage = obj["usageMetadata"]
+                    response["usage"] = {
+                        "completion_tokens": usage["candidatesTokenCount"],
+                        "total_tokens": usage["totalTokenCount"],
+                        "prompt_tokens": usage["promptTokenCount"],
+                    }
+
+                return ctx.log_json(self.to_response(response, chat, started_at))
+
+    ctx.add_provider(GoogleProvider)
+
+
+__install__ = install