lm-deluge 0.0.20__tar.gz → 0.0.22__tar.gz

{lm_deluge-0.0.20/src/lm_deluge.egg-info → lm_deluge-0.0.22}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lm_deluge
-Version: 0.0.20
+Version: 0.0.22
 Summary: Python utility for using LLM API models.
 Author-email: Benjamin Anderson <ben@trytaylor.ai>
 Requires-Python: >=3.10

{lm_deluge-0.0.20 → lm_deluge-0.0.22}/pyproject.toml

@@ -3,7 +3,7 @@ requires = ["setuptools", "wheel"]
 
 [project]
 name = "lm_deluge"
-version = "0.0.20"
+version = "0.0.22"
 authors = [{ name = "Benjamin Anderson", email = "ben@trytaylor.ai" }]
 description = "Python utility for using LLM API models."
 readme = "README.md"

{lm_deluge-0.0.20 → lm_deluge-0.0.22}/src/lm_deluge/api_requests/anthropic.py

@@ -4,8 +4,6 @@ import os
 from aiohttp import ClientResponse
 
 from lm_deluge.prompt import (
-    CachePattern,
-    Conversation,
     Message,
     Text,
     Thinking,
@@ -15,7 +13,6 @@ from lm_deluge.request_context import RequestContext
 from lm_deluge.tool import MCPServer, Tool
 from lm_deluge.usage import Usage
 
-from ..config import SamplingParams
 from ..models import APIModel
 from .base import APIRequestBase, APIResponse
 
@@ -30,12 +27,19 @@ def _add_beta(headers: dict, beta: str):
 
 def _build_anthropic_request(
     model: APIModel,
-    prompt: Conversation,
-    tools: list[Tool | dict | MCPServer] | None,
-    sampling_params: SamplingParams,
-    cache_pattern: CachePattern | None = None,
+    context: RequestContext,
+    # prompt: Conversation,
+    # tools: list[Tool | dict | MCPServer] | None,
+    # sampling_params: SamplingParams,
+    # cache_pattern: CachePattern | None = None,
 ):
+    prompt = context.prompt
+    cache_pattern = context.cache
+    tools = context.tools
+    sampling_params = context.sampling_params
     system_message, messages = prompt.to_anthropic(cache_pattern=cache_pattern)
+    if not system_message:
+        print("WARNING: system_message is None")
     base_headers = {
         "x-api-key": os.getenv(model.api_key_env_var),
         "anthropic-version": "2023-06-01",
@@ -69,6 +73,7 @@ def _build_anthropic_request(
             print("ignoring reasoning_effort for non-reasoning model")
     if system_message is not None:
         request_json["system"] = system_message
+
     if tools:
         mcp_servers = []
         tool_definitions = []
@@ -100,6 +105,7 @@ def _build_anthropic_request(
         if len(mcp_servers) > 0:
             request_json["mcp_servers"] = mcp_servers
 
+    # print("request json:", request_json)
     return request_json, base_headers
 
 
@@ -108,18 +114,15 @@ class AnthropicRequest(APIRequestBase):
         super().__init__(context=context)
 
         self.model = APIModel.from_registry(self.context.model_name)
-        self.url = f"{self.model.api_base}/messages"
 
         # Lock images as bytes if caching is enabled
         if self.context.cache is not None:
             self.context.prompt.lock_images_as_bytes()
 
+    async def build_request(self):
+        self.url = f"{self.model.api_base}/messages"
         self.request_json, base_headers = _build_anthropic_request(
-            self.model,
-            self.context.prompt,
-            self.context.tools,
-            self.context.sampling_params,
-            self.context.cache,
+            self.model, self.context
         )
         self.request_header = self.merge_headers(
             base_headers, exclude_patterns=["openai", "gemini", "mistral"]

{lm_deluge-0.0.20 → lm_deluge-0.0.22}/src/lm_deluge/api_requests/base.py

@@ -24,19 +24,21 @@ class APIRequestBase(ABC):
         self,
         context: RequestContext,
     ):
-        # If context is provided, use it; otherwise construct one from individual parameters
         self.context = context
 
         # Everything is now accessed through self.context - no copying!
         self.system_prompt = None
         self.result = []  # list of APIResponse objects from each attempt
 
-        # these should be set in the __init__ of the subclass
+        # these should be set in build_request of the subclass
         self.url = None
         self.request_header = None
         self.request_json = None
         self.region = None
 
+    async def build_request(self):
+        raise NotImplementedError("must be implemented in subclass")
+
     def increment_pbar(self):
         if self.context.status_tracker:
             self.context.status_tracker.increment_pbar()
@@ -76,7 +78,13 @@ class APIRequestBase(ABC):
 
     async def execute_once(self) -> APIResponse:
         """Send the HTTP request once and return the parsed APIResponse."""
+        await self.build_request()
         assert self.context.status_tracker
+        # try:
+        #     dumped = json.dumps(self.request_json)
+        # except Exception:
+        #     print("couldn't serialize request json")
+        #     print(self.request_json)
         try:
             self.context.status_tracker.total_requests += 1
             timeout = aiohttp.ClientTimeout(total=self.context.request_timeout)

{lm_deluge-0.0.20 → lm_deluge-0.0.22}/src/lm_deluge/api_requests/bedrock.py

@@ -12,8 +12,6 @@ except ImportError:
     )
 
 from lm_deluge.prompt import (
-    CachePattern,
-    Conversation,
     Message,
     Text,
     Thinking,
@@ -23,7 +21,6 @@ from lm_deluge.request_context import RequestContext
 from lm_deluge.tool import MCPServer, Tool
 from lm_deluge.usage import Usage
 
-from ..config import SamplingParams
 from ..models import APIModel
 from .base import APIRequestBase, APIResponse
 
@@ -38,13 +35,14 @@ def _add_beta(headers: dict, beta: str):
         headers["anthropic_beta"] = beta
 
 
-def _build_anthropic_bedrock_request(
+async def _build_anthropic_bedrock_request(
     model: APIModel,
-    prompt: Conversation,
-    tools: list[Tool | dict | MCPServer] | None,
-    sampling_params: SamplingParams,
-    cache_pattern: CachePattern | None = None,
+    context: RequestContext,
 ):
+    prompt = context.prompt
+    cache_pattern = context.cache
+    tools = context.tools
+    sampling_params = context.sampling_params
     system_message, messages = prompt.to_anthropic(cache_pattern=cache_pattern)
 
     # handle AWS auth
@@ -121,9 +119,10 @@ def _build_anthropic_bedrock_request(
                 elif tool["type"] == "code_execution_20250522":
                     _add_beta(base_headers, "code-execution-2025-05-22")
             elif isinstance(tool, MCPServer):
-                raise ValueError("bedrock doesn't support MCP connector right now")
-                # _add_beta(request_header, "mcp-client-2025-04-04")
-                # mcp_servers.append(tool.for_anthropic())
+                # Convert to individual tools locally (like OpenAI does)
+                individual_tools = await tool.to_tools()
+                for individual_tool in individual_tools:
+                    tool_definitions.append(individual_tool.dump_for("anthropic"))
 
         # Add cache control to last tool if tools_only caching is specified
         if cache_pattern == "tools_only" and tool_definitions:
@@ -141,21 +140,20 @@ class BedrockRequest(APIRequestBase):
         super().__init__(context=context)
 
         self.model = APIModel.from_registry(self.context.model_name)
+
+    async def build_request(self):
         self.url = f"{self.model.api_base}/messages"
 
         # Lock images as bytes if caching is enabled
         if self.context.cache is not None:
             self.context.prompt.lock_images_as_bytes()
 
-        self.request_json, base_headers, self.auth, self.url = (
-            _build_anthropic_bedrock_request(
-                self.model,
-                context.prompt,
-                context.tools,
-                context.sampling_params,
-                context.cache,
-            )
-        )
+        (
+            self.request_json,
+            base_headers,
+            self.auth,
+            self.url,
+        ) = await _build_anthropic_bedrock_request(self.model, self.context)
         self.request_header = self.merge_headers(
             base_headers, exclude_patterns=["anthropic", "openai", "gemini", "mistral"]
         )

{lm_deluge-0.0.20 → lm_deluge-0.0.22}/src/lm_deluge/api_requests/gemini.py

@@ -14,7 +14,7 @@ from ..usage import Usage
 from .base import APIRequestBase, APIResponse
 
 
-def _build_gemini_request(
+async def _build_gemini_request(
     model: APIModel,
     prompt: Conversation,
     tools: list[Tool] | None,
@@ -75,7 +75,8 @@ class GeminiRequest(APIRequestBase):
             )
 
         self.model = APIModel.from_registry(self.context.model_name)
-        # Gemini API endpoint format: https://generativelanguage.googleapis.com/v1beta/models/{model}:generateContent
+
+    async def build_request(self):
         self.url = f"{self.model.api_base}/models/{self.model.name}:generateContent"
         base_headers = {
             "Content-Type": "application/json",
@@ -92,7 +93,7 @@ class GeminiRequest(APIRequestBase):
             )
         self.url += f"?key={api_key}"
 
-        self.request_json = _build_gemini_request(
+        self.request_json = await _build_gemini_request(
             self.model,
             self.context.prompt,
             self.context.tools,

{lm_deluge-0.0.20 → lm_deluge-0.0.22}/src/lm_deluge/api_requests/mistral.py

@@ -21,6 +21,8 @@ class MistralRequest(APIRequestBase):
                 f"Cache parameter '{self.context.cache}' is only supported for Anthropic models, ignoring for {self.context.model_name}"
             )
         self.model = APIModel.from_registry(self.context.model_name)
+
+    async def build_request(self):
         self.url = f"{self.model.api_base}/chat/completions"
         base_headers = {
             "Authorization": f"Bearer {os.getenv(self.model.api_key_env_var)}"

{lm_deluge-0.0.20 → lm_deluge-0.0.22}/src/lm_deluge/api_requests/openai.py

@@ -1,6 +1,7 @@
 import json
 import os
 import warnings
+from types import SimpleNamespace
 
 import aiohttp
 from aiohttp import ClientResponse
@@ -15,12 +16,13 @@ from ..usage import Usage
 from .base import APIRequestBase, APIResponse
 
 
-def _build_oa_chat_request(
+async def _build_oa_chat_request(
     model: APIModel,
-    prompt: Conversation,
-    tools: list[Tool] | None,
-    sampling_params: SamplingParams,
+    context: RequestContext,
 ) -> dict:
+    prompt = context.prompt
+    sampling_params = context.sampling_params
+    tools = context.tools
     request_json = {
         "model": model.name,
         "messages": prompt.to_openai(),
@@ -48,19 +50,19 @@ def _build_oa_chat_request(
     if sampling_params.json_mode and model.supports_json:
         request_json["response_format"] = {"type": "json_object"}
     if tools:
-        request_json["tools"] = [tool.dump_for("openai-completions") for tool in tools]
+        request_tools = []
+        for tool in tools:
+            if isinstance(tool, Tool):
+                request_tools.append(tool.dump_for("openai-completions"))
+            elif isinstance(tool, MCPServer):
+                as_tools = await tool.to_tools()
+                request_tools.extend(
+                    [t.dump_for("openai-completions") for t in as_tools]
+                )
+        request_json["tools"] = request_tools
     return request_json
 
 
-def _build_oa_responses_request(
-    model: APIModel,
-    prompt: Conversation,
-    tools: list[Tool] | None,
-    sampling_params: SamplingParams,
-):
-    pass  # TODO: implement
-
-
 class OpenAIRequest(APIRequestBase):
     def __init__(self, context: RequestContext):
         # Pass context to parent, which will handle backwards compatibility
@@ -72,6 +74,8 @@ class OpenAIRequest(APIRequestBase):
                 f"Cache parameter '{self.context.cache}' is only supported for Anthropic models, ignoring for {self.context.model_name}"
             )
         self.model = APIModel.from_registry(self.context.model_name)
+
+    async def build_request(self):
         self.url = f"{self.model.api_base}/chat/completions"
         base_headers = {
             "Authorization": f"Bearer {os.getenv(self.model.api_key_env_var)}"
@@ -80,12 +84,7 @@ class OpenAIRequest(APIRequestBase):
             base_headers, exclude_patterns=["anthropic"]
         )
 
-        self.request_json = _build_oa_chat_request(
-            self.model,
-            self.context.prompt,
-            self.context.tools,
-            self.context.sampling_params,
-        )
+        self.request_json = await _build_oa_chat_request(self.model, self.context)
 
     async def handle_response(self, http_response: ClientResponse) -> APIResponse:
         is_error = False
@@ -187,6 +186,75 @@ class OpenAIRequest(APIRequestBase):
         )
 
 
+async def _build_oa_responses_request(
+    model: APIModel,
+    context: RequestContext,
+    # prompt: Conversation,
+    # tools: list[Tool] | None,
+    # sampling_params: SamplingParams,
+):
+    prompt = context.prompt
+    sampling_params = context.sampling_params
+    tools = context.tools
+    openai_responses_format = prompt.to_openai_responses()
+    request_json = {
+        "model": model.name,
+        "input": openai_responses_format["input"],
+        "temperature": sampling_params.temperature,
+        "top_p": sampling_params.top_p,
+    }
+    if sampling_params.max_new_tokens:
+        request_json["max_output_tokens"] = sampling_params.max_new_tokens
+
+    if model.reasoning_model:
+        if sampling_params.reasoning_effort in [None, "none"]:
+            # gemini models can switch reasoning off
+            if "gemini" in model.id:
+                sampling_params.reasoning_effort = "none"
+            else:
+                sampling_params.reasoning_effort = "low"
+        request_json["temperature"] = 1.0
+        request_json["top_p"] = 1.0
+        request_json["reasoning"] = {
+            "effort": sampling_params.reasoning_effort,
+            "summary": "auto",
+        }
+    else:
+        if sampling_params.reasoning_effort:
+            warnings.warn(
+                f"Ignoring reasoning_effort for non-reasoning model: {model.id}"
+            )
+
+    if sampling_params.json_mode and model.supports_json:
+        request_json["text"] = {"format": {"type": "json_object"}}
+
+    # Handle tools
+    request_tools = []
+    # Add regular function tools
+    for tool in tools or []:
+        if isinstance(tool, Tool):
+            request_tools.append(tool.dump_for("openai-responses"))
+        elif isinstance(tool, dict):
+            # if computer use, make sure model supports it
+            if tool["type"] == "computer_use_preview":
+                if model.name != "openai-computer-use-preview":
+                    raise ValueError(f"model {model.id} does not support computer use")
+                # have to use truncation
+                request_json["truncation"] = "auto"
+            request_tools.append(tool)  # allow passing dict
+        elif isinstance(tool, MCPServer):
+            if context.force_local_mcp:
+                as_tools = await tool.to_tools()
+                request_tools.extend([t.dump_for("openai-responses") for t in as_tools])
+            else:
+                request_tools.append(tool.for_openai_responses())
+
+    if request_tools:
+        request_json["tools"] = request_tools
+
+    return request_json
+
+
 class OpenAIResponsesRequest(APIRequestBase):
     def __init__(self, context: RequestContext):
         super().__init__(context)
@@ -196,73 +264,14 @@ class OpenAIResponsesRequest(APIRequestBase):
                 f"Cache parameter '{self.context.cache}' is only supported for Anthropic models, ignoring for {self.context.model_name}"
             )
         self.model = APIModel.from_registry(self.context.model_name)
+
+    async def build_request(self):
         self.url = f"{self.model.api_base}/responses"
         self.request_header = {
             "Authorization": f"Bearer {os.getenv(self.model.api_key_env_var)}"
         }
 
-        # Convert conversation to input format for Responses API
-        openai_responses_format = self.context.prompt.to_openai_responses()
-
-        self.request_json = {
-            "model": self.model.name,
-            "input": openai_responses_format["input"],
-            "temperature": self.context.sampling_params.temperature,
-            "top_p": self.context.sampling_params.top_p,
-        }
-
-        # Add max_output_tokens for responses API
-        if self.context.sampling_params.max_new_tokens:
-            self.request_json["max_output_tokens"] = (
-                self.context.sampling_params.max_new_tokens
-            )
-
-        if self.model.reasoning_model:
-            if self.context.sampling_params.reasoning_effort in [None, "none"]:
-                # gemini models can switch reasoning off
-                if "gemini" in self.model.id:
-                    self.context.sampling_params.reasoning_effort = (
-                        "none"  # expects string
-                    )
-                # openai models can only go down to "low"
-                else:
-                    self.context.sampling_params.reasoning_effort = "low"
-            self.request_json["temperature"] = 1.0
-            self.request_json["top_p"] = 1.0
-            self.request_json["reasoning"] = {
-                "effort": self.context.sampling_params.reasoning_effort
-            }
-        else:
-            if self.context.sampling_params.reasoning_effort:
-                warnings.warn(
-                    f"Ignoring reasoning_effort param for non-reasoning model: {self.context.model_name}"
-                )
-
-        if self.context.sampling_params.json_mode and self.model.supports_json:
-            self.request_json["text"] = {"format": {"type": "json_object"}}
-
-        # Handle tools
-        request_tools = []
-        if self.context.tools:
-            # Add regular function tools
-            for tool in self.context.tools:
-                if isinstance(tool, Tool):
-                    request_tools.append(tool.dump_for("openai-responses"))
-                elif isinstance(tool, dict):
-                    # if computer use, make sure model supports it
-                    if tool["type"] == "computer_use_preview":
-                        if self.context.model_name != "openai-computer-use-preview":
-                            raise ValueError(
-                                f"model {self.context.model_name} does not support computer use"
-                            )
-                        # have to use truncation
-                        self.request_json["truncation"] = "auto"
-                    request_tools.append(tool)  # allow passing dict
-                elif isinstance(tool, MCPServer):
-                    request_tools.append(tool.for_openai_responses())
-
-        if request_tools:
-            self.request_json["tools"] = request_tools
+        self.request_json = await _build_oa_responses_request(self.model, self.context)
 
     async def handle_response(self, http_response: ClientResponse) -> APIResponse:
         is_error = False
@@ -459,7 +468,11 @@ async def stream_chat(
         }
         request_header.update(filtered_extra)
 
-    request_json = _build_oa_chat_request(model, prompt, tools, sampling_params)
+    context = SimpleNamespace(
+        prompt=prompt, tools=tools, sampling_params=sampling_params
+    )
+
+    request_json = await _build_oa_chat_request(model, context)  # type: ignore
     request_json["stream"] = True
 
     async with aiohttp.ClientSession() as s:

{lm_deluge-0.0.20 → lm_deluge-0.0.22}/src/lm_deluge/api_requests/response.py

@@ -14,7 +14,7 @@ class APIResponse:
     # request information
     id: int  # should be unique to the request within a given prompt-processing call
     model_internal: str  # our internal model tag
-    prompt: Conversation
+    prompt: Conversation | dict
     sampling_params: SamplingParams
 
     # http response information
@@ -92,6 +92,8 @@ class APIResponse:
             print(
                 f"Warning: Completion provided without token counts for model {self.model_internal}."
             )
+        if isinstance(self.prompt, Conversation):
+            self.prompt = self.prompt.to_log()  # avoid keeping images in memory
 
     def to_dict(self):
         return {
@@ -99,7 +101,7 @@ class APIResponse:
             "model_internal": self.model_internal,
             "model_external": self.model_external,
             "region": self.region,
-            "prompt": self.prompt.to_log(),  # destroys image if present
+            "prompt": self.prompt,
             "sampling_params": self.sampling_params.__dict__,
             "status_code": self.status_code,
             "is_error": self.is_error,