lm-deluge 0.0.13__tar.gz → 0.0.14__tar.gz

{lm_deluge-0.0.13/src/lm_deluge.egg-info → lm_deluge-0.0.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lm_deluge
-Version: 0.0.13
+Version: 0.0.14
 Summary: Python utility for using LLM API models.
 Author-email: Benjamin Anderson <ben@trytaylor.ai>
 Requires-Python: >=3.10
@@ -30,6 +30,7 @@ Dynamic: license-file
 `lm-deluge` is a lightweight helper library for maxing out your rate limits with LLM providers. It provides the following:
 
 - **Unified client** – Send prompts to all relevant models with a single client.
+- **Files and Images** - Include images easily for multimodal models, and PDF files for models that support them (OpenAI and Anthropic).
 - **Massive concurrency with throttling** – Set `max_tokens_per_minute` and `max_requests_per_minute` and let it fly. The client will process as many requests as possible while respecting rate limits and retrying failures.
 - **Spray across models/providers** – Configure a client with multiple models from any provider(s), and sampling weights. The client samples a model for each request.
 - **Tool Use** – Unified API for defining tools for all providers, and creating tools automatically from python functions.
@@ -41,6 +42,8 @@ Dynamic: license-file
 
 **STREAMING IS NOT IN SCOPE.** There are plenty of packages that let you stream chat completions across providers. The sole purpose of this package is to do very fast batch inference using APIs. Sorry!
 
+**Update 06/02/2025:** I lied, it supports (very basic) streaming now via client.stream(...). It will print tokens as they arrive, then return an APIResponse at the end. More sophisticated streaming may or may not be implemented later, don't count on it.
+
 ## Installation
 
 ```bash

{lm_deluge-0.0.13 → lm_deluge-0.0.14}/README.md

@@ -3,6 +3,7 @@
 `lm-deluge` is a lightweight helper library for maxing out your rate limits with LLM providers. It provides the following:
 
 - **Unified client** – Send prompts to all relevant models with a single client.
+- **Files and Images** - Include images easily for multimodal models, and PDF files for models that support them (OpenAI and Anthropic).
 - **Massive concurrency with throttling** – Set `max_tokens_per_minute` and `max_requests_per_minute` and let it fly. The client will process as many requests as possible while respecting rate limits and retrying failures.
 - **Spray across models/providers** – Configure a client with multiple models from any provider(s), and sampling weights. The client samples a model for each request.
 - **Tool Use** – Unified API for defining tools for all providers, and creating tools automatically from python functions.
@@ -14,6 +15,8 @@
 
 **STREAMING IS NOT IN SCOPE.** There are plenty of packages that let you stream chat completions across providers. The sole purpose of this package is to do very fast batch inference using APIs. Sorry!
 
+**Update 06/02/2025:** I lied, it supports (very basic) streaming now via client.stream(...). It will print tokens as they arrive, then return an APIResponse at the end. More sophisticated streaming may or may not be implemented later, don't count on it.
+
 ## Installation
 
 ```bash

{lm_deluge-0.0.13 → lm_deluge-0.0.14}/pyproject.toml

@@ -3,7 +3,7 @@ requires = ["setuptools", "wheel"]
 
 [project]
 name = "lm_deluge"
-version = "0.0.13"
+version = "0.0.14"
 authors = [{ name = "Benjamin Anderson", email = "ben@trytaylor.ai" }]
 description = "Python utility for using LLM API models."
 readme = "README.md"

{lm_deluge-0.0.13 → lm_deluge-0.0.14}/src/lm_deluge/__init__.py

@@ -1,6 +1,7 @@
 from .client import LLMClient, SamplingParams, APIResponse
 from .prompt import Conversation, Message
 from .tool import Tool
+from .file import File
 import dotenv
 
 dotenv.load_dotenv()
@@ -12,4 +13,5 @@ __all__ = [
     "Conversation",
     "Message",
     "Tool",
+    "File",
 ]

{lm_deluge-0.0.13 → lm_deluge-0.0.14}/src/lm_deluge/api_requests/base.py

@@ -1,165 +1,19 @@
 import asyncio
-import json
 import random
 import traceback
 from abc import ABC, abstractmethod
-from dataclasses import dataclass
 from typing import Callable
 
 import aiohttp
 from aiohttp import ClientResponse
 
-from lm_deluge.prompt import CachePattern, Conversation, Message
-from lm_deluge.usage import Usage
+from lm_deluge.prompt import CachePattern, Conversation
 
 from ..config import SamplingParams
 from ..errors import raise_if_modal_exception
 from ..models import APIModel
 from ..tracker import StatusTracker
-
-
-@dataclass
-class APIResponse:
-    # request information
-    id: int  # should be unique to the request within a given prompt-processing call
-    model_internal: str  # our internal model tag
-    prompt: Conversation
-    sampling_params: SamplingParams
-
-    # http response information
-    status_code: int | None
-    is_error: bool | None
-    error_message: str | None
-
-    # completion information - unified usage tracking
-    usage: Usage | None = None
-
-    # response content - structured format
-    content: Message | None = None
-
-    # optional or calculated automatically
-    thinking: str | None = None  # if model shows thinking tokens
-    model_external: str | None = None  # the model tag used by the API
-    region: str | None = None
-    logprobs: list | None = None
-    finish_reason: str | None = None  # make required later
-    cost: float | None = None  # calculated automatically
-    cache_hit: bool = False  # manually set if true
-    # set to true if is_error and should be retried with a different model
-    retry_with_different_model: bool | None = False
-    # set to true if should NOT retry with the same model (unrecoverable error)
-    give_up_if_no_other_models: bool | None = False
-    # OpenAI Responses API specific - used for computer use continuation
-    response_id: str | None = None
-    # Raw API response for debugging
-    raw_response: dict | None = None
-
-    @property
-    def completion(self) -> str | None:
-        """Backward compatibility: extract text from content Message."""
-        if self.content is not None:
-            return self.content.completion
-        return None
-
-    @property
-    def input_tokens(self) -> int | None:
-        """Get input tokens from usage object."""
-        return self.usage.input_tokens if self.usage else None
-
-    @property
-    def output_tokens(self) -> int | None:
-        """Get output tokens from usage object."""
-        return self.usage.output_tokens if self.usage else None
-
-    @property
-    def cache_read_tokens(self) -> int | None:
-        """Get cache read tokens from usage object."""
-        return self.usage.cache_read_tokens if self.usage else None
-
-    @property
-    def cache_write_tokens(self) -> int | None:
-        """Get cache write tokens from usage object."""
-        return self.usage.cache_write_tokens if self.usage else None
-
-    def __post_init__(self):
-        # calculate cost & get external model name
-        self.id = int(self.id)
-        api_model = APIModel.from_registry(self.model_internal)
-        self.model_external = api_model.name
-        self.cost = None
-        if (
-            self.usage is not None
-            and api_model.input_cost is not None
-            and api_model.output_cost is not None
-        ):
-            self.cost = (
-                self.usage.input_tokens * api_model.input_cost / 1e6
-                + self.usage.output_tokens * api_model.output_cost / 1e6
-            )
-        elif self.content is not None and self.completion is not None:
-            print(
-                f"Warning: Completion provided without token counts for model {self.model_internal}."
-            )
-
-    def to_dict(self):
-        return {
-            "id": self.id,
-            "model_internal": self.model_internal,
-            "model_external": self.model_external,
-            "region": self.region,
-            "prompt": self.prompt.to_log(),  # destroys image if present
-            "sampling_params": self.sampling_params.__dict__,
-            "status_code": self.status_code,
-            "is_error": self.is_error,
-            "error_message": self.error_message,
-            "completion": self.completion,  # computed property
-            "content": self.content.to_log() if self.content else None,
-            "usage": self.usage.to_dict() if self.usage else None,
-            "finish_reason": self.finish_reason,
-            "cost": self.cost,
-        }
-
-    @classmethod
-    def from_dict(cls, data: dict):
-        # Handle backward compatibility for content/completion
-        content = None
-        if "content" in data and data["content"] is not None:
-            # Reconstruct message from log format
-            content = Message.from_log(data["content"])
-        elif "completion" in data and data["completion"] is not None:
-            # Backward compatibility: create a Message with just text
-            content = Message.ai(data["completion"])
-
-        usage = None
-        if "usage" in data and data["usage"] is not None:
-            usage = Usage.from_dict(data["usage"])
-
-        return cls(
-            id=data.get("id", random.randint(0, 1_000_000_000)),
-            model_internal=data["model_internal"],
-            prompt=Conversation.from_log(data["prompt"]),
-            sampling_params=SamplingParams(**data["sampling_params"]),
-            status_code=data["status_code"],
-            is_error=data["is_error"],
-            error_message=data["error_message"],
-            usage=usage,
-            content=content,
-            thinking=data.get("thinking"),
-            model_external=data.get("model_external"),
-            region=data.get("region"),
-            logprobs=data.get("logprobs"),
-            finish_reason=data.get("finish_reason"),
-            cost=data.get("cost"),
-            cache_hit=data.get("cache_hit", False),
-        )
-
-    def write_to_file(self, filename):
-        """
-        Writes the APIResponse as a line to a file.
-        If file exists, appends to it.
-        """
-        with open(filename, "a") as f:
-            f.write(json.dumps(self.to_dict()) + "\n")
+from .response import APIResponse
 
 
 class APIRequestBase(ABC):

{lm_deluge-0.0.13 → lm_deluge-0.0.14}/src/lm_deluge/api_requests/openai.py

@@ -1,17 +1,19 @@
-import warnings
-from aiohttp import ClientResponse
 import json
 import os
+import warnings
 from typing import Callable
 
+import aiohttp
+from aiohttp import ClientResponse
+
 from lm_deluge.tool import Tool
 
-from .base import APIRequestBase, APIResponse
-from ..prompt import Conversation, Message, Text, ToolCall, Thinking, CachePattern
-from ..usage import Usage
-from ..tracker import StatusTracker
 from ..config import SamplingParams
 from ..models import APIModel
+from ..prompt import CachePattern, Conversation, Message, Text, Thinking, ToolCall
+from ..tracker import StatusTracker
+from ..usage import Usage
+from .base import APIRequestBase, APIResponse
 
 
 def _build_oa_chat_request(
@@ -111,6 +113,7 @@ class OpenAIRequest(APIRequestBase):
         status_code = http_response.status
         mimetype = http_response.headers.get("Content-Type", None)
         data = None
+        finish_reason = None
         if status_code >= 200 and status_code < 300:
             try:
                 data = await http_response.json()
@@ -125,6 +128,7 @@ class OpenAIRequest(APIRequestBase):
                     # Parse response into Message with parts
                     parts = []
                     message = data["choices"][0]["message"]
+                    finish_reason = data["choices"][0]["finish_reason"]
 
                     # Add text content if present
                     if message.get("content"):
@@ -190,6 +194,7 @@ class OpenAIRequest(APIRequestBase):
             sampling_params=self.sampling_params,
             usage=usage,
             raw_response=data,
+            finish_reason=finish_reason,
         )
 
 
@@ -266,6 +271,13 @@ class OpenAIResponsesRequest(APIRequestBase):
             self.request_json["max_output_tokens"] = sampling_params.max_new_tokens
 
         if self.model.reasoning_model:
+            if sampling_params.reasoning_effort in [None, "none"]:
+                # gemini models can switch reasoning off
+                if "gemini" in self.model.id:
+                    self.sampling_params.reasoning_effort = "none"  # expects string
+                # openai models can only go down to "low"
+                else:
+                    self.sampling_params.reasoning_effort = "low"
             self.request_json["temperature"] = 1.0
             self.request_json["top_p"] = 1.0
             self.request_json["reasoning"] = {
@@ -413,3 +425,57 @@ class OpenAIResponsesRequest(APIRequestBase):
             usage=usage,
             raw_response=data,
         )
+
+
+async def stream_chat(
+    model_name: str,  # must correspond to registry
+    prompt: Conversation,
+    sampling_params: SamplingParams = SamplingParams(),
+    tools: list | None = None,
+    cache: CachePattern | None = None,
+):
+    if cache is not None:
+        warnings.warn(
+            f"Cache parameter '{cache}' is only supported for Anthropic models, ignoring for {model_name}"
+        )
+
+    model = APIModel.from_registry(model_name)
+    if model.api_spec != "openai":
+        raise ValueError("streaming only supported on openai models for now")
+    url = f"{model.api_base}/chat/completions"
+    request_header = {"Authorization": f"Bearer {os.getenv(model.api_key_env_var)}"}
+    request_json = _build_oa_chat_request(model, prompt, tools, sampling_params)
+    request_json["stream"] = True
+
+    async with aiohttp.ClientSession() as s:
+        async with s.post(url, headers=request_header, json=request_json) as r:
+            r.raise_for_status()  # bail on 4xx/5xx
+            content = ""
+            buf = ""
+            async for chunk in r.content.iter_any():  # raw bytes
+                buf += chunk.decode()
+                while "\n\n" in buf:  # full SSE frame
+                    event, buf = buf.split("\n\n", 1)
+                    if not event.startswith("data:"):
+                        continue  # ignore comments
+                    data = event[5:].strip()  # after "data:"
+                    if data == "[DONE]":
+                        yield APIResponse(
+                            id=0,
+                            status_code=None,
+                            is_error=False,
+                            error_message=None,
+                            prompt=prompt,
+                            content=Message(
+                                role="assistant", parts=[Text(text=content)]
+                            ),
+                            model_internal=model.id,
+                            sampling_params=sampling_params,
+                            usage=None,
+                            raw_response=None,
+                        )
+                    msg = json.loads(data)  # SSE payload
+                    delta = msg["choices"][0]["delta"].get("content")
+                    if delta:
+                        content += delta
+                        yield delta

lm_deluge-0.0.14/src/lm_deluge/api_requests/response.py

@@ -0,0 +1,153 @@
+import json
+import random
+from dataclasses import dataclass
+
+from lm_deluge.prompt import Conversation, Message
+from lm_deluge.usage import Usage
+
+from ..config import SamplingParams
+from ..models import APIModel
+
+
+@dataclass
+class APIResponse:
+    # request information
+    id: int  # should be unique to the request within a given prompt-processing call
+    model_internal: str  # our internal model tag
+    prompt: Conversation
+    sampling_params: SamplingParams
+
+    # http response information
+    status_code: int | None
+    is_error: bool | None
+    error_message: str | None
+
+    # completion information - unified usage tracking
+    usage: Usage | None = None
+
+    # response content - structured format
+    content: Message | None = None
+
+    # optional or calculated automatically
+    thinking: str | None = None  # if model shows thinking tokens
+    model_external: str | None = None  # the model tag used by the API
+    region: str | None = None
+    logprobs: list | None = None
+    finish_reason: str | None = None  # make required later
+    cost: float | None = None  # calculated automatically
+    cache_hit: bool = False  # manually set if true
+    # set to true if is_error and should be retried with a different model
+    retry_with_different_model: bool | None = False
+    # set to true if should NOT retry with the same model (unrecoverable error)
+    give_up_if_no_other_models: bool | None = False
+    # OpenAI Responses API specific - used for computer use continuation
+    response_id: str | None = None
+    # Raw API response for debugging
+    raw_response: dict | None = None
+
+    @property
+    def completion(self) -> str | None:
+        """Backward compatibility: extract text from content Message."""
+        if self.content is not None:
+            return self.content.completion
+        return None
+
+    @property
+    def input_tokens(self) -> int | None:
+        """Get input tokens from usage object."""
+        return self.usage.input_tokens if self.usage else None
+
+    @property
+    def output_tokens(self) -> int | None:
+        """Get output tokens from usage object."""
+        return self.usage.output_tokens if self.usage else None
+
+    @property
+    def cache_read_tokens(self) -> int | None:
+        """Get cache read tokens from usage object."""
+        return self.usage.cache_read_tokens if self.usage else None
+
+    @property
+    def cache_write_tokens(self) -> int | None:
+        """Get cache write tokens from usage object."""
+        return self.usage.cache_write_tokens if self.usage else None
+
+    def __post_init__(self):
+        # calculate cost & get external model name
+        self.id = int(self.id)
+        api_model = APIModel.from_registry(self.model_internal)
+        self.model_external = api_model.name
+        self.cost = None
+        if (
+            self.usage is not None
+            and api_model.input_cost is not None
+            and api_model.output_cost is not None
+        ):
+            self.cost = (
+                self.usage.input_tokens * api_model.input_cost / 1e6
+                + self.usage.output_tokens * api_model.output_cost / 1e6
+            )
+        elif self.content is not None and self.completion is not None:
+            print(
+                f"Warning: Completion provided without token counts for model {self.model_internal}."
+            )
+
+    def to_dict(self):
+        return {
+            "id": self.id,
+            "model_internal": self.model_internal,
+            "model_external": self.model_external,
+            "region": self.region,
+            "prompt": self.prompt.to_log(),  # destroys image if present
+            "sampling_params": self.sampling_params.__dict__,
+            "status_code": self.status_code,
+            "is_error": self.is_error,
+            "error_message": self.error_message,
+            "completion": self.completion,  # computed property
+            "content": self.content.to_log() if self.content else None,
+            "usage": self.usage.to_dict() if self.usage else None,
+            "finish_reason": self.finish_reason,
+            "cost": self.cost,
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        # Handle backward compatibility for content/completion
+        content = None
+        if "content" in data and data["content"] is not None:
+            # Reconstruct message from log format
+            content = Message.from_log(data["content"])
+        elif "completion" in data and data["completion"] is not None:
+            # Backward compatibility: create a Message with just text
+            content = Message.ai(data["completion"])
+
+        usage = None
+        if "usage" in data and data["usage"] is not None:
+            usage = Usage.from_dict(data["usage"])
+
+        return cls(
+            id=data.get("id", random.randint(0, 1_000_000_000)),
+            model_internal=data["model_internal"],
+            prompt=Conversation.from_log(data["prompt"]),
+            sampling_params=SamplingParams(**data["sampling_params"]),
+            status_code=data["status_code"],
+            is_error=data["is_error"],
+            error_message=data["error_message"],
+            usage=usage,
+            content=content,
+            thinking=data.get("thinking"),
+            model_external=data.get("model_external"),
+            region=data.get("region"),
+            logprobs=data.get("logprobs"),
+            finish_reason=data.get("finish_reason"),
+            cost=data.get("cost"),
+            cache_hit=data.get("cache_hit", False),
+        )
+
+    def write_to_file(self, filename):
+        """
+        Writes the APIResponse as a line to a file.
+        If file exists, appends to it.
+        """
+        with open(filename, "a") as f:
+            f.write(json.dumps(self.to_dict()) + "\n")

{lm_deluge-0.0.13 → lm_deluge-0.0.14}/src/lm_deluge/client.py

@@ -6,6 +6,7 @@ import yaml
 from pydantic import BaseModel
 from pydantic.functional_validators import model_validator
 
+from lm_deluge.api_requests.openai import stream_chat
 from lm_deluge.batches import (
     submit_batches_anthropic,
     submit_batches_oa,
@@ -34,6 +35,12 @@ class LLMClient(BaseModel):
     """
 
     model_names: list[str] = ["gpt-4.1-mini"]
+
+    def __init__(self, model_name: str | list[str] | None = None, **kwargs):
+        if model_name is not None:
+            kwargs["model_names"] = model_name
+        super().__init__(**kwargs)
+
     max_requests_per_minute: int = 1_000
     max_tokens_per_minute: int = 100_000
     max_concurrent_requests: int = 225
@@ -81,7 +88,7 @@ class LLMClient(BaseModel):
     @model_validator(mode="before")
     @classmethod
     def fix_lists(cls, data) -> "LLMClient":
-        if isinstance(data["model_names"], str):
+        if isinstance(data.get("model_names"), str):
             data["model_names"] = [data["model_names"]]
         if "sampling_params" not in data or len(data.get("sampling_params", [])) == 0:
             data["sampling_params"] = [
@@ -162,6 +169,11 @@ class LLMClient(BaseModel):
         kwargs["model_names"] = model
         return cls(**kwargs)
 
+    def _select_model(self):
+        assert isinstance(self.model_weights, list)
+        model_idx = np.random.choice(range(len(self.models)), p=self.model_weights)
+        return self.models[model_idx], self.sampling_params[model_idx]
+
     @overload
     async def process_prompts_async(
         self,
@@ -249,41 +261,6 @@ class LLMClient(BaseModel):
             if len(cache_hit_ids) > 0:
                 tracker.update_pbar(len(cache_hit_ids))
 
-            # api_task = asyncio.create_task(
-            #     process_api_prompts_async(
-            #         ids,
-            #         prompts,  # type: ignore -- fix later for dry running conversations
-            #         self.models,
-            #         self.model_weights,  # type: ignore
-            #         self.sampling_params,  # type: ignore
-            #         max_attempts=self.max_attempts,
-            #         max_concurrent_requests=self.max_concurrent_requests,
-            #         request_timeout=self.request_timeout,
-            #         status_tracker=tracker,
-            #         tools=tools,
-            #         cache=cache,
-            #         computer_use=computer_use,
-            #         display_width=display_width,
-            #         display_height=display_height,
-            #         use_responses_api=use_responses_api,
-            #     )
-            # )
-            # async def process_api_prompts_async(
-
-            #     models: str | list[str],
-            #     model_weights: list[float],
-            #     sampling_params: list[SamplingParams],
-            #     max_attempts: int = 5,
-            #     max_concurrent_requests: int = 1_000,
-            #     request_timeout: int = 30,
-            #     status_tracker: StatusTracker | None = None,
-            #     tools: list[Tool] | None = None,
-            #     cache: CachePattern | None = None,
-            #     computer_use: bool = False,
-            #     display_width: int = 1024,
-            #     display_height: int = 768,
-            #     use_responses_api: bool = False,
-            # ):
             if isinstance(ids, np.ndarray):
                 ids = ids.tolist()  # pyright: ignore
 
@@ -296,28 +273,28 @@ class LLMClient(BaseModel):
             assert tracker.retry_queue, "retry queue not initialized"
             while True:
                 # get next request (if one is not already waiting for capacity)
+                retry_request = False
                 if next_request is None:
                     if not tracker.retry_queue.empty():
                         next_request = tracker.retry_queue.get_nowait()
+                        retry_request = True
                         print(f"Retrying request {next_request.task_id}.")
                     elif prompts_not_finished:
                         try:
                             # get new request
                             id, prompt = next(prompts_iter)
                             # select model
-                            assert isinstance(self.model_weights, list)
-                            model_idx = np.random.choice(
-                                range(len(self.models)), p=self.model_weights
-                            )
+                            model, sampling_params = self._select_model()
+
                             next_request = create_api_request(
                                 task_id=id,
-                                model_name=self.models[model_idx],
+                                model_name=model,
                                 prompt=prompt,  # type: ignore
                                 request_timeout=self.request_timeout,
                                 attempts_left=self.max_attempts,
                                 status_tracker=tracker,
                                 results_arr=requests,
-                                sampling_params=self.sampling_params[model_idx],
+                                sampling_params=sampling_params,
                                 all_model_names=self.models,
                                 all_sampling_params=self.sampling_params,
                                 tools=tools,
@@ -339,10 +316,9 @@ class LLMClient(BaseModel):
                 # if enough capacity available, call API
                 if next_request:
                     next_request_tokens = next_request.num_tokens
-                    if tracker.check_capacity(next_request_tokens):
+                    if tracker.check_capacity(next_request_tokens, retry=retry_request):
                         tracker.set_limiting_factor(None)
-                        next_request.attempts_left -= 1
-                        # call API
+                        # call API (attempts_left will be decremented in handle_error if it fails)
                         asyncio.create_task(next_request.call_api())
                         next_request = None  # reset next_request to empty
                 # update pbar status
@@ -360,9 +336,10 @@ class LLMClient(BaseModel):
                     await asyncio.sleep(tracker.seconds_to_pause)
                     print(f"Pausing {tracker.seconds_to_pause}s to cool down.")
 
-                # after finishing, log final status
-                tracker.log_final_status()
-                # deduplicate results by id
+            # after finishing, log final status
+            tracker.log_final_status()
+
+            # deduplicate results by id
             api_results = deduplicate_responses(requests)
             for res in api_results:
                 results[res.id] = res
@@ -399,6 +376,17 @@ class LLMClient(BaseModel):
             )
         )
 
+    async def stream(self, prompt: str | Conversation, tools: list[Tool] | None = None):
+        model, sampling_params = self._select_model()
+        if isinstance(prompt, str):
+            prompt = Conversation.user(prompt)
+        async for item in stream_chat(model, prompt, sampling_params, tools, None):
+            if isinstance(item, str):
+                print(item, end="", flush=True)
+            else:
+                # final item
+                return item
+
     async def submit_batch_job(
         self,
         prompts: Sequence[str | list[dict] | Conversation],