lm-deluge 0.0.44__tar.gz → 0.0.46__tar.gz

{lm_deluge-0.0.44/src/lm_deluge.egg-info → lm_deluge-0.0.46}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lm_deluge
-Version: 0.0.44
+Version: 0.0.46
 Summary: Python utility for using LLM API models.
 Author-email: Benjamin Anderson <ben@trytaylor.ai>
 Requires-Python: >=3.10

{lm_deluge-0.0.44 → lm_deluge-0.0.46}/pyproject.toml

@@ -3,7 +3,7 @@ requires = ["setuptools", "wheel"]
 
 [project]
 name = "lm_deluge"
-version = "0.0.44"
+version = "0.0.46"
 authors = [{ name = "Benjamin Anderson", email = "ben@trytaylor.ai" }]
 description = "Python utility for using LLM API models."
 readme = "README.md"

{lm_deluge-0.0.44 → lm_deluge-0.0.46}/src/lm_deluge/client.py

@@ -217,17 +217,19 @@ class _LLMClient(BaseModel):
         self, num_tokens: int, tracker: StatusTracker, *, retry: bool = False
     ):
         while True:
+            # Enforce cooldown first, regardless of current capacity.
+            cooldown = tracker.seconds_to_pause
+            if cooldown > 0:
+                print(f"Pausing for {cooldown} seconds to cool down.")
+                await asyncio.sleep(cooldown)
+                continue
+
             async with self._capacity_lock:
                 if tracker.check_capacity(num_tokens, retry=retry):
                     tracker.set_limiting_factor(None)
                     return
-                seconds_to_pause = tracker.seconds_to_pause
-
-            if seconds_to_pause > 0:
-                print(f"Pausing for {seconds_to_pause} seconds to cool down.")
-                await asyncio.sleep(seconds_to_pause)
-            else:
-                await asyncio.sleep(30.0 / self.max_requests_per_minute)
+            # Idle wait before next capacity check. Aim for ~RPM spacing.
+            await asyncio.sleep(max(60.0 / self.max_requests_per_minute, 0.01))
 
     async def _execute_request(self, context: RequestContext) -> APIResponse:
         """Create and send a single API request using the provided context."""
@@ -363,7 +365,7 @@ class _LLMClient(BaseModel):
         # Create retry queue for failed requests
         retry_queue: asyncio.Queue[RequestContext] = asyncio.Queue()
 
-        # Calculate sleep time for rate limiting
+        # Calculate sleep time for rate limiting (legacy; gating happens in _wait_for_capacity)
         seconds_to_sleep_each_loop = (60.0 * 0.9) / tracker.max_requests_per_minute
 
         # Main dispatch loop - using original pattern but with all prompts
@@ -403,40 +405,37 @@ class _LLMClient(BaseModel):
                     except StopIteration:
                         prompts_not_finished = False
 
-            # Update capacity - original logic
-            tracker.update_capacity()
-
-            # Dispatch if capacity available - original logic
+            # Dispatch using shared capacity gate (consistent with start_nowait)
             if next_context:
-                if tracker.check_capacity(next_context.num_tokens, retry=next_is_retry):
-                    tracker.set_limiting_factor(None)
+                # Wait here until we have capacity to launch this context
+                await self._wait_for_capacity(
+                    next_context.num_tokens, tracker, retry=next_is_retry
+                )
+
+                # Launch simplified request processing
+                async def process_and_store(ctx: RequestContext):
+                    try:
+                        response = await self.process_single_request(ctx, retry_queue)
+                        results[ctx.task_id] = response
+                    except Exception as e:
+                        # Create an error response for validation errors and other exceptions
+                        error_response = APIResponse(
+                            id=ctx.task_id,
+                            model_internal=ctx.model_name,
+                            prompt=ctx.prompt,
+                            sampling_params=ctx.sampling_params,
+                            status_code=None,
+                            is_error=True,
+                            error_message=str(e),
+                        )
+                        results[ctx.task_id] = error_response
+                        # Mark task as completed so the main loop can finish
+                        if ctx.status_tracker:
+                            ctx.status_tracker.task_failed(ctx.task_id)
 
-                    # Launch simplified request processing
-                    async def process_and_store(ctx: RequestContext):
-                        try:
-                            response = await self.process_single_request(
-                                ctx, retry_queue
-                            )
-                            results[ctx.task_id] = response
-                        except Exception as e:
-                            # Create an error response for validation errors and other exceptions
-                            error_response = APIResponse(
-                                id=ctx.task_id,
-                                model_internal=ctx.model_name,
-                                prompt=ctx.prompt,
-                                sampling_params=ctx.sampling_params,
-                                status_code=None,
-                                is_error=True,
-                                error_message=str(e),
-                            )
-                            results[ctx.task_id] = error_response
-                            # Mark task as completed so the main loop can finish
-                            if ctx.status_tracker:
-                                ctx.status_tracker.task_failed(ctx.task_id)
-
-                    asyncio.create_task(process_and_store(next_context))
-                    next_context = None  # Reset after successful dispatch
-                    next_is_retry = False
+                asyncio.create_task(process_and_store(next_context))
+                next_context = None  # Reset after successful dispatch
+                next_is_retry = False
 
             # Update progress - original logic
             tracker.update_pbar()
@@ -448,8 +447,8 @@ class _LLMClient(BaseModel):
             ) and retry_queue.empty():
                 break
 
-            # Sleep - original logic
-            await asyncio.sleep(seconds_to_sleep_each_loop + tracker.seconds_to_pause)
+            # Yield briefly to allow in-flight tasks to progress
+            await asyncio.sleep(min(0.01, seconds_to_sleep_each_loop))
 
         if not tracker_preopened:
             self.close()

lm_deluge-0.0.46/src/lm_deluge/presets/cerebras.py

@@ -0,0 +1,17 @@
+from lm_deluge import LLMClient
+
+mixture_of_cerebras = LLMClient(
+    [
+        "gpt-oss-120b-cerebras",
+        "llama-4-scout-cerebras",
+        "llama-3.3-70b-cerebras",
+        "qwen-3-32b-cerebras",
+        "llama-4-maverick-cerebras",
+        "qwen-3-235b-instruct-cerebras",
+        "qwen-3-235b-thinking-cerebras",
+        "qwen-3-coder-cerebras",
+    ],
+    model_weights=[3, 3, 3, 3, 3, 3, 3, 1],
+    max_requests_per_minute=250,
+    max_tokens_per_minute=1_000_000,
+)

lm_deluge-0.0.46/src/lm_deluge/presets/meta.py

@@ -0,0 +1,13 @@
+from lm_deluge import LLMClient
+
+mixture_of_llamas = LLMClient(
+    ["llama-4-scout", "llama-4-maverick", "llama-3.3-70b", "llama-3.3-8b"],
+    max_requests_per_minute=12_000,
+    max_tokens_per_minute=4_000_000,
+)
+
+multimodal_llamas = LLMClient(
+    ["llama-4-scout", "llama-4-maverick"],
+    max_requests_per_minute=6_000,
+    max_tokens_per_minute=2_000_000,
+)

{lm_deluge-0.0.44 → lm_deluge-0.0.46}/src/lm_deluge/prompt.py

@@ -333,6 +333,23 @@ class Message:
         """
         Return a JSON-serialisable dict that fully captures the message.
         """
+        def _json_safe(value):
+            if isinstance(value, (str, int, float, bool)) or value is None:
+                return value
+            if isinstance(value, list):
+                return [_json_safe(v) for v in value]
+            if isinstance(value, dict):
+                return {k: _json_safe(v) for k, v in value.items()}
+            if isinstance(value, Text):
+                return {"type": "text", "text": value.text}
+            if isinstance(value, Image):
+                w, h = value.size
+                return {"type": "image", "tag": f"<Image ({w}×{h})>"}
+            if isinstance(value, File):
+                size = value.size
+                return {"type": "file", "tag": f"<File ({size} bytes)>"}
+            return repr(value)
+
         content_blocks: list[dict] = []
         for p in self.parts:
             if isinstance(p, Text):
@@ -349,7 +366,7 @@ class Message:
                         "type": "tool_call",
                         "id": p.id,
                         "name": p.name,
-                        "arguments": p.arguments,
+                        "arguments": _json_safe(p.arguments),
                     }
                 )
             elif isinstance(p, ToolResult):
@@ -357,7 +374,7 @@ class Message:
                     {
                         "type": "tool_result",
                         "tool_call_id": p.tool_call_id,
-                        "result": p.result,
+                        "result": _json_safe(p.result),
                     }
                 )
             elif isinstance(p, Thinking):

{lm_deluge-0.0.44 → lm_deluge-0.0.46/src/lm_deluge.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lm_deluge
-Version: 0.0.44
+Version: 0.0.46
 Summary: Python utility for using LLM API models.
 Author-email: Benjamin Anderson <ben@trytaylor.ai>
 Requires-Python: >=3.10

{lm_deluge-0.0.44 → lm_deluge-0.0.46}/src/lm_deluge.egg-info/SOURCES.txt

@@ -66,6 +66,8 @@ src/lm_deluge/models/mistral.py
 src/lm_deluge/models/openai.py
 src/lm_deluge/models/openrouter.py
 src/lm_deluge/models/together.py
+src/lm_deluge/presets/cerebras.py
+src/lm_deluge/presets/meta.py
 src/lm_deluge/util/harmony.py
 src/lm_deluge/util/json.py
 src/lm_deluge/util/logprobs.py