lm-deluge 0.0.56__py3-none-any.whl → 0.0.69__py3-none-any.whl

lm_deluge/mock_openai.py

@@ -0,0 +1,482 @@
+"""
+Mock OpenAI client that implements the AsyncOpenAI interface but uses lm-deluge's
+LLMClient internally. This allows using any lm-deluge-supported provider through
+the standard OpenAI Python SDK interface.
+
+Example usage:
+    from lm_deluge.mock_openai import MockAsyncOpenAI
+
+    # Use Claude through OpenAI interface
+    client = MockAsyncOpenAI(model="claude-sonnet-4")
+    response = await client.chat.completions.create(
+        model="claude-sonnet-4",  # Can override here
+        messages=[{"role": "user", "content": "Hello!"}],
+        temperature=0.7
+    )
+    print(response.choices[0].message.content)
+
+Installation:
+    pip install lm-deluge[openai]
+"""
+
+import json
+import time
+import uuid
+from typing import Any, AsyncIterator, Literal, Union, overload
+
+try:
+    from openai.types.chat import (
+        ChatCompletion,
+        ChatCompletionChunk,
+        ChatCompletionMessage,
+        ChatCompletionMessageToolCall,
+    )
+    from openai.types.chat.chat_completion import Choice as CompletionChoice
+    from openai.types.chat.chat_completion_chunk import (
+        Choice as ChunkChoice,
+        ChoiceDelta,
+        ChoiceDeltaToolCall,
+        ChoiceDeltaToolCallFunction,
+    )
+    from openai.types.chat.chat_completion_message_tool_call import Function
+    from openai.types.completion_usage import CompletionUsage
+except ImportError:
+    raise ImportError(
+        "The openai package is required to use MockAsyncOpenAI. "
+        "Install it with: pip install lm-deluge[openai]"
+    )
+
+from lm_deluge.client import LLMClient
+from lm_deluge.prompt import Conversation, Message, Part, Text, ToolCall, ToolResult
+
+
+def _messages_to_conversation(messages: list[dict[str, Any]]) -> Conversation:
+    """Convert OpenAI messages format to lm-deluge Conversation."""
+    conv_messages = []
+
+    for msg in messages:
+        role = msg["role"]
+        content = msg.get("content")
+        tool_calls = msg.get("tool_calls")
+        tool_call_id = msg.get("tool_call_id")
+
+        parts: list[Part] = []
+
+        # Handle regular content
+        if content:
+            if isinstance(content, str):
+                parts.append(Text(content))
+            elif isinstance(content, list):
+                # Multi-part content (text, images, etc.)
+                for item in content:
+                    if item.get("type") == "text":
+                        parts.append(Text(item["text"]))
+                    # Could add image support here later
+
+        # Handle tool calls (from assistant)
+        if tool_calls:
+            for tc in tool_calls:
+                # Parse arguments from JSON string to dict
+                args_str = tc["function"]["arguments"]
+                args_dict = (
+                    json.loads(args_str) if isinstance(args_str, str) else args_str
+                )
+                parts.append(
+                    ToolCall(
+                        id=tc["id"],
+                        name=tc["function"]["name"],
+                        arguments=args_dict,
+                    )
+                )
+
+        # Handle tool results (from tool role)
+        if role == "tool" and tool_call_id:
+            parts.append(ToolResult(tool_call_id=tool_call_id, result=content or ""))
+
+        conv_messages.append(Message(role=role, parts=parts))
+
+    return Conversation(messages=conv_messages)
+
+
+def _response_to_chat_completion(
+    response: Any,  # APIResponse
+    model: str,
+    request_id: str | None = None,
+) -> ChatCompletion:
+    """Convert lm-deluge APIResponse to OpenAI ChatCompletion."""
+    if request_id is None:
+        request_id = f"chatcmpl-{uuid.uuid4().hex[:24]}"
+
+    # Handle error responses
+    if response.is_error:
+        # For errors, create an empty response with error finish reason
+        message = ChatCompletionMessage(
+            role="assistant",
+            content=response.error_message or "Error occurred",
+        )
+        choice = CompletionChoice(
+            index=0,
+            message=message,
+            finish_reason="stop",  # or could use "error" but that's not standard
+        )
+        return ChatCompletion(
+            id=request_id,
+            choices=[choice],
+            created=int(time.time()),
+            model=model,
+            object="chat.completion",
+            usage=None,
+        )
+
+    # Extract content from response
+    content_text = None
+    tool_calls = None
+
+    if response.content:
+        # Extract text parts
+        text_parts = [p.text for p in response.content.parts if isinstance(p, Text)]
+        if text_parts:
+            content_text = "".join(text_parts)
+
+        # Extract tool calls
+        tool_call_parts = [p for p in response.content.parts if isinstance(p, ToolCall)]
+        if tool_call_parts:
+            tool_calls = [
+                ChatCompletionMessageToolCall(
+                    id=tc.id,
+                    type="function",
+                    function=Function(
+                        name=tc.name,
+                        # Convert dict arguments to JSON string for OpenAI format
+                        arguments=json.dumps(tc.arguments)
+                        if isinstance(tc.arguments, dict)
+                        else tc.arguments,
+                    ),
+                )
+                for tc in tool_call_parts
+            ]
+
+    # Create message
+    message = ChatCompletionMessage(
+        role="assistant",
+        content=content_text,
+        tool_calls=tool_calls,
+    )
+
+    # Create choice
+    choice = CompletionChoice(
+        index=0,
+        message=message,
+        finish_reason=response.finish_reason or "stop",
+    )
+
+    # Create usage
+    usage = None
+    if response.usage:
+        usage = CompletionUsage(
+            prompt_tokens=response.usage.input_tokens,
+            completion_tokens=response.usage.output_tokens,
+            total_tokens=response.usage.input_tokens + response.usage.output_tokens,
+        )
+
+    return ChatCompletion(
+        id=request_id,
+        choices=[choice],
+        created=int(time.time()),
+        model=model,
+        object="chat.completion",
+        usage=usage,
+    )
+
+
+class _AsyncStreamWrapper:
+    """Wrapper to convert lm-deluge streaming to OpenAI ChatCompletionChunk format."""
+
+    def __init__(self, stream: AsyncIterator, model: str, request_id: str):
+        self._stream = stream
+        self._model = model
+        self._request_id = request_id
+        self._first_chunk = True
+
+    def __aiter__(self):
+        return self
+
+    async def __anext__(self) -> ChatCompletionChunk:
+        chunk = await self._stream.__anext__()
+
+        # Create delta based on chunk content
+        delta = ChoiceDelta()
+
+        if self._first_chunk:
+            delta.role = "assistant"
+            self._first_chunk = False
+
+        # Extract content from chunk
+        if hasattr(chunk, "content") and chunk.content:
+            if isinstance(chunk.content, str):
+                delta.content = chunk.content
+            elif hasattr(chunk.content, "parts"):
+                # Extract text from parts
+                text_parts = [
+                    p.text for p in chunk.content.parts if isinstance(p, Text)
+                ]
+                if text_parts:
+                    delta.content = "".join(text_parts)
+
+                # Extract tool calls from parts
+                tool_call_parts = [
+                    p for p in chunk.content.parts if isinstance(p, ToolCall)
+                ]
+                if tool_call_parts:
+                    delta.tool_calls = [
+                        ChoiceDeltaToolCall(
+                            index=i,
+                            id=tc.id,
+                            type="function",
+                            function=ChoiceDeltaToolCallFunction(
+                                name=tc.name,
+                                # Convert dict arguments to JSON string for OpenAI format
+                                arguments=json.dumps(tc.arguments)
+                                if isinstance(tc.arguments, dict)
+                                else tc.arguments,
+                            ),
+                        )
+                        for i, tc in enumerate(tool_call_parts)
+                    ]
+
+        # Create choice
+        choice = ChunkChoice(
+            index=0,
+            delta=delta,
+            finish_reason=getattr(chunk, "finish_reason", None),
+        )
+
+        return ChatCompletionChunk(
+            id=self._request_id,
+            choices=[choice],
+            created=int(time.time()),
+            model=self._model,
+            object="chat.completion.chunk",
+        )
+
+
+class MockCompletions:
+    """Mock completions resource that implements OpenAI's completions.create interface."""
+
+    def __init__(self, parent: "MockAsyncOpenAI"):
+        self._parent = parent
+
+    @overload
+    async def create(
+        self,
+        *,
+        messages: list[dict[str, Any]],
+        model: str,
+        stream: Literal[False] = False,
+        **kwargs: Any,
+    ) -> ChatCompletion: ...
+
+    @overload
+    async def create(
+        self,
+        *,
+        messages: list[dict[str, Any]],
+        model: str,
+        stream: Literal[True],
+        **kwargs: Any,
+    ) -> AsyncIterator[ChatCompletionChunk]: ...
+
+    async def create(
+        self,
+        *,
+        messages: list[dict[str, Any]],
+        model: str,
+        stream: bool = False,
+        temperature: float | None = None,
+        max_tokens: int | None = None,
+        max_completion_tokens: int | None = None,
+        top_p: float | None = None,
+        seed: int | None = None,
+        tools: list[dict[str, Any]] | None = None,
+        tool_choice: Any | None = None,
+        reasoning_effort: str | None = None,
+        response_format: dict[str, Any] | None = None,
+        n: int | None = None,
+        stop: str | list[str] | None = None,
+        presence_penalty: float | None = None,
+        frequency_penalty: float | None = None,
+        **kwargs: Any,
+    ) -> Union[ChatCompletion, AsyncIterator[ChatCompletionChunk]]:
+        """
+        Create a chat completion using lm-deluge's LLMClient.
+
+        Args:
+            messages: List of message dictionaries with 'role' and 'content'
+            model: Model identifier (can override client's default model)
+            stream: Whether to stream the response
+            temperature: Sampling temperature (0-2)
+            max_tokens: Max tokens (deprecated, use max_completion_tokens)
+            max_completion_tokens: Max completion tokens
+            top_p: Nucleus sampling parameter
+            seed: Random seed for deterministic sampling
+            tools: List of tool definitions
+            tool_choice: Tool choice strategy
+            reasoning_effort: Reasoning effort for reasoning models
+            response_format: Response format (e.g., {"type": "json_object"})
+            **kwargs: Other parameters (mostly ignored for compatibility)
+
+        Returns:
+            ChatCompletion (non-streaming) or AsyncIterator[ChatCompletionChunk] (streaming)
+        """
+        # Get or create client for this model
+        client = self._parent._get_or_create_client(model)
+
+        # Convert messages to Conversation
+        conversation = _messages_to_conversation(messages)
+
+        # Build sampling params
+        sampling_kwargs = {}
+        if temperature is not None:
+            sampling_kwargs["temperature"] = temperature
+        if max_completion_tokens is not None:
+            sampling_kwargs["max_new_tokens"] = max_completion_tokens
+        elif max_tokens is not None:
+            sampling_kwargs["max_new_tokens"] = max_tokens
+        if top_p is not None:
+            sampling_kwargs["top_p"] = top_p
+        if seed is not None:
+            sampling_kwargs["seed"] = seed
+        if reasoning_effort is not None:
+            sampling_kwargs["reasoning_effort"] = reasoning_effort
+        if response_format and response_format.get("type") == "json_object":
+            sampling_kwargs["json_mode"] = True
+
+        # If sampling params are provided, create a new client with merged params
+        if sampling_kwargs:
+            # Merge with default params
+            merged_params = {**self._parent._default_sampling_params, **sampling_kwargs}
+            client = self._parent._create_client_with_params(model, merged_params)
+
+        # Convert tools if provided
+        lm_tools = None
+        if tools:
+            # For now, just pass through - lm-deluge will handle the format
+            lm_tools = tools
+
+        # Execute request
+        if stream:
+            # Streaming mode
+            request_id = f"chatcmpl-{uuid.uuid4().hex[:24]}"
+            # Note: client.stream() is an async generator, not a coroutine
+            # We can directly wrap it
+            stream_iter = client.stream(conversation, tools=lm_tools)
+            # Verify it's a generator, not a coroutine
+            if hasattr(stream_iter, "__anext__"):
+                return _AsyncStreamWrapper(stream_iter, model, request_id)
+            else:
+                # If it's a coroutine, we need to await it first
+                # But this shouldn't happen with the current implementation
+                raise TypeError(f"Expected async generator, got {type(stream_iter)}")
+        else:
+            # Non-streaming mode
+            response = await client.start(conversation, tools=lm_tools)
+            return _response_to_chat_completion(response, model)
+
+
+class MockChat:
+    """Mock chat resource that provides access to completions."""
+
+    def __init__(self, parent: "MockAsyncOpenAI"):
+        self._parent = parent
+        self._completions = MockCompletions(parent)
+
+    @property
+    def completions(self) -> MockCompletions:
+        """Access the completions resource."""
+        return self._completions
+
+
+class MockAsyncOpenAI:
+    """
+    Mock AsyncOpenAI client that uses lm-deluge's LLMClient internally.
+
+    This allows using any lm-deluge-supported provider (Anthropic, Google, etc.)
+    through the standard OpenAI Python SDK interface.
+
+    Example:
+        # Use Claude through OpenAI interface
+        client = MockAsyncOpenAI(model="claude-sonnet-4")
+        response = await client.chat.completions.create(
+            model="claude-sonnet-4",
+            messages=[{"role": "user", "content": "Hello!"}],
+            temperature=0.7
+        )
+
+    Args:
+        model: Default model to use (can be overridden in create())
+        temperature: Default temperature
+        max_completion_tokens: Default max completion tokens
+        top_p: Default top_p
+        **kwargs: Additional parameters passed to LLMClient
+    """
+
+    def __init__(
+        self,
+        *,
+        model: str,
+        temperature: float | None = None,
+        max_completion_tokens: int | None = None,
+        top_p: float | None = None,
+        seed: int | None = None,
+        **kwargs: Any,
+    ):
+        self._default_model = model
+        self._default_sampling_params = {}
+
+        if temperature is not None:
+            self._default_sampling_params["temperature"] = temperature
+        if max_completion_tokens is not None:
+            self._default_sampling_params["max_new_tokens"] = max_completion_tokens
+        if top_p is not None:
+            self._default_sampling_params["top_p"] = top_p
+        if seed is not None:
+            self._default_sampling_params["seed"] = seed
+
+        # Additional kwargs for LLMClient
+        self._client_kwargs = kwargs
+
+        # Cache of LLMClient instances by model
+        self._clients: dict[str, Any] = {}
+
+        # Create the default client
+        self._clients[model] = self._create_client(model)
+
+        # Create nested resources
+        self._chat = MockChat(self)
+
+    def _create_client(self, model: str) -> Any:
+        """Create a new LLMClient for the given model."""
+        return LLMClient(
+            model,
+            **self._default_sampling_params,
+            **self._client_kwargs,
+        )
+
+    def _create_client_with_params(self, model: str, params: dict[str, Any]) -> Any:
+        """Create a new LLMClient with specific sampling parameters."""
+        return LLMClient(
+            model,
+            **params,
+            **self._client_kwargs,
+        )
+
+    def _get_or_create_client(self, model: str) -> Any:
+        """Get existing client or create new one for the model."""
+        if model not in self._clients:
+            self._clients[model] = self._create_client(model)
+        return self._clients[model]
+
+    @property
+    def chat(self) -> MockChat:
+        """Access the chat resource."""
+        return self._chat

lm_deluge/models/__init__.py

@@ -15,7 +15,9 @@ from .fireworks import FIREWORKS_MODELS
 from .google import GOOGLE_MODELS
 from .grok import XAI_MODELS
 from .groq import GROQ_MODELS
+from .kimi import KIMI_MODELS
 from .meta import META_MODELS
+from .minimax import MINIMAX_MODELS
 from .mistral import MISTRAL_MODELS
 from .openai import OPENAI_MODELS
 from .openrouter import OPENROUTER_MODELS
@@ -38,9 +40,9 @@ class APIModel:
     supports_responses: bool = False
     reasoning_model: bool = False
     regions: list[str] | dict[str, int] = field(default_factory=list)
-    tokens_per_minute: int | None = None
-    requests_per_minute: int | None = None
-    gpus: list[str] | None = None
+    # tokens_per_minute: int | None = None
+    # requests_per_minute: int | None = None
+    # gpus: list[str] | None = None
 
     @classmethod
     def from_registry(cls, name: str):
@@ -62,7 +64,7 @@ class APIModel:
             raise ValueError("no regions to sample")
         random.sample(regions, 1, counts=weights)[0]
 
-    def make_request(self, context: RequestContext):  # -> "APIRequestBase"
+    def make_request(self, context: RequestContext):
         from ..api_requests.common import CLASSES
 
         api_spec = self.api_spec
@@ -97,8 +99,8 @@ def register_model(
     supports_responses: bool = False,
     reasoning_model: bool = False,
     regions: list[str] | dict[str, int] = field(default_factory=list),
-    tokens_per_minute: int | None = None,
-    requests_per_minute: int | None = None,
+    # tokens_per_minute: int | None = None,
+    # requests_per_minute: int | None = None,
 ) -> APIModel:
     """Register a model configuration and return the created APIModel."""
     model = APIModel(
@@ -116,8 +118,8 @@ def register_model(
         supports_responses=supports_responses,
         reasoning_model=reasoning_model,
         regions=regions,
-        tokens_per_minute=tokens_per_minute,
-        requests_per_minute=requests_per_minute,
+        # tokens_per_minute=tokens_per_minute,
+        # requests_per_minute=requests_per_minute,
     )
     registry[model.id] = model
     return model
@@ -132,7 +134,9 @@ for model_dict in [
     FIREWORKS_MODELS,
     GOOGLE_MODELS,
     XAI_MODELS,
+    KIMI_MODELS,
     META_MODELS,
+    MINIMAX_MODELS,
     MISTRAL_MODELS,
     OPENAI_MODELS,
     OPENROUTER_MODELS,

lm_deluge/models/anthropic.py

@@ -10,6 +10,18 @@ ANTHROPIC_MODELS = {
     #                                                                   ░███
     #                                                                   █████
     #
+    "claude-4.5-haiku": {
+        "id": "claude-4.5-haiku",
+        "name": "claude-haiku-4-5-20251001",
+        "api_base": "https://api.anthropic.com/v1",
+        "api_key_env_var": "ANTHROPIC_API_KEY",
+        "supports_json": False,
+        "api_spec": "anthropic",
+        "input_cost": 1.0,
+        "cached_input_cost": 0.10,
+        "cache_write_cost": 1.25,
+        "output_cost": 3.0,
+    },
     "claude-4.5-sonnet": {
         "id": "claude-4.5-sonnet",
         "name": "claude-sonnet-4-5-20250929",
@@ -21,8 +33,6 @@ ANTHROPIC_MODELS = {
         "cached_input_cost": 0.30,
         "cache_write_cost": 3.75,
         "output_cost": 15.0,
-        "requests_per_minute": 4_000,
-        "tokens_per_minute": 400_000,
     },
     "claude-4.1-opus": {
         "id": "claude-4.1-opus",
@@ -35,8 +45,6 @@ ANTHROPIC_MODELS = {
         "cached_input_cost": 1.50,
         "cache_write_cost": 18.75,
         "output_cost": 75.0,
-        "requests_per_minute": 4_000,
-        "tokens_per_minute": 400_000,
         "reasoning_model": True,
     },
     "claude-4-opus": {
@@ -50,8 +58,6 @@ ANTHROPIC_MODELS = {
         "cached_input_cost": 1.50,
         "cache_write_cost": 18.75,
         "output_cost": 75.0,
-        "requests_per_minute": 4_000,
-        "tokens_per_minute": 400_000,
         "reasoning_model": True,
     },
     "claude-4-sonnet": {
@@ -65,8 +71,6 @@ ANTHROPIC_MODELS = {
         "cached_input_cost": 0.30,
         "cache_write_cost": 3.75,
         "output_cost": 15.0,
-        "requests_per_minute": 4_000,
-        "tokens_per_minute": 400_000,
     },
     "claude-3.7-sonnet": {
         "id": "claude-3.7-sonnet",
@@ -79,8 +83,6 @@ ANTHROPIC_MODELS = {
         "cached_input_cost": 0.30,
         "cache_write_cost": 3.75,
         "output_cost": 15.0,
-        "requests_per_minute": 4_000,
-        "tokens_per_minute": 400_000,
         "reasoning_model": True,
     },
     "claude-3.6-sonnet": {
@@ -94,8 +96,6 @@ ANTHROPIC_MODELS = {
         "cached_input_cost": 0.30,
         "cache_write_cost": 3.75,
         "output_cost": 15.0,
-        "requests_per_minute": 4_000,
-        "tokens_per_minute": 400_000,
     },
     "claude-3.5-sonnet": {
         "id": "claude-3.5-sonnet",
@@ -108,8 +108,6 @@ ANTHROPIC_MODELS = {
         "cached_input_cost": 0.30,
         "cache_write_cost": 3.75,
         "output_cost": 15.0,
-        "requests_per_minute": 4_000,
-        "tokens_per_minute": 400_000,
     },
     "claude-3-opus": {
         "id": "claude-3-opus",
@@ -120,8 +118,6 @@ ANTHROPIC_MODELS = {
         "api_spec": "anthropic",
         "input_cost": 15.0,
         "output_cost": 75.0,
-        "requests_per_minute": 4_000,
-        "tokens_per_minute": 400_000,
     },
     "claude-3.5-haiku": {
         "id": "claude-3.5-haiku",
@@ -134,8 +130,6 @@ ANTHROPIC_MODELS = {
         "cached_input_cost": 0.08,
         "cache_write_cost": 1.00,
         "output_cost": 4.00,
-        "requests_per_minute": 20_000,
-        "tokens_per_minute": 4_000_000,  # supposed to be this but they fucked up
     },
     "claude-3-haiku": {
         "id": "claude-3-haiku",
@@ -148,7 +142,5 @@ ANTHROPIC_MODELS = {
         "cache_write_cost": 0.30,
         "cached_input_cost": 0.03,
         "output_cost": 1.25,
-        "requests_per_minute": 10_000,
-        "tokens_per_minute": 4_000_000,  # supposed to be this but they fucked up
     },
 }