lm-deluge 0.0.67__py3-none-any.whl → 0.0.90__py3-none-any.whl

lm_deluge/request_context.py

@@ -1,11 +1,14 @@
 from dataclasses import dataclass, field
 from functools import cached_property
-from typing import Any, Callable
+from typing import Any, Callable, Sequence, TYPE_CHECKING
 
 from .config import SamplingParams
 from .prompt import CachePattern, Conversation
 from .tracker import StatusTracker
 
+if TYPE_CHECKING:
+    from pydantic import BaseModel
+
 
 @dataclass
 class RequestContext:
@@ -31,7 +34,8 @@ class RequestContext:
     callback: Callable | None = None
 
     # Optional features
-    tools: list | None = None
+    tools: Sequence[Any] | None = None
+    output_schema: "type[BaseModel] | dict | None" = None
     cache: CachePattern | None = None
     use_responses_api: bool = False
     background: bool = False
@@ -66,6 +70,7 @@ class RequestContext:
             "results_arr": self.results_arr,
             "callback": self.callback,
             "tools": self.tools,
+            "output_schema": self.output_schema,
             "cache": self.cache,
             "use_responses_api": self.use_responses_api,
             "background": self.background,

lm_deluge/server/__init__.py

@@ -0,0 +1,24 @@
+"""
+LM-Deluge Proxy Server
+
+A FastAPI-based proxy server that exposes OpenAI-compatible and
+Anthropic-compatible API endpoints, routing requests through lm-deluge
+to any supported provider.
+
+Usage:
+    python -m lm_deluge.server
+
+Environment Variables:
+    DELUGE_PROXY_API_KEY: Optional API key that clients must provide
+    DELUGE_PROXY_PORT: Port to run on (default: 8000)
+    DELUGE_PROXY_HOST: Host to bind (default: 0.0.0.0)
+    DELUGE_PROXY_LOG_REQUESTS: Log full incoming proxy requests when set
+    DELUGE_PROXY_LOG_PROVIDER_REQUESTS: Log outbound provider requests when set
+
+    Provider keys (same as LLMClient):
+    OPENAI_API_KEY, ANTHROPIC_API_KEY, GOOGLE_API_KEY, etc.
+"""
+
+from .app import create_app
+
+__all__ = ["create_app"]

lm_deluge/server/__main__.py

@@ -0,0 +1,144 @@
+"""
+Entry point for running the server with: python -m lm_deluge.server
+
+Usage:
+    python -m lm_deluge.server [--host HOST] [--port PORT] [--config PATH]
+
+Environment Variables:
+    DELUGE_PROXY_HOST: Host to bind (default: 0.0.0.0)
+    DELUGE_PROXY_PORT: Port to run on (default: 8000)
+    DELUGE_PROXY_API_KEY: Optional API key that clients must provide
+    DELUGE_PROXY_LOG_REQUESTS: Log full incoming proxy requests when set
+    DELUGE_PROXY_LOG_PROVIDER_REQUESTS: Log outbound provider requests when set
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+
+import json5
+
+from .app import create_app
+from .model_policy import build_policy
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="LM-Deluge Proxy Server - OpenAI and Anthropic compatible API proxy"
+    )
+    parser.add_argument(
+        "--host",
+        type=str,
+        default=os.getenv("DELUGE_PROXY_HOST", "0.0.0.0"),
+        help="Host to bind (default: 0.0.0.0)",
+    )
+    parser.add_argument(
+        "--port",
+        type=int,
+        default=int(os.getenv("DELUGE_PROXY_PORT", "8000")),
+        help="Port to run on (default: 8000)",
+    )
+    parser.add_argument(
+        "--reload",
+        action="store_true",
+        help="Enable auto-reload for development",
+    )
+    parser.add_argument(
+        "--config",
+        type=str,
+        help="Path to proxy YAML config file",
+    )
+    parser.add_argument(
+        "--mode",
+        type=str,
+        choices=["allow_user_pick", "force_default", "alias_only"],
+        help="Override model policy mode",
+    )
+    parser.add_argument(
+        "--allow-model",
+        action="append",
+        dest="allowed_models",
+        help="Allow a model id (repeat to allow multiple models)",
+    )
+    parser.add_argument(
+        "--default-model",
+        type=str,
+        help="Default model or alias for force_default mode",
+    )
+    parser.add_argument(
+        "--routes",
+        type=str,
+        help="JSON5 string defining route aliases and strategies",
+    )
+    alias_group = parser.add_mutually_exclusive_group()
+    alias_group.add_argument(
+        "--expose-aliases",
+        action="store_true",
+        help="Expose route aliases in /v1/models",
+    )
+    alias_group.add_argument(
+        "--hide-aliases",
+        action="store_true",
+        help="Hide route aliases in /v1/models",
+    )
+
+    args = parser.parse_args()
+
+    # Import here to avoid loading uvicorn unless needed
+    try:
+        import uvicorn
+    except ImportError:
+        print("Error: uvicorn is required to run the server.")
+        print("Install it with: pip install lm-deluge[server]")
+        raise SystemExit(1)
+
+    policy_overrides = {}
+    if args.mode:
+        policy_overrides["mode"] = args.mode
+    if args.allowed_models:
+        policy_overrides["allowed_models"] = args.allowed_models
+    if args.default_model:
+        policy_overrides["default_model"] = args.default_model
+    if args.routes:
+        try:
+            policy_overrides["routes"] = json5.loads(args.routes)
+        except Exception as exc:
+            print(f"Error parsing --routes JSON5: {exc}")
+            raise SystemExit(2)
+    if args.expose_aliases:
+        policy_overrides["expose_aliases"] = True
+    elif args.hide_aliases:
+        policy_overrides["expose_aliases"] = False
+
+    try:
+        policy = build_policy(path=args.config, overrides=policy_overrides)
+    except Exception as exc:
+        print(f"Invalid proxy model policy: {exc}")
+        raise SystemExit(2)
+    app = create_app(policy)
+
+    print(f"Starting LM-Deluge Proxy Server on {args.host}:{args.port}")
+    print("Endpoints:")
+    print(f"  OpenAI:    http://{args.host}:{args.port}/v1/chat/completions")
+    print(f"  Anthropic: http://{args.host}:{args.port}/v1/messages")
+    print(f"  Models:    http://{args.host}:{args.port}/v1/models")
+    print(f"  Health:    http://{args.host}:{args.port}/health")
+
+    if os.getenv("DELUGE_PROXY_API_KEY"):
+        print("\nAuthentication: ENABLED (DELUGE_PROXY_API_KEY is set)")
+    else:
+        print("\nAuthentication: DISABLED (set DELUGE_PROXY_API_KEY to enable)")
+
+    print()
+
+    uvicorn.run(
+        app,
+        host=args.host,
+        port=args.port,
+        reload=args.reload,
+    )
+
+
+if __name__ == "__main__":
+    main()

lm_deluge/server/adapters.py

@@ -0,0 +1,369 @@
+"""
+Adapters for converting between API formats and lm-deluge types.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from lm_deluge.api_requests.response import APIResponse
+from lm_deluge.config import SamplingParams
+from lm_deluge.prompt import Conversation, Text, ThoughtSignature, Thinking, ToolCall
+from lm_deluge.tool import Tool
+
+from .models_anthropic import (
+    AnthropicMessagesRequest,
+    AnthropicMessagesResponse,
+    AnthropicResponseContentBlock,
+    AnthropicUsage,
+)
+from .models_openai import (
+    OpenAIChatCompletionsRequest,
+    OpenAIChatCompletionsResponse,
+    OpenAIChoice,
+    OpenAIFunctionCall,
+    OpenAIResponseMessage,
+    OpenAIToolCall,
+    OpenAIUsage,
+)
+
+
+# ============================================================================
+# OpenAI Request Conversion
+# ============================================================================
+
+
+def openai_request_to_conversation(req: OpenAIChatCompletionsRequest) -> Conversation:
+    """Convert OpenAI request messages to lm-deluge Conversation."""
+    # Use existing conversion - it handles all the complexity
+    messages_dicts = [msg.model_dump(exclude_none=True) for msg in req.messages]
+    return Conversation.from_openai_chat(messages_dicts)
+
+
+def openai_request_to_sampling_params(
+    req: OpenAIChatCompletionsRequest,
+) -> SamplingParams:
+    """Extract SamplingParams from OpenAI request."""
+    params: dict[str, Any] = {}
+
+    if req.temperature is not None:
+        params["temperature"] = req.temperature
+    if req.top_p is not None:
+        params["top_p"] = req.top_p
+    if req.max_completion_tokens is not None:
+        params["max_new_tokens"] = req.max_completion_tokens
+    elif req.max_tokens is not None:
+        params["max_new_tokens"] = req.max_tokens
+    if req.reasoning_effort is not None:
+        params["reasoning_effort"] = req.reasoning_effort
+    if req.response_format and req.response_format.get("type") == "json_object":
+        params["json_mode"] = True
+    if req.logprobs:
+        params["logprobs"] = True
+    if req.top_logprobs is not None:
+        params["top_logprobs"] = req.top_logprobs
+
+    return SamplingParams(**params)
+
+
+def openai_tools_to_lm_deluge(tools: list[Any]) -> list[Tool]:
+    """Convert OpenAI tool definitions to lm-deluge Tools."""
+    lm_tools = []
+    for tool in tools:
+        if hasattr(tool, "model_dump"):
+            tool = tool.model_dump()
+        if tool.get("type") == "function":
+            func = tool["function"]
+            params_schema = func.get("parameters") or {}
+            properties = params_schema.get("properties", {})
+            required = params_schema.get("required", [])
+
+            lm_tool = Tool(
+                name=func["name"],
+                description=func.get("description"),
+                parameters=properties if properties else None,
+                required=required,
+            )
+            lm_tools.append(lm_tool)
+    return lm_tools
+
+
+def _signature_for_provider(
+    signature: ThoughtSignature | str | None, provider: str
+) -> str | None:
+    if signature is None:
+        return None
+    if isinstance(signature, ThoughtSignature):
+        if signature.provider is None or signature.provider == provider:
+            return signature.value
+        return None
+    return signature
+
+
+# ============================================================================
+# OpenAI Response Conversion
+# ============================================================================
+
+
+def api_response_to_openai(
+    response: APIResponse, model: str
+) -> OpenAIChatCompletionsResponse:
+    """Convert lm-deluge APIResponse to OpenAI ChatCompletion format."""
+    # Handle error responses
+    if response.is_error:
+        message = OpenAIResponseMessage(
+            role="assistant",
+            content=response.error_message or "An error occurred",
+        )
+        choice = OpenAIChoice(
+            index=0,
+            message=message,
+            finish_reason="stop",
+        )
+        return OpenAIChatCompletionsResponse(
+            model=model,
+            choices=[choice],
+            usage=None,
+        )
+
+    # Extract content from response
+    content_text: str | None = None
+    tool_calls: list[OpenAIToolCall] | None = None
+
+    if response.content:
+        # Extract text parts
+        text_parts = [p.text for p in response.content.parts if isinstance(p, Text)]
+        if text_parts:
+            content_text = "".join(text_parts)
+
+        # Extract tool calls
+        tool_call_parts = [p for p in response.content.parts if isinstance(p, ToolCall)]
+        if tool_call_parts:
+            tool_calls = [
+                OpenAIToolCall(
+                    id=tc.id,
+                    type="function",
+                    function=OpenAIFunctionCall(
+                        name=tc.name,
+                        arguments=json.dumps(tc.arguments)
+                        if isinstance(tc.arguments, dict)
+                        else tc.arguments,
+                    ),
+                )
+                for tc in tool_call_parts
+            ]
+
+    # Create message
+    message = OpenAIResponseMessage(
+        role="assistant",
+        content=content_text,
+        tool_calls=tool_calls,
+    )
+
+    # Create choice
+    choice = OpenAIChoice(
+        index=0,
+        message=message,
+        finish_reason=response.finish_reason or "stop",
+    )
+
+    # Create usage
+    usage = None
+    if response.usage:
+        usage = OpenAIUsage(
+            prompt_tokens=response.usage.input_tokens,
+            completion_tokens=response.usage.output_tokens,
+            total_tokens=response.usage.input_tokens + response.usage.output_tokens,
+        )
+
+    return OpenAIChatCompletionsResponse(
+        model=model,
+        choices=[choice],
+        usage=usage,
+    )
+
+
+# ============================================================================
+# Anthropic Request Conversion
+# ============================================================================
+
+
+def anthropic_request_to_conversation(req: AnthropicMessagesRequest) -> Conversation:
+    """Convert Anthropic request messages to lm-deluge Conversation."""
+
+    def _dump(value: Any) -> Any:
+        if hasattr(value, "model_dump"):
+            return value.model_dump(exclude_none=True)
+        return value
+
+    messages = [_dump(msg) for msg in req.messages]
+    system = req.system
+    if isinstance(system, list):
+        system = [_dump(block) for block in system]
+
+    return Conversation.from_anthropic(messages, system=system)
+
+
+def anthropic_request_to_sampling_params(
+    req: AnthropicMessagesRequest,
+) -> SamplingParams:
+    """Extract SamplingParams from Anthropic request."""
+    params: dict[str, Any] = {
+        "max_new_tokens": req.max_tokens,
+    }
+
+    if req.temperature is not None:
+        params["temperature"] = req.temperature
+    if req.top_p is not None:
+        params["top_p"] = req.top_p
+    if isinstance(req.thinking, dict):
+        thinking_type = req.thinking.get("type")
+        if thinking_type == "enabled":
+            budget_tokens = req.thinking.get("budget_tokens")
+            if isinstance(budget_tokens, int):
+                params["thinking_budget"] = budget_tokens
+        elif thinking_type == "disabled":
+            params["thinking_budget"] = 0
+
+    return SamplingParams(**params)
+
+
+def anthropic_tools_to_lm_deluge(tools: list[Any]) -> list[Tool]:
+    """Convert Anthropic tool definitions to lm-deluge Tools."""
+    lm_tools = []
+    for tool in tools:
+        if hasattr(tool, "model_dump"):
+            tool = tool.model_dump()
+
+        input_schema = tool.get("input_schema") or {}
+        properties = input_schema.get("properties", {})
+        required = input_schema.get("required", [])
+
+        lm_tool = Tool(
+            name=tool["name"],
+            description=tool.get("description"),
+            parameters=properties if properties else None,
+            required=required,
+        )
+        lm_tools.append(lm_tool)
+    return lm_tools
+
+
+# ============================================================================
+# Anthropic Response Conversion
+# ============================================================================
+
+
+def api_response_to_anthropic(
+    response: APIResponse, model: str
+) -> AnthropicMessagesResponse:
+    """Convert lm-deluge APIResponse to Anthropic Messages format."""
+
+    def _map_stop_reason(value: str | None) -> str:
+        if not value:
+            return "end_turn"
+        if value in {"end_turn", "max_tokens", "stop_sequence", "tool_use"}:
+            return value
+        return {
+            "stop": "end_turn",
+            "length": "max_tokens",
+            "tool_calls": "tool_use",
+        }.get(value, "end_turn")
+
+    # Handle error responses
+    if response.is_error:
+        content = [
+            AnthropicResponseContentBlock(
+                type="text",
+                text=response.error_message or "An error occurred",
+            )
+        ]
+        return AnthropicMessagesResponse(
+            model=model,
+            content=content,
+            stop_reason="end_turn",
+            usage=AnthropicUsage(input_tokens=0, output_tokens=0),
+        )
+
+    # Build content blocks
+    content_blocks: list[AnthropicResponseContentBlock] = []
+
+    last_signature = None
+    if response.content:
+        for part in response.content.parts:
+            if isinstance(part, Text):
+                content_blocks.append(
+                    AnthropicResponseContentBlock(type="text", text=part.text)
+                )
+            elif isinstance(part, ToolCall):
+                signature = _signature_for_provider(
+                    part.thought_signature,
+                    "anthropic",
+                )
+                if signature and signature != last_signature:
+                    content_blocks.append(
+                        AnthropicResponseContentBlock(
+                            type="thinking",
+                            thinking="",
+                            signature=signature,
+                        )
+                    )
+                    last_signature = signature
+                content_blocks.append(
+                    AnthropicResponseContentBlock(
+                        type="tool_use",
+                        id=part.id,
+                        name=part.name,
+                        input=part.arguments,
+                    )
+                )
+            elif isinstance(part, Thinking):
+                signature = _signature_for_provider(
+                    part.thought_signature,
+                    "anthropic",
+                )
+                if signature is None and part.raw_payload is None:
+                    continue
+                content_blocks.append(
+                    AnthropicResponseContentBlock(
+                        type="thinking",
+                        thinking=part.content,
+                        signature=signature,
+                    )
+                )
+                if signature:
+                    last_signature = signature
+
+    # Ensure at least one content block
+    if not content_blocks:
+        content_blocks.append(AnthropicResponseContentBlock(type="text", text=""))
+
+    # Map finish reason
+    raw_stop_reason = None
+    raw_stop_sequence = None
+    if isinstance(response.raw_response, dict):
+        raw_stop_reason = response.raw_response.get("stop_reason")
+        raw_stop_sequence = response.raw_response.get("stop_sequence")
+
+    stop_reason = _map_stop_reason(raw_stop_reason or response.finish_reason)
+
+    # Build usage (including cache tokens if present)
+    usage = AnthropicUsage(
+        input_tokens=response.usage.input_tokens if response.usage else 0,
+        output_tokens=response.usage.output_tokens if response.usage else 0,
+        cache_creation_input_tokens=response.usage.cache_write_tokens
+        if response.usage and response.usage.cache_write_tokens
+        else None,
+        cache_read_input_tokens=response.usage.cache_read_tokens
+        if response.usage and response.usage.cache_read_tokens
+        else None,
+    )
+
+    return AnthropicMessagesResponse(
+        model=model,
+        content=content_blocks,
+        stop_reason=stop_reason,
+        stop_sequence=raw_stop_sequence if isinstance(raw_stop_sequence, str) else None,
+        usage=usage,
+    )