devorch 0.1.2__py3-none-any.whl

providers/kimi.py

@@ -0,0 +1,181 @@
+"""
+Kimi (Moonshot AI) provider.
+
+Moonshot AI's Kimi model with extremely long context (200K+ tokens).
+Uses OpenAI-compatible API.
+"""
+
+import json
+
+from openai import OpenAI
+
+from providers.base import LLMProvider, ModelInfo
+from schemas.message import LLMResponse, Message, ToolCall
+
+
+class KimiProvider(LLMProvider):
+    name = "kimi"
+
+    DEFAULT_MODELS = [
+        "moonshot-v1-8k",
+        "moonshot-v1-32k",
+        "moonshot-v1-128k",
+    ]
+
+    def __init__(
+        self,
+        model: str = "moonshot-v1-32k",
+        api_key: str | None = None,
+        base_url: str = "https://api.moonshot.cn/v1",
+    ):
+        """
+        Initialize Kimi provider.
+
+        Args:
+            model: Model to use (moonshot-v1-8k, moonshot-v1-32k, moonshot-v1-128k)
+            api_key: Moonshot API key (defaults to MOONSHOT_API_KEY env var)
+            base_url: Base URL for the API
+        """
+        self.client = OpenAI(api_key=api_key, base_url=base_url)
+        self.model = model
+
+    def list_models(self) -> list[ModelInfo]:
+        """Fetch available models from Moonshot AI API."""
+        try:
+            response = self.client.models.list()
+            models = []
+            for model in response.data:
+                # Extract context length from model ID if possible
+                context = None
+                if "8k" in model.id.lower():
+                    context = 8000
+                elif "32k" in model.id.lower():
+                    context = 32000
+                elif "128k" in model.id.lower():
+                    context = 128000
+
+                models.append(
+                    ModelInfo(
+                        id=model.id,
+                        name=model.id,
+                        context_length=context,
+                        description=getattr(model, "description", None),
+                    )
+                )
+            return models if models else self._get_default_models()
+        except Exception:
+            # Fallback to default models if API call fails
+            return self._get_default_models()
+
+    def _get_default_models(self) -> list[ModelInfo]:
+        """Get hardcoded default models as fallback."""
+        return [
+            ModelInfo(
+                id="moonshot-v1-8k",
+                name="Kimi 8K",
+                context_length=8000,
+                description="Fast model with 8K context",
+            ),
+            ModelInfo(
+                id="moonshot-v1-32k",
+                name="Kimi 32K",
+                context_length=32000,
+                description="Balanced model with 32K context",
+            ),
+            ModelInfo(
+                id="moonshot-v1-128k",
+                name="Kimi 128K",
+                context_length=128000,
+                description="Long context model with 128K tokens",
+            ),
+        ]
+
+    def generate(
+        self,
+        messages: list[Message],
+        tools: list | None = None,
+        stream: bool = False,
+    ) -> LLMResponse:
+        # Format messages for OpenAI-compatible API
+        formatted_messages = []
+        for msg in messages:
+            if msg.role == "tool":
+                formatted_msg = {
+                    "role": "tool",
+                    "content": msg.content,
+                    "tool_call_id": msg.tool_call_id or msg.name,
+                }
+            elif msg.role == "assistant" and msg.metadata and msg.metadata.get("tool_calls"):
+                formatted_msg = {
+                    "role": "assistant",
+                    "content": msg.content or "",
+                    "tool_calls": msg.metadata["tool_calls"],
+                }
+            else:
+                formatted_msg = {"role": msg.role, "content": msg.content}
+            formatted_messages.append(formatted_msg)
+
+        # Format tools
+        formatted_tools = None
+        if tools:
+            formatted_tools = []
+            for tool in tools:
+                formatted_tools.append(
+                    {
+                        "type": "function",
+                        "function": {
+                            "name": tool["name"],
+                            "description": tool["description"],
+                            "parameters": tool.get(
+                                "parameters", {"type": "object", "properties": {}}
+                            ),
+                        },
+                    }
+                )
+
+        response = self.client.chat.completions.create(
+            model=self.model,
+            messages=formatted_messages,
+            tools=formatted_tools,
+            temperature=0.0,
+        )
+
+        choice = response.choices[0]
+        message = choice.message
+
+        tool_calls = []
+        if message.tool_calls:
+            for tc in message.tool_calls:
+                tool_calls.append(
+                    ToolCall(
+                        id=tc.id,
+                        name=tc.function.name,
+                        arguments=json.loads(tc.function.arguments),
+                    )
+                )
+
+        return LLMResponse(
+            content=message.content or "",
+            tool_calls=tool_calls,
+            metadata={
+                "model": response.model,
+                "usage": {
+                    "prompt_tokens": response.usage.prompt_tokens if response.usage else 0,
+                    "completion_tokens": (
+                        response.usage.completion_tokens if response.usage else 0
+                    ),
+                    "total_tokens": response.usage.total_tokens if response.usage else 0,
+                },
+                "tool_calls": [
+                    {
+                        "id": tc.id,
+                        "type": "function",
+                        "function": {
+                            "name": tc.function.name,
+                            "arguments": tc.function.arguments,
+                        },
+                    }
+                    for tc in (message.tool_calls or [])
+                ],
+            },
+        )

providers/lmstudio.py

@@ -0,0 +1,147 @@
+"""
+LM Studio Provider - Local models via OpenAI-compatible API
+https://lmstudio.ai/
+"""
+
+import json
+
+import httpx
+
+from providers.base import LLMProvider, ModelInfo
+from schemas.message import LLMResponse, Message, ToolCall
+
+
+class LMStudioProvider(LLMProvider):
+    """
+    LM Studio provider for running local models.
+    Uses OpenAI-compatible API format.
+    """
+
+    name = "lmstudio"
+
+    DEFAULT_MODELS = [
+        "local-model",  # LM Studio uses the loaded model
+    ]
+
+    def __init__(
+        self,
+        model: str = "local-model",
+        api_key: str | None = None,  # Not needed for local
+        base_url: str = "http://localhost:1234/v1",
+    ):
+        self.model = model
+        self.api_key = api_key or "lm-studio"  # Placeholder
+        self.base_url = base_url.rstrip("/")
+        self.client = httpx.Client(timeout=300.0)  # Longer timeout for local
+
+    def _get_headers(self) -> dict:
+        return {
+            "Authorization": f"Bearer {self.api_key}",
+            "Content-Type": "application/json",
+        }
+
+    def list_models(self) -> list[ModelInfo]:
+        """Fetch loaded models from LM Studio."""
+        try:
+            response = self.client.get(f"{self.base_url}/models", headers=self._get_headers())
+            response.raise_for_status()
+            data = response.json()
+
+            models = []
+            for model in data.get("data", []):
+                models.append(
+                    ModelInfo(
+                        id=model.get("id"),
+                        name=model.get("id"),
+                    )
+                )
+
+            return models if models else [ModelInfo(id="local-model", name="Local Model")]
+
+        except Exception:
+            return [ModelInfo(id="local-model", name="Local Model (LM Studio)")]
+
+    def generate(
+        self,
+        messages: list[Message],
+        tools: list | None = None,
+        stream: bool = False,
+    ) -> LLMResponse:
+        # Format messages for LM Studio API
+        formatted_messages = []
+        for msg in messages:
+            if msg.role == "tool":
+                formatted_msg = {
+                    "role": "tool",
+                    "content": msg.content,
+                    "tool_call_id": msg.tool_call_id or msg.name,
+                }
+            elif msg.role == "assistant" and msg.metadata and msg.metadata.get("tool_calls"):
+                # Preserve tool_calls in assistant messages
+                formatted_msg = {
+                    "role": "assistant",
+                    "content": msg.content or "",
+                    "tool_calls": msg.metadata["tool_calls"],
+                }
+            else:
+                formatted_msg = {"role": msg.role, "content": msg.content}
+            formatted_messages.append(formatted_msg)
+
+        # Format tools (if model supports it)
+        formatted_tools = None
+        if tools:
+            formatted_tools = []
+            for tool in tools:
+                formatted_tools.append(
+                    {
+                        "type": "function",
+                        "function": {
+                            "name": tool["name"],
+                            "description": tool["description"],
+                            "parameters": tool.get(
+                                "parameters", {"type": "object", "properties": {}}
+                            ),
+                        },
+                    }
+                )
+
+        payload = {
+            "model": self.model,
+            "messages": formatted_messages,
+            "temperature": 0.0,
+        }
+
+        if formatted_tools:
+            payload["tools"] = formatted_tools
+
+        response = self.client.post(
+            f"{self.base_url}/chat/completions", headers=self._get_headers(), json=payload
+        )
+        response.raise_for_status()
+        data = response.json()
+
+        choice = data["choices"][0]
+        message = choice["message"]
+
+        tool_calls = []
+        if message.get("tool_calls"):
+            for tc in message["tool_calls"]:
+                args = (
+                    json.loads(tc["function"]["arguments"])
+                    if tc["function"].get("arguments")
+                    else {}
+                )
+                tool_call = ToolCall(
+                    name=tc["function"]["name"],
+                    arguments=args,
+                    id=tc.get("id", tc["function"]["name"]),
+                )
+                tool_calls.append(tool_call)
+
+        content = message.get("content") or "Calling tool..."
+
+        return LLMResponse(
+            message=Message(role="assistant", content=content),
+            tool_calls=tool_calls if tool_calls else None,
+            raw=data,
+        )

providers/local.py

@@ -0,0 +1,214 @@
+import json
+
+import httpx
+from openai import OpenAI
+
+from providers.base import LLMProvider, ModelInfo
+from schemas.message import LLMResponse, Message, ToolCall
+
+
+class LocalProvider(LLMProvider):
+    """Local LLM provider using Ollama's OpenAI-compatible API."""
+
+    name = "local"
+
+    # Models known to support function calling well
+    TOOL_CAPABLE_MODELS = [
+        "llama3.1",
+        "llama3.2",
+        "qwen2.5:7b",
+        "qwen2.5:14b",
+        "qwen2.5:32b",
+        "qwen2.5-coder",
+        "mistral",
+        "mixtral",
+        "command-r",
+        "firefunction",
+    ]
+
+    DEFAULT_MODELS = [
+        "llama3.1",
+        "llama3.2",
+        "llama3",
+        "codellama",
+        "mistral",
+        "mixtral",
+        "gemma2",
+        "qwen2.5",
+        "qwen2.5-coder",
+        "deepseek-coder-v2",
+    ]
+
+    def __init__(
+        self,
+        model: str | None = None,
+        base_url: str = "http://localhost:11434/v1",
+        api_key: str | None = None,
+    ):
+        self.base_url = base_url
+
+        # Ollama's OpenAI-compatible endpoint
+        self.client = OpenAI(
+            base_url=base_url,
+            api_key=api_key or "ollama",  # Placeholder, Ollama ignores this
+        )
+
+        # Auto-detect model if not specified
+        if model:
+            self.model = model
+        else:
+            self.model = self._detect_default_model()
+
+        # Track if we've warned about tool capability
+        self._tool_warning_shown = False
+
+    def _detect_default_model(self) -> str:
+        """Auto-detect the first available model from Ollama."""
+        try:
+            ollama_base = self.base_url.replace("/v1", "")
+            response = httpx.get(f"{ollama_base}/api/tags", timeout=5.0)
+            response.raise_for_status()
+            data = response.json()
+
+            models = data.get("models", [])
+            if models:
+                # Return the first available model
+                return models[0].get("name", "llama3")
+
+        except Exception:
+            pass
+
+        # Fallback to first default model
+        return self.DEFAULT_MODELS[0]
+
+    def _is_tool_capable(self, model_name: str) -> bool:
+        """Check if a model is known to support function calling."""
+        model_lower = model_name.lower()
+        for capable in self.TOOL_CAPABLE_MODELS:
+            if capable in model_lower:
+                return True
+        # Small models (under 3B) generally don't support tools well
+        if ":0.5b" in model_lower or ":1b" in model_lower or ":2b" in model_lower:
+            return False
+        return True  # Assume capable for unknown larger models
+
+    def list_models(self) -> list[ModelInfo]:
+        """Fetch models from local Ollama instance."""
+        try:
+            # Ollama API endpoint for listing models
+            ollama_base = self.base_url.replace("/v1", "")
+            response = httpx.get(f"{ollama_base}/api/tags", timeout=5.0)
+            response.raise_for_status()
+            data = response.json()
+
+            models = []
+            for model in data.get("models", []):
+                name = model.get("name")
+                tool_note = "" if self._is_tool_capable(name) else " (no tool support)"
+                models.append(
+                    ModelInfo(
+                        id=name,
+                        name=name,
+                        description=tool_note if tool_note else None,
+                    )
+                )
+
+            return models if models else [ModelInfo(id=m, name=m) for m in self.DEFAULT_MODELS]
+
+        except Exception:
+            return [ModelInfo(id=m, name=m + " (not pulled)") for m in self.DEFAULT_MODELS]
+
+    def generate(
+        self,
+        messages: list[Message],
+        tools: list | None = None,
+        stream: bool = False,
+    ) -> LLMResponse:
+        # Format messages (same as OpenAI)
+        formatted_messages = []
+        for msg in messages:
+            if msg.role == "tool":
+                formatted_msg = {
+                    "role": "tool",
+                    "content": msg.content,
+                    "tool_call_id": msg.tool_call_id or msg.name,
+                }
+            elif msg.role == "assistant" and msg.metadata and msg.metadata.get("tool_calls"):
+                # Preserve tool_calls in assistant messages
+                formatted_msg = {
+                    "role": "assistant",
+                    "content": msg.content or "",
+                    "tool_calls": msg.metadata["tool_calls"],
+                }
+            else:
+                formatted_msg = {"role": msg.role, "content": msg.content}
+            formatted_messages.append(formatted_msg)
+
+        # Format tools (same as OpenAI)
+        formatted_tools = None
+        if tools:
+            formatted_tools = []
+            for tool in tools:
+                formatted_tools.append(
+                    {
+                        "type": "function",
+                        "function": {
+                            "name": tool["name"],
+                            "description": tool["description"],
+                            "parameters": tool.get(
+                                "parameters", {"type": "object", "properties": {}}
+                            ),
+                        },
+                    }
+                )
+
+        # Warn if model may not support tools
+        if (
+            formatted_tools
+            and not self._tool_warning_shown
+            and not self._is_tool_capable(self.model)
+        ):
+            import sys
+
+            print(f"\n⚠️  Warning: {self.model} may not support function calling.", file=sys.stderr)
+            print(
+                "   For best results, use a larger model like llama3.1, qwen2.5:7b, or mistral.",
+                file=sys.stderr,
+            )
+            print("   Run: ollama pull llama3.1\n", file=sys.stderr)
+            self._tool_warning_shown = True
+
+        # Try with tools first, fall back to without if model doesn't support them
+        try:
+            kwargs = {"model": self.model, "messages": formatted_messages, "temperature": 0.0}
+            if formatted_tools:
+                kwargs["tools"] = formatted_tools
+
+            response = self.client.chat.completions.create(**kwargs)
+        except Exception as e:
+            error_str = str(e).lower()
+            # Retry without tools if the model doesn't support function calling
+            if formatted_tools and ("tool" in error_str or "function" in error_str):
+                response = self.client.chat.completions.create(
+                    model=self.model, messages=formatted_messages, temperature=0.0
+                )
+            else:
+                raise
+
+        choice = response.choices[0]
+        message = choice.message
+
+        tool_calls = []
+        if hasattr(message, "tool_calls") and message.tool_calls:
+            for tc in message.tool_calls:
+                args = json.loads(tc.function.arguments) if tc.function.arguments else {}
+                tool_call = ToolCall(name=tc.function.name, arguments=args, id=tc.id)
+                tool_calls.append(tool_call)
+
+        content = message.content if message.content else "Calling tool..."
+
+        return LLMResponse(
+            message=Message(role="assistant", content=content),
+            tool_calls=tool_calls if tool_calls else None,
+            raw=response,
+        )

providers/mistral.py

@@ -0,0 +1,161 @@
+"""
+Mistral AI Provider
+https://mistral.ai/
+"""
+
+import json
+
+import httpx
+
+from providers.base import LLMProvider, ModelInfo
+from schemas.message import LLMResponse, Message, ToolCall
+
+
+class MistralProvider(LLMProvider):
+    """
+    Mistral AI provider for Mistral models.
+    """
+
+    name = "mistral"
+
+    DEFAULT_MODELS = [
+        "mistral-large-latest",
+        "mistral-medium-latest",
+        "mistral-small-latest",
+        "open-mistral-7b",
+        "open-mixtral-8x7b",
+        "open-mixtral-8x22b",
+        "codestral-latest",
+    ]
+
+    BASE_URL = "https://api.mistral.ai/v1"
+
+    def __init__(
+        self,
+        model: str = "mistral-large-latest",
+        api_key: str | None = None,
+    ):
+        self.model = model
+        self.api_key = api_key
+        self.client = httpx.Client(timeout=120.0)
+
+    def _get_headers(self) -> dict:
+        return {
+            "Authorization": f"Bearer {self.api_key}",
+            "Content-Type": "application/json",
+        }
+
+    def list_models(self) -> list[ModelInfo]:
+        """Fetch available models from Mistral API."""
+        try:
+            response = self.client.get(f"{self.BASE_URL}/models", headers=self._get_headers())
+            response.raise_for_status()
+            data = response.json()
+
+            models = []
+            for model in data.get("data", []):
+                models.append(
+                    ModelInfo(
+                        id=model.get("id"),
+                        name=model.get("id"),
+                    )
+                )
+
+            models.sort(key=lambda m: m.id)
+            return models if models else [ModelInfo(id=m, name=m) for m in self.DEFAULT_MODELS]
+
+        except Exception:
+            return [ModelInfo(id=m, name=m) for m in self.DEFAULT_MODELS]
+
+    def generate(
+        self,
+        messages: list[Message],
+        tools: list | None = None,
+        stream: bool = False,
+    ) -> LLMResponse:
+        # Format messages for Mistral API
+        formatted_messages = []
+        for msg in messages:
+            if msg.role == "tool":
+                # Mistral requires tool results with specific format
+                formatted_msg = {
+                    "role": "tool",
+                    "content": msg.content,
+                    "tool_call_id": msg.tool_call_id or msg.name,
+                    "name": msg.name,
+                }
+            elif (
+                msg.role == "assistant"
+                and hasattr(msg, "metadata")
+                and msg.metadata
+                and msg.metadata.get("tool_calls")
+            ):
+                # Preserve tool_calls in assistant messages
+                formatted_msg = {
+                    "role": "assistant",
+                    "content": msg.content or "",
+                    "tool_calls": msg.metadata["tool_calls"],
+                }
+            else:
+                formatted_msg = {"role": msg.role, "content": msg.content}
+            formatted_messages.append(formatted_msg)
+
+        # Format tools
+        formatted_tools = None
+        if tools:
+            formatted_tools = []
+            for tool in tools:
+                formatted_tools.append(
+                    {
+                        "type": "function",
+                        "function": {
+                            "name": tool["name"],
+                            "description": tool["description"],
+                            "parameters": tool.get(
+                                "parameters", {"type": "object", "properties": {}}
+                            ),
+                        },
+                    }
+                )
+
+        payload = {
+            "model": self.model,
+            "messages": formatted_messages,
+            "temperature": 0.0,
+        }
+
+        if formatted_tools:
+            payload["tools"] = formatted_tools
+            payload["tool_choice"] = "auto"
+
+        response = self.client.post(
+            f"{self.BASE_URL}/chat/completions", headers=self._get_headers(), json=payload
+        )
+        response.raise_for_status()
+        data = response.json()
+
+        choice = data["choices"][0]
+        message = choice["message"]
+
+        tool_calls = []
+        if message.get("tool_calls"):
+            for tc in message["tool_calls"]:
+                args = (
+                    json.loads(tc["function"]["arguments"])
+                    if tc["function"].get("arguments")
+                    else {}
+                )
+                tool_call = ToolCall(
+                    name=tc["function"]["name"],
+                    arguments=args,
+                    id=tc.get("id", tc["function"]["name"]),
+                )
+                tool_calls.append(tool_call)
+
+        content = message.get("content") or "Calling tool..."
+
+        return LLMResponse(
+            message=Message(role="assistant", content=content),
+            tool_calls=tool_calls if tool_calls else None,
+            raw=data,
+        )