realtimex-deeptutor 0.5.0.post1__py3-none-any.whl

src/services/llm/capabilities.py

@@ -0,0 +1,313 @@
+"""
+Provider Capabilities
+=====================
+
+Centralized configuration for LLM provider capabilities.
+This replaces scattered hardcoded checks throughout the codebase.
+
+Usage:
+    from src.services.llm.capabilities import get_capability, supports_response_format
+
+    # Check if a provider supports response_format
+    if supports_response_format(binding, model):
+        kwargs["response_format"] = {"type": "json_object"}
+
+    # Generic capability check
+    if get_capability(binding, "streaming", default=True):
+        # use streaming
+"""
+
+from typing import Any, Optional
+
+# Provider capabilities configuration
+# Keys are binding names (lowercase), values are capability dictionaries
+PROVIDER_CAPABILITIES: dict[str, dict[str, Any]] = {
+    # OpenAI and OpenAI-compatible providers
+    "openai": {
+        "supports_response_format": True,
+        "supports_streaming": True,
+        "supports_tools": True,
+        "system_in_messages": True,  # System prompt goes in messages array
+        "newer_models_use_max_completion_tokens": True,
+    },
+    "azure_openai": {
+        "supports_response_format": True,
+        "supports_streaming": True,
+        "supports_tools": True,
+        "system_in_messages": True,
+        "newer_models_use_max_completion_tokens": True,
+        "requires_api_version": True,
+    },
+    # Anthropic
+    "anthropic": {
+        "supports_response_format": False,  # Anthropic uses different format
+        "supports_streaming": True,
+        "supports_tools": True,
+        "system_in_messages": False,  # System is a separate parameter
+        "has_thinking_tags": False,
+    },
+    "claude": {  # Alias for anthropic
+        "supports_response_format": False,
+        "supports_streaming": True,
+        "supports_tools": True,
+        "system_in_messages": False,
+        "has_thinking_tags": False,
+    },
+    # DeepSeek
+    "deepseek": {
+        "supports_response_format": False,  # DeepSeek doesn't support strict JSON schema yet
+        "supports_streaming": True,
+        "supports_tools": True,
+        "system_in_messages": True,
+        "has_thinking_tags": True,  # DeepSeek reasoner has thinking tags
+    },
+    # OpenRouter (aggregator, generally OpenAI-compatible)
+    "openrouter": {
+        "supports_response_format": True,  # Depends on underlying model
+        "supports_streaming": True,
+        "supports_tools": True,
+        "system_in_messages": True,
+    },
+    # Groq (fast inference)
+    "groq": {
+        "supports_response_format": True,
+        "supports_streaming": True,
+        "supports_tools": True,
+        "system_in_messages": True,
+    },
+    # Together AI
+    "together": {
+        "supports_response_format": True,
+        "supports_streaming": True,
+        "supports_tools": True,
+        "system_in_messages": True,
+    },
+    "together_ai": {  # Alias
+        "supports_response_format": True,
+        "supports_streaming": True,
+        "supports_tools": True,
+        "system_in_messages": True,
+    },
+    # Mistral
+    "mistral": {
+        "supports_response_format": True,
+        "supports_streaming": True,
+        "supports_tools": True,
+        "system_in_messages": True,
+    },
+    # Local providers (generally OpenAI-compatible)
+    "ollama": {
+        "supports_response_format": True,  # Ollama supports JSON mode
+        "supports_streaming": True,
+        "supports_tools": False,  # Limited tool support
+        "system_in_messages": True,
+    },
+    "lm_studio": {
+        "supports_response_format": True,
+        "supports_streaming": True,
+        "supports_tools": False,
+        "system_in_messages": True,
+    },
+    "vllm": {
+        "supports_response_format": True,
+        "supports_streaming": True,
+        "supports_tools": False,
+        "system_in_messages": True,
+    },
+    "llama_cpp": {
+        "supports_response_format": True,  # llama.cpp server supports JSON grammar
+        "supports_streaming": True,
+        "supports_tools": False,
+        "system_in_messages": True,
+    },
+}
+
+# Default capabilities for unknown providers (assume OpenAI-compatible)
+DEFAULT_CAPABILITIES: dict[str, Any] = {
+    "supports_response_format": True,
+    "supports_streaming": True,
+    "supports_tools": False,
+    "system_in_messages": True,
+    "has_thinking_tags": False,
+}
+
+# Model-specific overrides
+# Format: {model_pattern: {capability: value}}
+# Patterns are matched with case-insensitive startswith
+MODEL_OVERRIDES: dict[str, dict[str, Any]] = {
+    "deepseek": {
+        "supports_response_format": False,
+        "has_thinking_tags": True,
+    },
+    "deepseek-reasoner": {
+        "supports_response_format": False,
+        "has_thinking_tags": True,
+    },
+    "qwen": {
+        # Qwen models may have thinking tags
+        "has_thinking_tags": True,
+    },
+    "qwq": {
+        # QwQ is Qwen's reasoning model with thinking tags
+        "has_thinking_tags": True,
+    },
+    # Claude models through OpenRouter or other providers
+    "claude": {
+        "supports_response_format": False,
+        "system_in_messages": False,
+    },
+    # Anthropic models
+    "anthropic/": {
+        "supports_response_format": False,
+        "system_in_messages": False,
+    },
+}
+
+
+def get_capability(
+    binding: str,
+    capability: str,
+    model: Optional[str] = None,
+    default: Any = None,
+) -> Any:
+    """
+    Get a capability value for a provider/model combination.
+
+    Checks in order:
+    1. Model-specific overrides (matched by prefix)
+    2. Provider/binding capabilities
+    3. Default capabilities for unknown providers
+    4. Explicit default value
+
+    Args:
+        binding: Provider binding name (e.g., "openai", "anthropic", "deepseek")
+        capability: Capability name (e.g., "supports_response_format")
+        model: Optional model name for model-specific overrides
+        default: Default value if capability is not defined
+
+    Returns:
+        Capability value or default
+    """
+    binding_lower = (binding or "openai").lower()
+
+    # 1. Check model-specific overrides first
+    if model:
+        model_lower = model.lower()
+        # Sort by pattern length descending to match most specific first
+        for pattern, overrides in sorted(MODEL_OVERRIDES.items(), key=lambda x: -len(x[0])):
+            if model_lower.startswith(pattern):
+                if capability in overrides:
+                    return overrides[capability]
+
+    # 2. Check provider capabilities
+    provider_caps = PROVIDER_CAPABILITIES.get(binding_lower, {})
+    if capability in provider_caps:
+        return provider_caps[capability]
+
+    # 3. Check default capabilities for unknown providers
+    if capability in DEFAULT_CAPABILITIES:
+        return DEFAULT_CAPABILITIES[capability]
+
+    # 4. Return explicit default
+    return default
+
+
+def supports_response_format(binding: str, model: Optional[str] = None) -> bool:
+    """
+    Check if the provider/model supports response_format parameter.
+
+    This is a convenience function for the most common capability check.
+
+    Args:
+        binding: Provider binding name
+        model: Optional model name for model-specific overrides
+
+    Returns:
+        True if response_format is supported
+    """
+    return get_capability(binding, "supports_response_format", model, default=True)
+
+
+def supports_streaming(binding: str, model: Optional[str] = None) -> bool:
+    """
+    Check if the provider/model supports streaming responses.
+
+    Args:
+        binding: Provider binding name
+        model: Optional model name
+
+    Returns:
+        True if streaming is supported
+    """
+    return get_capability(binding, "supports_streaming", model, default=True)
+
+
+def system_in_messages(binding: str, model: Optional[str] = None) -> bool:
+    """
+    Check if system prompt should be in messages array (OpenAI style)
+    or as a separate parameter (Anthropic style).
+
+    Args:
+        binding: Provider binding name
+        model: Optional model name
+
+    Returns:
+        True if system prompt goes in messages array
+    """
+    return get_capability(binding, "system_in_messages", model, default=True)
+
+
+def has_thinking_tags(binding: str, model: Optional[str] = None) -> bool:
+    """
+    Check if the model output may contain thinking tags (<think>...</think>).
+
+    Args:
+        binding: Provider binding name
+        model: Optional model name
+
+    Returns:
+        True if thinking tags should be filtered
+    """
+    return get_capability(binding, "has_thinking_tags", model, default=False)
+
+
+def supports_tools(binding: str, model: Optional[str] = None) -> bool:
+    """
+    Check if the provider/model supports function calling / tools.
+
+    Args:
+        binding: Provider binding name
+        model: Optional model name
+
+    Returns:
+        True if tools/function calling is supported
+    """
+    return get_capability(binding, "supports_tools", model, default=False)
+
+
+def requires_api_version(binding: str, model: Optional[str] = None) -> bool:
+    """
+    Check if the provider requires an API version parameter (e.g., Azure OpenAI).
+
+    Args:
+        binding: Provider binding name
+        model: Optional model name
+
+    Returns:
+        True if api_version is required
+    """
+    return get_capability(binding, "requires_api_version", model, default=False)
+
+
+__all__ = [
+    "PROVIDER_CAPABILITIES",
+    "MODEL_OVERRIDES",
+    "DEFAULT_CAPABILITIES",
+    "get_capability",
+    "supports_response_format",
+    "supports_streaming",
+    "system_in_messages",
+    "has_thinking_tags",
+    "supports_tools",
+    "requires_api_version",
+]

src/services/llm/client.py

@@ -0,0 +1,302 @@
+"""
+LLM Client
+==========
+
+Unified LLM client for all DeepTutor services.
+
+Note: This is a legacy interface. Prefer using the factory functions directly:
+    from src.services.llm import complete, stream
+"""
+
+from typing import Any, Dict, List, Optional
+
+from src.logging import get_logger
+
+from .capabilities import system_in_messages
+from .config import LLMConfig, get_llm_config
+
+
+class LLMClient:
+    """
+    Unified LLM client for all services.
+
+    Wraps the LLM Factory with a class-based interface.
+    Prefer using factory functions (complete, stream) directly for new code.
+    """
+
+    def __init__(self, config: Optional[LLMConfig] = None):
+        """
+        Initialize LLM client.
+
+        Args:
+            config: LLM configuration. If None, loads from environment.
+        """
+        self.config = config or get_llm_config()
+        self.logger = get_logger("LLMClient")
+
+    async def complete(
+        self,
+        prompt: str,
+        system_prompt: Optional[str] = None,
+        history: Optional[List[Dict[str, str]]] = None,
+        **kwargs: Any,
+    ) -> str:
+        """
+        Call LLM completion via Factory.
+
+        Args:
+            prompt: User prompt
+            system_prompt: Optional system prompt
+            history: Optional conversation history
+            **kwargs: Additional arguments passed to the API
+
+        Returns:
+            LLM response text
+        """
+        from . import factory
+
+        # Delegate to factory for unified routing and retry handling
+        return await factory.complete(
+            prompt=prompt,
+            system_prompt=system_prompt or "You are a helpful assistant.",
+            model=self.config.model,
+            api_key=self.config.api_key,
+            base_url=self.config.base_url,
+            api_version=getattr(self.config, "api_version", None),
+            binding=getattr(self.config, "binding", "openai"),
+            **kwargs,
+        )
+
+    def complete_sync(
+        self,
+        prompt: str,
+        system_prompt: Optional[str] = None,
+        history: Optional[List[Dict[str, str]]] = None,
+        **kwargs: Any,
+    ) -> str:
+        """
+        Synchronous wrapper for complete().
+
+        Use this when you need to call from non-async context.
+        """
+        import asyncio
+
+        try:
+            asyncio.get_running_loop()
+        except RuntimeError:
+            # No running event loop -> safe to run synchronously.
+            return asyncio.run(self.complete(prompt, system_prompt, history, **kwargs))
+
+        raise RuntimeError(
+            "LLMClient.complete_sync() cannot be called from a running event loop. "
+            "Use `await llm.complete(...)` instead."
+        )
+
+    def get_model_func(self):
+        """
+        Get a function compatible with LightRAG's llm_model_func parameter.
+
+        Returns:
+            Callable that can be used as llm_model_func
+        """
+        binding = getattr(self.config, "binding", "openai")
+
+        # Use capabilities to determine if provider uses OpenAI-style messages
+        uses_openai_style = system_in_messages(binding, self.config.model)
+
+        # For non-OpenAI-compatible providers (e.g., Anthropic), use Factory
+        if not uses_openai_style:
+            from . import factory
+
+            def llm_model_func_via_factory(
+                prompt: str,
+                system_prompt: Optional[str] = None,
+                history_messages: Optional[List[Dict]] = None,
+                **kwargs: Any,
+            ):
+                return factory.complete(
+                    prompt=prompt,
+                    system_prompt=system_prompt or "You are a helpful assistant.",
+                    model=self.config.model,
+                    api_key=self.config.api_key,
+                    base_url=self.config.base_url,
+                    binding=binding,
+                    history_messages=history_messages,
+                    **kwargs,
+                )
+
+            return llm_model_func_via_factory
+
+        # OpenAI-compatible bindings use lightrag (has caching)
+        from lightrag.llm.openai import openai_complete_if_cache
+
+        def llm_model_func(
+            prompt: str,
+            system_prompt: Optional[str] = None,
+            history_messages: Optional[List[Dict]] = None,
+            **kwargs: Any,
+        ):
+            # Only pass api_version if set (for Azure OpenAI)
+            lightrag_kwargs = {
+                "system_prompt": system_prompt,
+                "history_messages": history_messages or [],
+                "api_key": self.config.api_key,
+                "base_url": self.config.base_url,
+                **kwargs,
+            }
+            api_version = getattr(self.config, "api_version", None)
+            if api_version:
+                lightrag_kwargs["api_version"] = api_version
+            return openai_complete_if_cache(
+                self.config.model,
+                prompt,
+                **lightrag_kwargs,
+            )
+
+        return llm_model_func
+
+    def get_vision_model_func(self):
+        """
+        Get a function compatible with RAG-Anything's vision_model_func parameter.
+
+        Returns:
+            Callable that can be used as vision_model_func
+        """
+        binding = getattr(self.config, "binding", "openai")
+
+        # Use capabilities to determine if provider uses OpenAI-style messages
+        uses_openai_style = system_in_messages(binding, self.config.model)
+
+        # For non-OpenAI-compatible providers, use Factory
+        if not uses_openai_style:
+            from . import factory
+
+            def vision_model_func_via_factory(
+                prompt: str,
+                system_prompt: Optional[str] = None,
+                history_messages: Optional[List[Dict]] = None,
+                image_data: Optional[str] = None,
+                messages: Optional[List[Dict]] = None,
+                **kwargs: Any,
+            ):
+                # Use factory for unified handling
+                return factory.complete(
+                    prompt=prompt,
+                    system_prompt=system_prompt or "You are a helpful assistant.",
+                    model=self.config.model,
+                    api_key=self.config.api_key,
+                    base_url=self.config.base_url,
+                    binding=binding,
+                    messages=messages,
+                    history_messages=history_messages,
+                    image_data=image_data,
+                    **kwargs,
+                )
+
+            return vision_model_func_via_factory
+
+        # OpenAI-compatible bindings
+        from lightrag.llm.openai import openai_complete_if_cache
+
+        # Get api_version once for reuse
+        api_version = getattr(self.config, "api_version", None)
+
+        def vision_model_func(
+            prompt: str,
+            system_prompt: Optional[str] = None,
+            history_messages: Optional[List[Dict]] = None,
+            image_data: Optional[str] = None,
+            messages: Optional[List[Dict]] = None,
+            **kwargs: Any,
+        ):
+            # Handle multimodal messages
+            if messages:
+                clean_kwargs = {
+                    k: v
+                    for k, v in kwargs.items()
+                    if k not in ["messages", "prompt", "system_prompt", "history_messages"]
+                }
+                lightrag_kwargs = {
+                    "messages": messages,
+                    "api_key": self.config.api_key,
+                    "base_url": self.config.base_url,
+                    **clean_kwargs,
+                }
+                if api_version:
+                    lightrag_kwargs["api_version"] = api_version
+                return openai_complete_if_cache(
+                    self.config.model,
+                    prompt="",
+                    **lightrag_kwargs,
+                )
+
+            # Handle image data
+            if image_data:
+                # Build image message
+                image_message = {
+                    "role": "user",
+                    "content": [
+                        {"type": "text", "text": prompt},
+                        {
+                            "type": "image_url",
+                            "image_url": {"url": f"data:image/jpeg;base64,{image_data}"},
+                        },
+                    ],
+                }
+                lightrag_kwargs = {
+                    "messages": [image_message],
+                    "api_key": self.config.api_key,
+                    "base_url": self.config.base_url,
+                    **kwargs,
+                }
+                if api_version:
+                    lightrag_kwargs["api_version"] = api_version
+                return openai_complete_if_cache(
+                    self.config.model,
+                    prompt="",
+                    **lightrag_kwargs,
+                )
+
+            # Fallback to regular completion
+            lightrag_kwargs = {
+                "system_prompt": system_prompt,
+                "history_messages": history_messages or [],
+                "api_key": self.config.api_key,
+                "base_url": self.config.base_url,
+                **kwargs,
+            }
+            if api_version:
+                lightrag_kwargs["api_version"] = api_version
+            return openai_complete_if_cache(
+                self.config.model,
+                prompt,
+                **lightrag_kwargs,
+            )
+
+        return vision_model_func
+
+
+# Singleton instance
+_client: Optional[LLMClient] = None
+
+
+def get_llm_client(config: Optional[LLMConfig] = None) -> LLMClient:
+    """
+    Get or create the singleton LLM client.
+
+    Args:
+        config: Optional configuration. Only used on first call.
+
+    Returns:
+        LLMClient instance
+    """
+    global _client
+    if _client is None:
+        _client = LLMClient(config)
+    return _client
+
+
+def reset_llm_client():
+    """Reset the singleton LLM client."""
+    global _client
+    _client = None