aiecs 1.7.17__py3-none-any.whl → 1.8.4__py3-none-any.whl

aiecs/llm/clients/googleai_client.py

@@ -1,7 +1,8 @@
 import json
 import logging
 import os
-from typing import Optional, List, AsyncGenerator
+import base64
+from typing import Optional, List, AsyncGenerator, Dict, Any
 
 from google import genai
 from google.genai import types
@@ -14,6 +15,7 @@ from aiecs.llm.clients.base_client import (
     RateLimitError,
 )
 from aiecs.config.config import get_settings
+from aiecs.llm.utils.image_utils import parse_image_source, ImageContent
 
 logger = logging.getLogger(__name__)
 
@@ -91,6 +93,33 @@ class GoogleAIClient(BaseLLMClient):
                 parts = []
                 if msg.content:
                     parts.append(types.Part(text=msg.content))
+                
+                # Add images if present
+                if msg.images:
+                    for image_source in msg.images:
+                        image_content = parse_image_source(image_source)
+                        
+                        if image_content.is_url():
+                            # For URLs, use inline_data with downloaded content
+                            # Note: Google AI SDK may support URL directly, but we'll use base64 for compatibility
+                            try:
+                                import urllib.request
+                                with urllib.request.urlopen(image_content.get_url()) as response:
+                                    image_bytes = response.read()
+                                parts.append(types.Part.from_bytes(
+                                    data=image_bytes,
+                                    mime_type=image_content.mime_type
+                                ))
+                            except Exception as e:
+                                logger.warning(f"Failed to download image from URL: {e}")
+                        else:
+                            # Convert to bytes for inline_data
+                            base64_data = image_content.get_base64_data()
+                            image_bytes = base64.b64decode(base64_data)
+                            parts.append(types.Part.from_bytes(
+                                data=image_bytes,
+                                mime_type=image_content.mime_type
+                            ))
 
                 for tool_call in msg.tool_calls:
                     func = tool_call.get("function", {})
@@ -120,10 +149,42 @@ class GoogleAIClient(BaseLLMClient):
             # Handle regular messages (user, assistant without tool_calls)
             else:
                 role = "model" if msg.role == "assistant" else msg.role
+                parts = []
+                
+                # Add text content if present
                 if msg.content:
+                    parts.append(types.Part(text=msg.content))
+                
+                # Add images if present
+                if msg.images:
+                    for image_source in msg.images:
+                        image_content = parse_image_source(image_source)
+                        
+                        if image_content.is_url():
+                            # Download URL and convert to bytes
+                            try:
+                                import urllib.request
+                                with urllib.request.urlopen(image_content.get_url()) as response:
+                                    image_bytes = response.read()
+                                parts.append(types.Part.from_bytes(
+                                    data=image_bytes,
+                                    mime_type=image_content.mime_type
+                                ))
+                            except Exception as e:
+                                logger.warning(f"Failed to download image from URL: {e}")
+                        else:
+                            # Convert to bytes for inline_data
+                            base64_data = image_content.get_base64_data()
+                            image_bytes = base64.b64decode(base64_data)
+                            parts.append(types.Part.from_bytes(
+                                data=image_bytes,
+                                mime_type=image_content.mime_type
+                            ))
+                
+                if parts:
                     contents.append(types.Content(
                         role=role,
-                        parts=[types.Part(text=msg.content)]
+                        parts=parts
                     ))
 
         return contents
@@ -134,10 +195,30 @@ class GoogleAIClient(BaseLLMClient):
         model: Optional[str] = None,
         temperature: float = 0.7,
         max_tokens: Optional[int] = None,
+        context: Optional[Dict[str, Any]] = None,
         system_instruction: Optional[str] = None,
         **kwargs,
     ) -> LLMResponse:
-        """Generate text using Google AI (google.genai SDK)"""
+        """
+        Generate text using Google AI (google.genai SDK).
+
+        Args:
+            messages: List of conversation messages
+            model: Model name (optional, uses default if not provided)
+            temperature: Sampling temperature (0.0 to 1.0)
+            max_tokens: Maximum tokens to generate
+            context: Optional context dictionary containing metadata such as:
+                - user_id: User identifier for tracking/billing
+                - tenant_id: Tenant identifier for multi-tenant setups
+                - request_id: Request identifier for tracing
+                - session_id: Session identifier
+                - Any other custom metadata for observability or middleware
+            system_instruction: System instruction for the model
+            **kwargs: Additional provider-specific parameters
+
+        Returns:
+            LLMResponse with generated text and metadata
+        """
         client = self._init_google_ai()
 
         # Get model name from config if not provided
@@ -237,10 +318,30 @@ class GoogleAIClient(BaseLLMClient):
         model: Optional[str] = None,
         temperature: float = 0.7,
         max_tokens: Optional[int] = None,
+        context: Optional[Dict[str, Any]] = None,
         system_instruction: Optional[str] = None,
         **kwargs,
     ) -> AsyncGenerator[str, None]:
-        """Stream text generation using Google AI (google.genai SDK)"""
+        """
+        Stream text generation using Google AI (google.genai SDK).
+
+        Args:
+            messages: List of conversation messages
+            model: Model name (optional, uses default if not provided)
+            temperature: Sampling temperature (0.0 to 1.0)
+            max_tokens: Maximum tokens to generate
+            context: Optional context dictionary containing metadata such as:
+                - user_id: User identifier for tracking/billing
+                - tenant_id: Tenant identifier for multi-tenant setups
+                - request_id: Request identifier for tracing
+                - session_id: Session identifier
+                - Any other custom metadata for observability or middleware
+            system_instruction: System instruction for the model
+            **kwargs: Additional provider-specific parameters
+
+        Yields:
+            Text tokens as they are generated
+        """
         client = self._init_google_ai()
 
         # Get model name from config if not provided

aiecs/llm/clients/openai_client.py

@@ -52,6 +52,7 @@ class OpenAIClient(BaseLLMClient, OpenAICompatibleFunctionCallingMixin):
         model: Optional[str] = None,
         temperature: float = 0.7,
         max_tokens: Optional[int] = None,
+        context: Optional[Dict[str, Any]] = None,
         functions: Optional[List[Dict[str, Any]]] = None,
         tools: Optional[List[Dict[str, Any]]] = None,
         tool_choice: Optional[Any] = None,
@@ -65,6 +66,11 @@ class OpenAIClient(BaseLLMClient, OpenAICompatibleFunctionCallingMixin):
             model: Model name (optional)
             temperature: Temperature for generation
             max_tokens: Maximum tokens to generate
+            context: Optional context dictionary containing metadata such as:
+                - user_id: User identifier for tracking/billing
+                - tenant_id: Tenant identifier for multi-tenant setups
+                - request_id: Request identifier for tracing
+                - session_id: Session identifier
             functions: List of function schemas (legacy format)
             tools: List of tool schemas (new format, recommended)
             tool_choice: Tool choice strategy ("auto", "none", or specific tool)
@@ -103,6 +109,7 @@ class OpenAIClient(BaseLLMClient, OpenAICompatibleFunctionCallingMixin):
         model: Optional[str] = None,
         temperature: float = 0.7,
         max_tokens: Optional[int] = None,
+        context: Optional[Dict[str, Any]] = None,
         functions: Optional[List[Dict[str, Any]]] = None,
         tools: Optional[List[Dict[str, Any]]] = None,
         tool_choice: Optional[Any] = None,
@@ -117,6 +124,11 @@ class OpenAIClient(BaseLLMClient, OpenAICompatibleFunctionCallingMixin):
             model: Model name (optional)
             temperature: Temperature for generation
             max_tokens: Maximum tokens to generate
+            context: Optional context dictionary containing metadata such as:
+                - user_id: User identifier for tracking/billing
+                - tenant_id: Tenant identifier for multi-tenant setups
+                - request_id: Request identifier for tracing
+                - session_id: Session identifier
             functions: List of function schemas (legacy format)
             tools: List of tool schemas (new format, recommended)
             tool_choice: Tool choice strategy ("auto", "none", or specific tool)

aiecs/llm/clients/openai_compatible_mixin.py

@@ -11,6 +11,7 @@ from dataclasses import dataclass
 from openai import AsyncOpenAI
 
 from .base_client import LLMMessage, LLMResponse
+from aiecs.llm.utils.image_utils import parse_image_source, ImageContent
 
 logger = logging.getLogger(__name__)
 
@@ -49,7 +50,7 @@ class OpenAICompatibleFunctionCallingMixin:
 
     def _convert_messages_to_openai_format(self, messages: List[LLMMessage]) -> List[Dict[str, Any]]:
         """
-        Convert LLMMessage list to OpenAI message format (support tool calls).
+        Convert LLMMessage list to OpenAI message format (support tool calls and vision).
         
         Args:
             messages: List of LLMMessage objects
@@ -60,8 +61,47 @@ class OpenAICompatibleFunctionCallingMixin:
         openai_messages = []
         for msg in messages:
             msg_dict: Dict[str, Any] = {"role": msg.role}
-            if msg.content is not None:
+            
+            # Handle multimodal content (text + images)
+            if msg.images:
+                # Build content array with text and images
+                content_array = []
+                
+                # Add text content if present
+                if msg.content:
+                    content_array.append({"type": "text", "text": msg.content})
+                
+                # Add images
+                for image_source in msg.images:
+                    image_content = parse_image_source(image_source)
+                    
+                    if image_content.is_url():
+                        # Use URL directly
+                        content_array.append({
+                            "type": "image_url",
+                            "image_url": {
+                                "url": image_content.get_url(),
+                                "detail": image_content.detail,
+                            }
+                        })
+                    else:
+                        # Convert to base64 data URI
+                        base64_data = image_content.get_base64_data()
+                        mime_type = image_content.mime_type
+                        data_uri = f"data:{mime_type};base64,{base64_data}"
+                        content_array.append({
+                            "type": "image_url",
+                            "image_url": {
+                                "url": data_uri,
+                                "detail": image_content.detail,
+                            }
+                        })
+                
+                msg_dict["content"] = content_array
+            elif msg.content is not None:
+                # Text-only content
                 msg_dict["content"] = msg.content
+            
             if msg.tool_calls:
                 msg_dict["tool_calls"] = msg.tool_calls
             if msg.tool_call_id:

aiecs/llm/clients/openrouter_client.py

@@ -0,0 +1,272 @@
+from openai import AsyncOpenAI
+from aiecs.config.config import get_settings
+from aiecs.llm.clients.base_client import (
+    BaseLLMClient,
+    LLMMessage,
+    LLMResponse,
+    ProviderNotAvailableError,
+    RateLimitError,
+)
+from aiecs.llm.clients.openai_compatible_mixin import (
+    OpenAICompatibleFunctionCallingMixin,
+    StreamChunk,
+)
+from tenacity import (
+    retry,
+    stop_after_attempt,
+    wait_exponential,
+    retry_if_exception_type,
+)
+import logging
+from typing import Dict, Optional, List, AsyncGenerator, cast, Any
+
+# Lazy import to avoid circular dependency
+
+
+def _get_config_loader():
+    """Lazy import of config loader to avoid circular dependency"""
+    from aiecs.llm.config import get_llm_config_loader
+
+    return get_llm_config_loader()
+
+
+logger = logging.getLogger(__name__)
+
+
+class OpenRouterClient(BaseLLMClient, OpenAICompatibleFunctionCallingMixin):
+    """OpenRouter provider client using OpenAI-compatible API"""
+
+    def __init__(self) -> None:
+        super().__init__("OpenRouter")
+        self.settings = get_settings()
+        self._openai_client: Optional[AsyncOpenAI] = None
+        self._model_map: Optional[Dict[str, str]] = None
+
+    def _get_openai_client(self) -> AsyncOpenAI:
+        """Lazy initialization of OpenAI client for OpenRouter"""
+        if not self._openai_client:
+            api_key = self._get_api_key()
+            self._openai_client = AsyncOpenAI(
+                api_key=api_key,
+                base_url="https://openrouter.ai/api/v1",
+                timeout=360.0,
+            )
+        return self._openai_client
+
+    def _get_api_key(self) -> str:
+        """Get API key from settings"""
+        api_key = getattr(self.settings, "openrouter_api_key", None)
+        if not api_key:
+            raise ProviderNotAvailableError("OpenRouter API key not configured. Set OPENROUTER_API_KEY.")
+        return api_key
+
+    def _get_model_map(self) -> Dict[str, str]:
+        """Get model mappings from configuration"""
+        if self._model_map is None:
+            try:
+                loader = _get_config_loader()
+                provider_config = loader.get_provider_config("OpenRouter")
+                if provider_config and provider_config.model_mappings:
+                    self._model_map = provider_config.model_mappings
+                else:
+                    self._model_map = {}
+            except Exception as e:
+                self.logger.warning(f"Failed to load model mappings from config: {e}")
+                self._model_map = {}
+        return self._model_map
+
+    def _get_extra_headers(self, **kwargs) -> Dict[str, str]:
+        """
+        Get extra headers for OpenRouter API.
+        
+        Supports HTTP-Referer and X-Title headers from kwargs or settings.
+        
+        Args:
+            **kwargs: May contain http_referer and x_title
+            
+        Returns:
+            Dictionary with extra headers
+        """
+        extra_headers: Dict[str, str] = {}
+        
+        # Get from kwargs first, then from settings
+        http_referer = kwargs.get("http_referer") or getattr(self.settings, "openrouter_http_referer", None)
+        x_title = kwargs.get("x_title") or getattr(self.settings, "openrouter_x_title", None)
+        
+        if http_referer:
+            extra_headers["HTTP-Referer"] = http_referer
+        if x_title:
+            extra_headers["X-Title"] = x_title
+            
+        return extra_headers
+
+    @retry(
+        stop=stop_after_attempt(3),
+        wait=wait_exponential(multiplier=1, min=4, max=10),
+        retry=retry_if_exception_type((Exception, RateLimitError)),
+    )
+    async def generate_text(
+        self,
+        messages: List[LLMMessage],
+        model: Optional[str] = None,
+        temperature: float = 0.7,
+        max_tokens: Optional[int] = None,
+        functions: Optional[List[Dict[str, Any]]] = None,
+        tools: Optional[List[Dict[str, Any]]] = None,
+        tool_choice: Optional[Any] = None,
+        **kwargs,
+    ) -> LLMResponse:
+        """
+        Generate text using OpenRouter API via OpenAI library.
+        
+        OpenRouter API is OpenAI-compatible, so it supports Function Calling and Vision.
+        
+        Args:
+            messages: List of LLM messages
+            model: Model name (optional, uses default from config if not provided)
+            temperature: Temperature for generation
+            max_tokens: Maximum tokens to generate
+            functions: List of function schemas (legacy format)
+            tools: List of tool schemas (new format, recommended)
+            tool_choice: Tool choice strategy ("auto", "none", or specific tool)
+            http_referer: Optional HTTP-Referer header for OpenRouter rankings
+            x_title: Optional X-Title header for OpenRouter rankings
+            **kwargs: Additional arguments passed to OpenRouter API
+            
+        Returns:
+            LLMResponse with content and optional function_call information
+        """
+        # Check API key availability
+        api_key = self._get_api_key()
+        if not api_key:
+            raise ProviderNotAvailableError("OpenRouter API key is not configured.")
+
+        client = self._get_openai_client()
+
+        # Get model name from config if not provided
+        selected_model = model or self._get_default_model() or "openai/gpt-4o"
+
+        # Get model mappings from config
+        model_map = self._get_model_map()
+        api_model = model_map.get(selected_model, selected_model)
+
+        # Extract extra headers from kwargs
+        extra_headers = self._get_extra_headers(**kwargs)
+        
+        # Remove extra header kwargs to avoid passing them to API
+        kwargs_clean = {k: v for k, v in kwargs.items() if k not in ("http_referer", "x_title")}
+        
+        # Add extra_headers to kwargs if present
+        if extra_headers:
+            kwargs_clean["extra_headers"] = extra_headers
+
+        try:
+            # Use mixin method for Function Calling support
+            response = await self._generate_text_with_function_calling(
+                client=client,
+                messages=messages,
+                model=api_model,
+                temperature=temperature,
+                max_tokens=max_tokens,
+                functions=functions,
+                tools=tools,
+                tool_choice=tool_choice,
+                **kwargs_clean,
+            )
+            
+            # Override provider and model name for OpenRouter
+            response.provider = self.provider_name
+            response.model = selected_model
+            
+            return response
+
+        except Exception as e:
+            if "rate limit" in str(e).lower() or "429" in str(e):
+                raise RateLimitError(f"OpenRouter rate limit exceeded: {str(e)}")
+            logger.error(f"OpenRouter API error: {str(e)}")
+            raise
+
+    async def stream_text(  # type: ignore[override]
+        self,
+        messages: List[LLMMessage],
+        model: Optional[str] = None,
+        temperature: float = 0.7,
+        max_tokens: Optional[int] = None,
+        functions: Optional[List[Dict[str, Any]]] = None,
+        tools: Optional[List[Dict[str, Any]]] = None,
+        tool_choice: Optional[Any] = None,
+        return_chunks: bool = False,
+        **kwargs,
+    ) -> AsyncGenerator[Any, None]:
+        """
+        Stream text using OpenRouter API via OpenAI library.
+        
+        OpenRouter API is OpenAI-compatible, so it supports Function Calling and Vision.
+        
+        Args:
+            messages: List of LLM messages
+            model: Model name (optional, uses default from config if not provided)
+            temperature: Temperature for generation
+            max_tokens: Maximum tokens to generate
+            functions: List of function schemas (legacy format)
+            tools: List of tool schemas (new format, recommended)
+            tool_choice: Tool choice strategy ("auto", "none", or specific tool)
+            return_chunks: If True, returns StreamChunk objects with tool_calls info; if False, returns str tokens only
+            http_referer: Optional HTTP-Referer header for OpenRouter rankings
+            x_title: Optional X-Title header for OpenRouter rankings
+            **kwargs: Additional arguments passed to OpenRouter API
+            
+        Yields:
+            str or StreamChunk: Text tokens as they are generated, or StreamChunk objects if return_chunks=True
+        """
+        # Check API key availability
+        api_key = self._get_api_key()
+        if not api_key:
+            raise ProviderNotAvailableError("OpenRouter API key is not configured.")
+
+        client = self._get_openai_client()
+
+        # Get model name from config if not provided
+        selected_model = model or self._get_default_model() or "openai/gpt-4o"
+
+        # Get model mappings from config
+        model_map = self._get_model_map()
+        api_model = model_map.get(selected_model, selected_model)
+
+        # Extract extra headers from kwargs
+        extra_headers = self._get_extra_headers(**kwargs)
+        
+        # Remove extra header kwargs to avoid passing them to API
+        kwargs_clean = {k: v for k, v in kwargs.items() if k not in ("http_referer", "x_title")}
+        
+        # Add extra_headers to kwargs if present
+        if extra_headers:
+            kwargs_clean["extra_headers"] = extra_headers
+
+        try:
+            # Use mixin method for Function Calling support
+            async for chunk in self._stream_text_with_function_calling(
+                client=client,
+                messages=messages,
+                model=api_model,
+                temperature=temperature,
+                max_tokens=max_tokens,
+                functions=functions,
+                tools=tools,
+                tool_choice=tool_choice,
+                return_chunks=return_chunks,
+                **kwargs_clean,
+            ):
+                yield chunk
+
+        except Exception as e:
+            if "rate limit" in str(e).lower() or "429" in str(e):
+                raise RateLimitError(f"OpenRouter rate limit exceeded: {str(e)}")
+            logger.error(f"OpenRouter API streaming error: {str(e)}")
+            raise
+
+    async def close(self):
+        """Clean up resources"""
+        if self._openai_client:
+            await self._openai_client.close()
+            self._openai_client = None

aiecs/llm/clients/vertex_client.py

@@ -4,6 +4,7 @@ import logging
 import os
 import warnings
 import hashlib
+import base64
 from typing import Dict, Any, Optional, List, AsyncGenerator, Union
 import vertexai
 from vertexai.generative_models import (
@@ -16,6 +17,8 @@ from vertexai.generative_models import (
     Part,
 )
 
+from aiecs.llm.utils.image_utils import parse_image_source, ImageContent
+
 logger = logging.getLogger(__name__)
 
 # Try to import CachedContent for prompt caching support
@@ -450,6 +453,24 @@ class VertexAIClient(BaseLLMClient, GoogleFunctionCallingMixin):
                 parts = []
                 if msg.content:
                     parts.append(Part.from_text(msg.content))
+                
+                # Add images if present
+                if msg.images:
+                    for image_source in msg.images:
+                        image_content = parse_image_source(image_source)
+                        
+                        if image_content.is_url():
+                            parts.append(Part.from_uri(
+                                uri=image_content.get_url(),
+                                mime_type=image_content.mime_type
+                            ))
+                        else:
+                            base64_data = image_content.get_base64_data()
+                            image_bytes = base64.b64decode(base64_data)
+                            parts.append(Part.from_bytes(
+                                data=image_bytes,
+                                mime_type=image_content.mime_type
+                            ))
 
                 for tool_call in msg.tool_calls:
                     func = tool_call.get("function", {})
@@ -481,11 +502,34 @@ class VertexAIClient(BaseLLMClient, GoogleFunctionCallingMixin):
             # Handle regular messages (user, assistant without tool_calls)
             else:
                 role = "model" if msg.role == "assistant" else msg.role
+                parts = []
+                
+                # Add text content if present
                 if msg.content:
-                    contents.append(Content(
-                        role=role,
-                        parts=[Part.from_text(msg.content)]
-                    ))
+                    parts.append(Part.from_text(msg.content))
+                
+                # Add images if present
+                if msg.images:
+                    for image_source in msg.images:
+                        image_content = parse_image_source(image_source)
+                        
+                        if image_content.is_url():
+                            # Use Part.from_uri for URLs
+                            parts.append(Part.from_uri(
+                                uri=image_content.get_url(),
+                                mime_type=image_content.mime_type
+                            ))
+                        else:
+                            # Convert to bytes for inline_data
+                            base64_data = image_content.get_base64_data()
+                            image_bytes = base64.b64decode(base64_data)
+                            parts.append(Part.from_bytes(
+                                data=image_bytes,
+                                mime_type=image_content.mime_type
+                            ))
+                
+                if parts:
+                    contents.append(Content(role=role, parts=parts))
 
         return contents
 
@@ -495,13 +539,36 @@ class VertexAIClient(BaseLLMClient, GoogleFunctionCallingMixin):
         model: Optional[str] = None,
         temperature: float = 0.7,
         max_tokens: Optional[int] = None,
+        context: Optional[Dict[str, Any]] = None,
         functions: Optional[List[Dict[str, Any]]] = None,
         tools: Optional[List[Dict[str, Any]]] = None,
         tool_choice: Optional[Any] = None,
         system_instruction: Optional[str] = None,
         **kwargs,
     ) -> LLMResponse:
-        """Generate text using Vertex AI"""
+        """
+        Generate text using Vertex AI.
+
+        Args:
+            messages: List of conversation messages
+            model: Model name (optional, uses default if not provided)
+            temperature: Sampling temperature (0.0 to 1.0)
+            max_tokens: Maximum tokens to generate
+            context: Optional context dictionary containing metadata such as:
+                - user_id: User identifier for tracking/billing
+                - tenant_id: Tenant identifier for multi-tenant setups
+                - request_id: Request identifier for tracing
+                - session_id: Session identifier
+                - Any other custom metadata for observability or middleware
+            functions: List of function schemas (legacy format)
+            tools: List of tool schemas (new format, recommended)
+            tool_choice: Tool choice strategy
+            system_instruction: System instruction for the model
+            **kwargs: Additional provider-specific parameters
+
+        Returns:
+            LLMResponse with generated text and metadata
+        """
         self._init_vertex_ai()
 
         # Get model name from config if not provided
@@ -889,6 +956,7 @@ class VertexAIClient(BaseLLMClient, GoogleFunctionCallingMixin):
         model: Optional[str] = None,
         temperature: float = 0.7,
         max_tokens: Optional[int] = None,
+        context: Optional[Dict[str, Any]] = None,
         functions: Optional[List[Dict[str, Any]]] = None,
         tools: Optional[List[Dict[str, Any]]] = None,
         tool_choice: Optional[Any] = None,
@@ -904,6 +972,12 @@ class VertexAIClient(BaseLLMClient, GoogleFunctionCallingMixin):
             model: Model name (optional)
             temperature: Temperature for generation
             max_tokens: Maximum tokens to generate
+            context: Optional context dictionary containing metadata such as:
+                - user_id: User identifier for tracking/billing
+                - tenant_id: Tenant identifier for multi-tenant setups
+                - request_id: Request identifier for tracing
+                - session_id: Session identifier
+                - Any other custom metadata for observability or middleware
             functions: List of function schemas (legacy format)
             tools: List of tool schemas (new format)
             tool_choice: Tool choice strategy (not used for Google Vertex AI)