autobyteus 1.1.5__py3-none-any.whl → 1.1.7__py3-none-any.whl

autobyteus/llm/token_counter/token_counter_factory.py

@@ -25,7 +25,7 @@ def get_token_counter(model: LLMModel, llm: 'BaseLLM') -> BaseTokenCounter:
     if model.provider == LLMProvider.OPENAI:
         return OpenAITokenCounter(model, llm)
     elif model.provider == LLMProvider.ANTHROPIC:
-        return ClaudeTokenCounter(model, llm)
+        return OpenAITokenCounter(model, llm)
     elif model.provider == LLMProvider.MISTRAL:
         return MistralTokenCounter(model, llm)
     elif model.provider == LLMProvider.DEEPSEEK:
@@ -34,6 +34,8 @@ def get_token_counter(model: LLMModel, llm: 'BaseLLM') -> BaseTokenCounter:
         return DeepSeekTokenCounter(model, llm)
     elif model.provider == LLMProvider.KIMI:
         return KimiTokenCounter(model, llm)
+    elif model.provider == LLMProvider.QWEN:
+        return OpenAITokenCounter(model, llm)
     elif model.provider == LLMProvider.OLLAMA:
         return OpenAITokenCounter(model, llm)
     elif model.provider == LLMProvider.LMSTUDIO:

autobyteus/llm/user_message.py

@@ -7,53 +7,65 @@ logger = logging.getLogger(__name__)
 class LLMUserMessage:
     """
     Represents a user message formatted specifically for input to an LLM.
-    It includes content and optionally image URLs.
-    This structure is typically used when constructing prompts for multimodal LLMs
-    or when sending a "user" role message in a conversation.
+    It includes content and optionally URLs for various media types.
+    This structure is typically used when constructing prompts for multimodal LLMs.
     """
     def __init__(self,
                  content: str,
-                 image_urls: Optional[List[str]] = None):
+                 image_urls: Optional[List[str]] = None,
+                 audio_urls: Optional[List[str]] = None,
+                 video_urls: Optional[List[str]] = None):
         """
         Initializes an LLMUserMessage.
 
         Args:
             content: The textual content of the user's message.
-            image_urls: An optional list of URLs or local paths to images
-                        to be included with the message for the LLM.
+            image_urls: An optional list of URLs or local paths to images.
+            audio_urls: An optional list of URLs or local paths to audio files.
+            video_urls: An optional list of URLs or local paths to video files.
         """
-        if not isinstance(content, str):
-            # Allow empty string for content, as images might be the only input.
-            # But content must still be a string type.
-            pass # Validation can be more strict if empty content is disallowed with no images
-        
-        if image_urls is None:
-            image_urls = [] # Default to empty list for easier processing
+        self.content: str = content
+        self.image_urls: List[str] = image_urls or []
+        self.audio_urls: List[str] = audio_urls or []
+        self.video_urls: List[str] = video_urls or []
 
-        if not (isinstance(image_urls, list) and all(isinstance(url, str) for url in image_urls)):
+        # --- Validation ---
+        if not isinstance(self.content, str):
+            raise TypeError("LLMUserMessage 'content' must be a string.")
+        if not (isinstance(self.image_urls, list) and all(isinstance(url, str) for url in self.image_urls)):
             raise TypeError("LLMUserMessage 'image_urls' must be a list of strings.")
-        
-        if not content and not image_urls:
-            raise ValueError("LLMUserMessage must have either content or image_urls or both.")
+        if not (isinstance(self.audio_urls, list) and all(isinstance(url, str) for url in self.audio_urls)):
+            raise TypeError("LLMUserMessage 'audio_urls' must be a list of strings.")
+        if not (isinstance(self.video_urls, list) and all(isinstance(url, str) for url in self.video_urls)):
+            raise TypeError("LLMUserMessage 'video_urls' must be a list of strings.")
 
-        self.content: str = content
-        self.image_urls: List[str] = image_urls
+        if not self.content and not self.image_urls and not self.audio_urls and not self.video_urls:
+            raise ValueError("LLMUserMessage must have either content or at least one media URL.")
 
-        logger.debug(f"LLMUserMessage created. Content: '{content[:50]}...', Image URLs: {image_urls}")
+        logger.debug(f"LLMUserMessage created. Content: '{self.content[:50]}...', "
+                     f"Images: {len(self.image_urls)}, Audio: {len(self.audio_urls)}, Video: {len(self.video_urls)}")
 
     def __repr__(self) -> str:
-        image_urls_repr = f", image_urls={self.image_urls}" if self.image_urls else ""
-        return f"LLMUserMessage(content='{self.content[:100]}...'{image_urls_repr})"
+        parts = [f"content='{self.content[:100]}...'"]
+        if self.image_urls:
+            parts.append(f"image_urls={self.image_urls}")
+        if self.audio_urls:
+            parts.append(f"audio_urls={self.audio_urls}")
+        if self.video_urls:
+            parts.append(f"video_urls={self.video_urls}")
+        return f"LLMUserMessage({', '.join(parts)})"
 
     def to_dict(self) -> Dict[str, Any]:
         """
-        Serializes the LLMUserMessage to a dictionary. This method might be less used
-        now that BaseLLM._add_user_message handles the conversion to the Message format.
-        Kept for potential direct use or testing.
+        Serializes the LLMUserMessage to a dictionary.
         """
         data = {"content": self.content}
         if self.image_urls:
             data["image_urls"] = self.image_urls
+        if self.audio_urls:
+            data["audio_urls"] = self.audio_urls
+        if self.video_urls:
+            data["video_urls"] = self.video_urls
         return data
 
     @classmethod
@@ -61,13 +73,9 @@ class LLMUserMessage:
         """
         Deserializes an LLMUserMessage from a dictionary.
         """
-        content = data.get("content", "") # Default to empty string if not present
-        image_urls = data.get("image_urls") # Expects a list or None
-
-        # Basic validation, more can be added if needed
-        if not isinstance(content, str):
-             raise ValueError("LLMUserMessage 'content' in dictionary must be a string.")
-        if image_urls is not None and not (isinstance(image_urls, list) and all(isinstance(url, str) for url in image_urls)):
-            raise ValueError("LLMUserMessage 'image_urls' in dictionary must be a list of strings if provided.")
-
-        return cls(content=content, image_urls=image_urls)
+        return cls(
+            content=data.get("content", ""),
+            image_urls=data.get("image_urls"),
+            audio_urls=data.get("audio_urls"),
+            video_urls=data.get("video_urls")
+        )

autobyteus/llm/utils/llm_config.py

@@ -55,6 +55,7 @@ class LLMConfig:
     frequency_penalty: Optional[float] = None
     presence_penalty: Optional[float] = None
     stop_sequences: Optional[List] = None
+    uses_max_completion_tokens: bool = False
     extra_params: Dict[str, Any] = field(default_factory=dict)
     pricing_config: TokenPricingConfig = field(default_factory=TokenPricingConfig)
 
@@ -102,17 +103,28 @@ class LLMConfig:
         data_copy = data.copy()
         pricing_config_data = data_copy.pop('pricing_config', {})
         
+        # Create a new dictionary for known fields to avoid passing them in twice
+        known_fields = {
+            'rate_limit', 'token_limit', 'system_message', 'temperature', 
+            'max_tokens', 'top_p', 'frequency_penalty', 'presence_penalty', 
+            'stop_sequences', 'uses_max_completion_tokens', 'extra_params', 
+            'pricing_config'
+        }
+        
+        init_kwargs = {k: v for k, v in data_copy.items() if k in known_fields}
+        
         config = cls(
-            rate_limit=data_copy.get('rate_limit'),
-            token_limit=data_copy.get('token_limit'),
-            system_message=data_copy.get('system_message', "You are a helpful assistant."),
-            temperature=data_copy.get('temperature', 0.7),
-            max_tokens=data_copy.get('max_tokens'),
-            top_p=data_copy.get('top_p'),
-            frequency_penalty=data_copy.get('frequency_penalty'),
-            presence_penalty=data_copy.get('presence_penalty'),
-            stop_sequences=data_copy.get('stop_sequences'),
-            extra_params=data_copy.get('extra_params', {}),
+            rate_limit=init_kwargs.get('rate_limit'),
+            token_limit=init_kwargs.get('token_limit'),
+            system_message=init_kwargs.get('system_message', "You are a helpful assistant."),
+            temperature=init_kwargs.get('temperature', 0.7),
+            max_tokens=init_kwargs.get('max_tokens'),
+            top_p=init_kwargs.get('top_p'),
+            frequency_penalty=init_kwargs.get('frequency_penalty'),
+            presence_penalty=init_kwargs.get('presence_penalty'),
+            stop_sequences=init_kwargs.get('stop_sequences'),
+            uses_max_completion_tokens=init_kwargs.get('uses_max_completion_tokens', False),
+            extra_params=init_kwargs.get('extra_params', {}),
             pricing_config=pricing_config_data 
         )
         return config
@@ -162,26 +174,30 @@ class LLMConfig:
         for f_info in fields(override_config):
             override_value = getattr(override_config, f_info.name)
             
+            # Special handling for booleans where we want to merge if it's not the default
+            # For `uses_max_completion_tokens`, the default is False, so `if override_value:` is fine
+            is_boolean_field = f_info.type == bool
+            
+            # Standard check for None, but also merge if it's a non-default boolean
             if override_value is not None:
-                if f_info.name == 'pricing_config':
-                    # Ensure self.pricing_config is an object (should be by __post_init__)
+                # For uses_max_completion_tokens, `False` is a valid override value, but `None` is not
+                if is_boolean_field and override_value is False and getattr(self, f_info.name) is True:
+                     setattr(self, f_info.name, override_value)
+                elif f_info.name == 'pricing_config':
                     if not isinstance(self.pricing_config, TokenPricingConfig):
-                        self.pricing_config = TokenPricingConfig() # Should not be needed
+                        self.pricing_config = TokenPricingConfig()
                     
-                    # override_value here is override_config.pricing_config, which is TokenPricingConfig
                     if isinstance(override_value, TokenPricingConfig):
                          self.pricing_config.merge_with(override_value)
-                    elif isinstance(override_value, dict): # Should not happen if override_config is LLMConfig
+                    elif isinstance(override_value, dict):
                         self.pricing_config.merge_with(TokenPricingConfig.from_dict(override_value))
                     else:
                         logger.warning(f"Skipping merge for pricing_config due to unexpected override type: {type(override_value)}")
                 elif f_info.name == 'extra_params':
-                    # For extra_params (dict), merge dictionaries
                     if isinstance(override_value, dict) and isinstance(self.extra_params, dict):
                         self.extra_params.update(override_value)
                     else:
-                        setattr(self, f_info.name, override_value) # Fallback to direct set if types mismatch
+                        setattr(self, f_info.name, override_value)
                 else:
                     setattr(self, f_info.name, override_value)
         logger.debug(f"LLMConfig merged. Current state after merge: rate_limit={self.rate_limit}, temp={self.temperature}, system_message='{self.system_message}'")
-

autobyteus/llm/utils/media_payload_formatter.py

@@ -0,0 +1,99 @@
+import base64
+import mimetypes
+from typing import Dict, Union
+from pathlib import Path
+import httpx
+import logging
+
+logger = logging.getLogger(__name__)
+
+# FIX: Instantiate the client with verify=False to allow for self-signed certificates
+# in local development environments, which is a common use case.
+_http_client = httpx.AsyncClient(verify=False)
+
+# Add a prominent security warning to inform developers about the disabled SSL verification.
+logger.warning(
+    "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
+    "SECURITY WARNING: SSL certificate verification is DISABLED for the image "
+    "downloader (httpx client in media_payload_formatter.py).\n"
+    "This is intended for development and testing with local servers using "
+    "self-signed certificates. In a production environment, this could expose "
+    "the system to Man-in-the-Middle (MitM) attacks when downloading images from "
+    "the public internet.\n"
+    "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
+)
+
+
+def get_mime_type(file_path: str) -> str:
+    """Determine MIME type of file."""
+    mime_type, _ = mimetypes.guess_type(file_path)
+    if not mime_type or not mime_type.startswith('image/'):
+        return 'image/jpeg'  # default fallback
+    return mime_type
+
+
+def is_base64(s: str) -> bool:
+    """Check if a string is a valid base64 encoded string."""
+    try:
+        # Check if the string has valid base64 characters and padding
+        if not isinstance(s, str) or len(s) % 4 != 0:
+            return False
+        base64.b64decode(s, validate=True)
+        return True
+    except (ValueError, TypeError):
+        return False
+
+
+def is_valid_image_path(path: str) -> bool:
+    """Check if path exists and has a valid image extension."""
+    valid_extensions = {".jpg", ".jpeg", ".png", ".gif", ".webp"}
+    try:
+        file_path = Path(path)
+        return file_path.is_file() and file_path.suffix.lower() in valid_extensions
+    except (TypeError, ValueError):
+        return False
+
+
+def create_data_uri(mime_type: str, base64_data: str) -> Dict:
+    """Create properly structured data URI object for API."""
+    return {
+        "type": "image_url",
+        "image_url": {
+            "url": f"data:{mime_type};base64,{base64_data}"
+        }
+    }
+
+def file_to_base64(path: str) -> str:
+    """Reads an image file from a local path and returns it as a base64 encoded string."""
+    try:
+        with open(path, "rb") as img_file:
+            return base64.b64encode(img_file.read()).decode("utf-8")
+    except Exception as e:
+        logger.error(f"Failed to read and encode image file at {path}: {e}")
+        raise
+
+async def url_to_base64(url: str) -> str:
+    """Downloads an image from a URL and returns it as a base64 encoded string."""
+    try:
+        response = await _http_client.get(url)
+        response.raise_for_status()
+        return base64.b64encode(response.content).decode("utf-8")
+    except httpx.HTTPError as e:
+        logger.error(f"Failed to download image from URL {url}: {e}")
+        raise
+
+async def image_source_to_base64(image_source: str) -> str:
+    """
+    Orchestrator function that converts an image source (file path, URL, or existing base64)
+    into a base64 encoded string by delegating to specialized functions.
+    """
+    if is_valid_image_path(image_source):
+        return file_to_base64(image_source)
+    
+    if image_source.startswith(("http://", "https://")):
+        return await url_to_base64(image_source)
+
+    if is_base64(image_source):
+        return image_source
+
+    raise ValueError(f"Invalid image source: not a valid file path, URL, or base64 string.")

autobyteus/llm/utils/messages.py

@@ -7,34 +7,41 @@ class MessageRole(Enum):
     ASSISTANT = "assistant"
 
 class Message:
-    def __init__(self, role: MessageRole, content: Union[str, List[Dict]], reasoning_content: Optional[str] = None):
+    def __init__(self,
+                 role: MessageRole,
+                 content: Optional[str] = None,
+                 reasoning_content: Optional[str] = None,
+                 image_urls: Optional[List[str]] = None,
+                 audio_urls: Optional[List[str]] = None,
+                 video_urls: Optional[List[str]] = None):
         """
-        Initializes a Message.
-        
+        Initializes a rich Message object for conversation history.
+
         Args:
-            role (MessageRole): The role of the message.
-            content (Union[str, List[Dict]]): The content of the message.
-            reasoning_content (Optional[str]): Optional reasoning content for reasoning models.
+            role: The role of the message originator.
+            content: The textual content of the message.
+            reasoning_content: Optional reasoning/thought process from an assistant.
+            image_urls: Optional list of image URIs.
+            audio_urls: Optional list of audio URIs.
+            video_urls: Optional list of video URIs.
         """
         self.role = role
         self.content = content
-        self.reasoning_content = reasoning_content  # Optional field for reasoning content
-
-    def to_dict(self) -> Dict[str, Union[str, List[Dict]]]:
-        result: Dict[str, Union[str, List[Dict]]] = {"role": self.role.value, "content": self.content}
-        if self.reasoning_content:
-            result["reasoning_content"] = self.reasoning_content
-        return result
+        self.reasoning_content = reasoning_content
+        self.image_urls = image_urls or []
+        self.audio_urls = audio_urls or []
+        self.video_urls = video_urls or []
 
-    def to_mistral_message(self):
-        if self.role == MessageRole.USER:
-            from mistralai import UserMessage
-            return UserMessage(content=self.content)
-        elif self.role == MessageRole.ASSISTANT:
-            from mistralai import AssistantMessage
-            return AssistantMessage(content=self.content)
-        elif self.role == MessageRole.SYSTEM:
-            from mistralai import SystemMessage
-            return SystemMessage(content=self.content)
-        else:
-            raise ValueError(f"Unsupported message role: {self.role}")
+    def to_dict(self) -> Dict[str, Union[str, List[str], None]]:
+        """
+        Returns a simple dictionary representation of the Message object.
+        This is for internal use and does not format for any specific API.
+        """
+        return {
+            "role": self.role.value,
+            "content": self.content,
+            "reasoning_content": self.reasoning_content,
+            "image_urls": self.image_urls,
+            "audio_urls": self.audio_urls,
+            "video_urls": self.video_urls,
+        }

autobyteus/llm/utils/response_types.py

@@ -1,5 +1,5 @@
-from dataclasses import dataclass
-from typing import Optional
+from dataclasses import dataclass, field
+from typing import Optional, List
 from autobyteus.llm.utils.token_usage import TokenUsage
 
 @dataclass
@@ -7,6 +7,9 @@ class CompleteResponse:
     content: str
     reasoning: Optional[str] = None
     usage: Optional[TokenUsage] = None
+    image_urls: List[str] = field(default_factory=list)
+    audio_urls: List[str] = field(default_factory=list)
+    video_urls: List[str] = field(default_factory=list)
 
     @classmethod
     def from_content(cls, content: str) -> 'CompleteResponse':
@@ -17,4 +20,7 @@ class ChunkResponse:
     content: str  # The actual content/text of the chunk
     reasoning: Optional[str] = None
     is_complete: bool = False  # Indicates if this is the final chunk
-    usage: Optional[TokenUsage] = None  # Token usage stats, typically available in final chunk
+    usage: Optional[TokenUsage] = None  # Token usage stats, typically available in final chunk
+    image_urls: List[str] = field(default_factory=list)
+    audio_urls: List[str] = field(default_factory=list)
+    video_urls: List[str] = field(default_factory=list)

autobyteus/llm/utils/token_usage.py

@@ -1,8 +1,8 @@
 # file: autobyteus/autobyteus/llm/utils/token_usage.py
 from typing import Optional
-from pydantic import BaseModel # MODIFIED: Import BaseModel
+from pydantic import BaseModel, ConfigDict # MODIFIED: Import ConfigDict
 
-# MODIFIED: Change from dataclass to Pydantic BaseModel
+# MODIFIED: Change from dataclass to Pydantic BaseModel and use model_config
 class TokenUsage(BaseModel):
     prompt_tokens: int
     completion_tokens: int
@@ -11,6 +11,7 @@ class TokenUsage(BaseModel):
     completion_cost: Optional[float] = None
     total_cost: Optional[float] = None
 
-    class Config:
-        populate_by_name = True # If you use aliases, or for general Pydantic v2 compatibility
-        # or model_config = ConfigDict(populate_by_name=True) for Pydantic v2
+    # FIX: Use model_config with ConfigDict for Pydantic v2 compatibility
+    model_config = ConfigDict(
+        populate_by_name=True,
+    )

autobyteus/multimedia/__init__.py

@@ -0,0 +1,31 @@
+from .providers import MultimediaProvider
+from .runtimes import MultimediaRuntime
+from .utils import *
+from .image import *
+from .audio import *
+
+
+__all__ = [
+    # Factories
+    "image_client_factory",
+    "ImageClientFactory",
+    "audio_client_factory",
+    "AudioClientFactory",
+
+    # Models
+    "ImageModel",
+    "AudioModel",
+    
+    # Base Clients
+    "BaseImageClient",
+    "BaseAudioClient",
+
+    # Enums
+    "MultimediaProvider",
+    "MultimediaRuntime",
+
+    # Response Types and Config
+    "ImageGenerationResponse",
+    "SpeechGenerationResponse",
+    "MultimediaConfig",
+]

autobyteus/multimedia/audio/__init__.py

@@ -0,0 +1,11 @@
+from .audio_client_factory import audio_client_factory, AudioClientFactory
+from .audio_model import AudioModel
+from .base_audio_client import BaseAudioClient
+from .api import *
+
+__all__ = [
+    "audio_client_factory",
+    "AudioClientFactory",
+    "AudioModel",
+    "BaseAudioClient",
+]

autobyteus/multimedia/audio/api/__init__.py

@@ -0,0 +1,4 @@
+from .gemini_audio_client import GeminiAudioClient
+from .autobyteus_audio_client import AutobyteusAudioClient
+
+__all__ = ["GeminiAudioClient", "AutobyteusAudioClient"]

autobyteus/multimedia/audio/api/autobyteus_audio_client.py

@@ -0,0 +1,59 @@
+import logging
+from typing import Optional, List, Dict, Any, TYPE_CHECKING
+from autobyteus_llm_client import AutobyteusClient
+from autobyteus.multimedia.audio.base_audio_client import BaseAudioClient
+from autobyteus.multimedia.utils.response_types import SpeechGenerationResponse
+
+if TYPE_CHECKING:
+    from autobyteus.multimedia.audio.audio_model import AudioModel
+    from autobyteus.multimedia.utils.multimedia_config import MultimediaConfig
+
+logger = logging.getLogger(__name__)
+
+class AutobyteusAudioClient(BaseAudioClient):
+    """
+    An audio client that connects to an Autobyteus server instance for audio tasks.
+    """
+
+    def __init__(self, model: "AudioModel", config: "MultimediaConfig"):
+        super().__init__(model, config)
+        if not model.host_url:
+            raise ValueError("AutobyteusAudioClient requires a host_url in its AudioModel.")
+        
+        self.autobyteus_client = AutobyteusClient(server_url=model.host_url)
+        logger.info(f"AutobyteusAudioClient initialized for model '{model.name}' on host '{model.host_url}'.")
+
+    async def generate_speech(
+        self,
+        prompt: str,
+        generation_config: Optional[Dict[str, Any]] = None
+    ) -> SpeechGenerationResponse:
+        """
+        Generates speech by calling the generate_speech endpoint on the remote Autobyteus server.
+        """
+        try:
+            logger.info(f"Sending speech generation request for model '{self.model.name}' to {self.model.host_url}")
+            
+            model_name_for_server = self.model.name
+
+            response_data = await self.autobyteus_client.generate_speech(
+                model_name=model_name_for_server,
+                prompt=prompt,
+                generation_config=generation_config
+            )
+            
+            audio_urls = response_data.get("audio_urls", [])
+            if not audio_urls:
+                raise ValueError("Remote Autobyteus server did not return any audio URLs.")
+                
+            return SpeechGenerationResponse(audio_urls=audio_urls)
+            
+        except Exception as e:
+            logger.error(f"Error calling Autobyteus server for speech generation: {e}", exc_info=True)
+            raise
+
+    async def cleanup(self):
+        """Closes the underlying AutobyteusClient."""
+        if self.autobyteus_client:
+            await self.autobyteus_client.close()
+        logger.debug("AutobyteusAudioClient cleaned up.")