autobyteus 1.1.5__py3-none-any.whl → 1.1.6__py3-none-any.whl

autobyteus/multimedia/image/api/openai_image_client.py

@@ -0,0 +1,142 @@
+import logging
+import os
+from typing import Optional, List, Dict, Any, TYPE_CHECKING
+from openai import OpenAI
+from autobyteus.multimedia.image.base_image_client import BaseImageClient
+from autobyteus.multimedia.utils.response_types import ImageGenerationResponse
+
+if TYPE_CHECKING:
+    from autobyteus.multimedia.image.image_model import ImageModel
+    from autobyteus.multimedia.utils.multimedia_config import MultimediaConfig
+
+logger = logging.getLogger(__name__)
+
+class OpenAIImageClient(BaseImageClient):
+    """
+    An image client that uses OpenAI's DALL-E models.
+    """
+
+    def __init__(self, model: "ImageModel", config: "MultimediaConfig"):
+        super().__init__(model, config)
+        api_key = os.getenv("OPENAI_API_KEY")
+        if not api_key:
+            logger.error("OPENAI_API_KEY environment variable is not set.")
+            raise ValueError("OPENAI_API_KEY environment variable is not set.")
+
+        self.client = OpenAI(api_key=api_key, base_url="https://api.openai.com/v1")
+        logger.info(f"OpenAIImageClient initialized for model '{self.model.name}'.")
+
+    async def generate_image(
+        self,
+        prompt: str,
+        input_image_urls: Optional[List[str]] = None,
+        generation_config: Optional[Dict[str, Any]] = None
+    ) -> ImageGenerationResponse:
+        """
+        Generates an image using an OpenAI DALL-E model via the v1/images/generations endpoint.
+        Note: This endpoint does not support image inputs, even for multimodal models like gpt-image-1.
+        """
+        if input_image_urls:
+            logger.warning(
+                f"The OpenAI `images.generate` API used by this client does not support input images. "
+                f"The images provided for model '{self.model.value}' will be ignored. "
+                f"To use image inputs, a client based on the Chat Completions API is required."
+            )
+
+        try:
+            image_model = self.model.value
+            logger.info(f"Generating image with OpenAI model '{image_model}' and prompt: '{prompt[:50]}...'")
+
+            # Combine default config with any overrides
+            final_config = self.config.to_dict().copy()
+            if generation_config:
+                final_config.update(generation_config)
+
+            response = self.client.images.generate(
+                model=image_model,
+                prompt=prompt,
+                n=final_config.get("n", 1),
+                size=final_config.get("size", "1024x1024"),
+                quality=final_config.get("quality", "standard"),
+                style=final_config.get("style", "vivid"),
+                response_format="url"
+            )
+
+            image_urls_list: List[str] = [img.url for img in response.data if img.url]
+            revised_prompt: Optional[str] = response.data[0].revised_prompt if response.data and hasattr(response.data[0], 'revised_prompt') else None
+
+            if not image_urls_list:
+                raise ValueError("OpenAI API did not return any image URLs.")
+
+            logger.info(f"Successfully generated {len(image_urls_list)} image(s).")
+
+            return ImageGenerationResponse(
+                image_urls=image_urls_list,
+                revised_prompt=revised_prompt
+            )
+        except Exception as e:
+            logger.error(f"Error during OpenAI image generation: {str(e)}")
+            raise ValueError(f"OpenAI image generation failed: {str(e)}")
+
+    async def edit_image(
+        self,
+        prompt: str,
+        input_image_urls: List[str],
+        mask_url: Optional[str] = None,
+        generation_config: Optional[Dict[str, Any]] = None
+    ) -> ImageGenerationResponse:
+        """
+        Edits an image using an OpenAI model that supports the v1/images/edits endpoint.
+        """
+        if not input_image_urls:
+            raise ValueError("At least one input image URL must be provided for editing.")
+
+        source_image_url = input_image_urls[0]
+        if len(input_image_urls) > 1:
+            logger.warning(f"OpenAI edit endpoint only supports one input image. Using '{source_image_url}' and ignoring the rest.")
+
+        try:
+            logger.info(f"Editing image '{source_image_url}' with prompt: '{prompt[:50]}...'")
+
+            # Combine default config with any overrides
+            final_config = self.config.to_dict().copy()
+            if generation_config:
+                final_config.update(generation_config)
+
+            with open(source_image_url, "rb") as image_file:
+                mask_file = open(mask_url, "rb") if mask_url else None
+                try:
+                    response = self.client.images.edit(
+                        image=image_file,
+                        mask=mask_file,
+                        prompt=prompt,
+                        model=self.model.value,
+                        n=final_config.get("n", 1),
+                        size=final_config.get("size", "1024x1024"),
+                        response_format="url"
+                    )
+                finally:
+                    if mask_file:
+                        mask_file.close()
+
+            image_urls_list: List[str] = [img.url for img in response.data if img.url]
+            if not image_urls_list:
+                raise ValueError("OpenAI API did not return any edited image URLs.")
+
+            logger.info(f"Successfully edited image, generated {len(image_urls_list)} version(s).")
+            return ImageGenerationResponse(image_urls=image_urls_list)
+
+        except FileNotFoundError as e:
+            logger.error(f"Image file not found for editing: {e.filename}")
+            raise
+        except Exception as e:
+            logger.error(f"Error during OpenAI image editing: {str(e)}")
+            # The API might return a 400 Bad Request if the model doesn't support edits
+            if "does not support image editing" in str(e):
+                raise ValueError(f"The model '{self.model.value}' does not support the image editing endpoint.")
+            raise ValueError(f"OpenAI image editing failed: {str(e)}")
+
+
+    async def cleanup(self):
+        # The OpenAI client does not require explicit cleanup of a session.
+        logger.debug("OpenAIImageClient cleanup called.")

autobyteus/multimedia/image/autobyteus_image_provider.py

@@ -0,0 +1,109 @@
+import logging
+from typing import Dict, Any, List
+import os
+from urllib.parse import urlparse
+
+from autobyteus_llm_client import AutobyteusClient
+from autobyteus.multimedia.image.api.autobyteus_image_client import AutobyteusImageClient
+from autobyteus.multimedia.image.image_model import ImageModel
+from autobyteus.multimedia.providers import MultimediaProvider
+from autobyteus.multimedia.runtimes import MultimediaRuntime
+
+logger = logging.getLogger(__name__)
+
+class AutobyteusImageModelProvider:
+    """
+    Discovers and registers image models from remote Autobyteus server instances.
+    """
+    DEFAULT_SERVER_URL = 'http://localhost:8000'
+
+    @staticmethod
+    def _get_hosts() -> List[str]:
+        """Gets Autobyteus server hosts from env vars."""
+        hosts_str = os.getenv('AUTOBYTEUS_LLM_SERVER_HOSTS')
+        if hosts_str:
+            return [host.strip() for host in hosts_str.split(',')]
+        
+        legacy_host = os.getenv('AUTOBYTEUS_LLM_SERVER_URL')
+        if legacy_host:
+            return [legacy_host]
+            
+        return [AutobyteusImageModelProvider.DEFAULT_SERVER_URL]
+
+    @staticmethod
+    def discover_and_register():
+        """Discover and register image models from all configured hosts."""
+        try:
+            from autobyteus.multimedia.image.image_client_factory import ImageClientFactory
+
+            hosts = AutobyteusImageModelProvider._get_hosts()
+            total_registered_count = 0
+
+            for host_url in hosts:
+                if not AutobyteusImageModelProvider.is_valid_url(host_url):
+                    logger.error(f"Invalid Autobyteus host URL for image model discovery: {host_url}, skipping.")
+                    continue
+                
+                logger.info(f"Discovering image models from host: {host_url}")
+                client = None
+                try:
+                    client = AutobyteusClient(server_url=host_url)
+                    response = client.get_available_image_models_sync()
+                except Exception as e:
+                    logger.warning(f"Could not fetch models from Autobyteus server at {host_url}: {e}")
+                    continue
+                finally:
+                    if client:
+                        client.sync_client.close()
+
+                if not response.get('models'):
+                    logger.info(f"No image models found on host {host_url}.")
+                    continue
+
+                models = response.get('models', [])
+                host_registered_count = 0
+                for model_info in models:
+                    try:
+                        if not all(k in model_info for k in ["name", "value", "provider"]):
+                            logger.warning(f"Skipping malformed image model from {host_url}: {model_info}")
+                            continue
+
+                        # Heuristic to ensure it's an image model if the server doesn't specify modality
+                        if "parameter_schema" not in model_info:
+                             logger.debug(f"Skipping model from {host_url} as it lacks a parameter schema, likely not an image model: {model_info.get('name')}")
+                             continue
+
+                        image_model = ImageModel(
+                            name=model_info["name"],
+                            value=model_info["value"],
+                            provider=MultimediaProvider(model_info["provider"]),
+                            client_class=AutobyteusImageClient,
+                            runtime=MultimediaRuntime.AUTOBYTEUS,
+                            host_url=host_url,
+                            parameter_schema=model_info.get("parameter_schema")
+                        )
+                        
+                        ImageClientFactory.register_model(image_model)
+                        host_registered_count += 1
+                        
+                    except Exception as e:
+                        logger.error(f"Failed to register image model '{model_info.get('name')}' from {host_url}: {e}")
+                
+                if host_registered_count > 0:
+                    logger.info(f"Registered {host_registered_count} image models from Autobyteus host {host_url}")
+                total_registered_count += host_registered_count
+            
+            if total_registered_count > 0:
+                 logger.info(f"Finished Autobyteus image model discovery. Total models registered: {total_registered_count}")
+
+        except Exception as e:
+            logger.error(f"An unexpected error occurred during Autobyteus image model discovery: {e}", exc_info=True)
+
+    @staticmethod
+    def is_valid_url(url: str) -> bool:
+        """Validate URL format"""
+        try:
+            result = urlparse(url)
+            return all([result.scheme, result.netloc])
+        except Exception:
+            return False

autobyteus/multimedia/image/base_image_client.py

@@ -0,0 +1,67 @@
+from __future__ import annotations
+from abc import ABC, abstractmethod
+from typing import Optional, Dict, Any, List, TYPE_CHECKING
+from autobyteus.multimedia.utils.response_types import ImageGenerationResponse
+
+if TYPE_CHECKING:
+    from autobyteus.multimedia.image.image_model import ImageModel
+    from autobyteus.multimedia.utils.multimedia_config import MultimediaConfig
+
+
+class BaseImageClient(ABC):
+    """
+    Abstract base class for image clients that connect to models for image generation and editing.
+    """
+    def __init__(self, model: "ImageModel", config: "MultimediaConfig"):
+        self.model = model
+        self.config = config
+
+    @abstractmethod
+    async def generate_image(
+        self,
+        prompt: str,
+        input_image_urls: Optional[List[str]] = None,
+        generation_config: Optional[Dict[str, Any]] = None
+    ) -> ImageGenerationResponse:
+        """
+        Generates an image based on a textual prompt.
+
+        Args:
+            prompt (str): The text prompt describing the image to generate.
+            input_image_urls (Optional[List[str]]): A list of URLs or local paths to input images
+                                                    for image-to-image generation.
+            generation_config (Optional[Dict[str, Any]]): Provider-specific parameters for image generation
+                                                        to override defaults.
+                                                        (e.g., n, size, quality, style).
+
+        Returns:
+            ImageGenerationResponse: An object containing URLs to the generated images.
+        """
+        pass
+
+    @abstractmethod
+    async def edit_image(
+        self,
+        prompt: str,
+        input_image_urls: List[str],
+        mask_url: Optional[str] = None,
+        generation_config: Optional[Dict[str, Any]] = None
+    ) -> ImageGenerationResponse:
+        """
+        Edits an existing image based on a textual prompt.
+
+        Args:
+            prompt (str): A text prompt describing the desired edits.
+            input_image_urls (List[str]): The path(s) or URL(s) to the source image(s) to edit.
+            mask_url (Optional[str]): The path to a mask image. The transparent areas of the mask
+                                       indicate where the image should be edited.
+            generation_config (Optional[Dict[str, Any]]): Provider-specific parameters.
+
+        Returns:
+            ImageGenerationResponse: An object containing URLs to the edited images.
+        """
+        pass
+
+    async def cleanup(self):
+        """Optional cleanup method for resources like network clients."""
+        pass

autobyteus/multimedia/image/image_client_factory.py

@@ -0,0 +1,118 @@
+import logging
+from typing import Dict, Optional
+from autobyteus.multimedia.image.autobyteus_image_provider import AutobyteusImageModelProvider
+from autobyteus.multimedia.image.base_image_client import BaseImageClient
+from autobyteus.multimedia.image.image_model import ImageModel
+from autobyteus.multimedia.providers import MultimediaProvider
+from autobyteus.multimedia.image.api.openai_image_client import OpenAIImageClient
+from autobyteus.multimedia.image.api.gemini_image_client import GeminiImageClient
+from autobyteus.multimedia.utils.multimedia_config import MultimediaConfig
+from autobyteus.utils.singleton import SingletonMeta
+
+logger = logging.getLogger(__name__)
+
+class ImageClientFactory(metaclass=SingletonMeta):
+    """
+    A factory for creating instances of image generation clients based on registered ImageModels.
+    """
+    _models_by_identifier: Dict[str, ImageModel] = {}
+    _initialized = False
+
+    @staticmethod
+    def ensure_initialized():
+        """Ensures the factory is initialized before use."""
+        if not ImageClientFactory._initialized:
+            ImageClientFactory._initialize_registry()
+            ImageClientFactory._initialized = True
+
+    @staticmethod
+    def reinitialize():
+        """Reinitializes the model registry, clearing all models and re-discovering them."""
+        logger.info("Reinitializing Image model registry...")
+        ImageClientFactory._initialized = False
+        ImageClientFactory._models_by_identifier.clear()
+        ImageClientFactory.ensure_initialized()
+        logger.info("Image model registry reinitialized successfully.")
+
+    @staticmethod
+    def _initialize_registry():
+        """Initializes the registry with built-in image models and discovers remote ones."""
+        
+        # OpenAI Models
+        gpt_image_1_model = ImageModel(
+            name="gpt-image-1",
+            value="dall-e-3",
+            provider=MultimediaProvider.OPENAI,
+            client_class=OpenAIImageClient,
+            parameter_schema={
+                "n": {"type": "integer", "default": 1, "allowed_values": [1], "description": "The number of images to generate."},
+                "size": {"type": "string", "default": "1024x1024", "allowed_values": ["1024x1024", "1792x1024", "1024x1792"], "description": "The size of the generated images."},
+                "quality": {"type": "string", "default": "hd", "allowed_values": ["standard", "hd"], "description": "The quality of the image that will be generated."},
+                "style": {"type": "string", "default": "vivid", "allowed_values": ["vivid", "natural"], "description": "The style of the generated images."}
+            }
+        )
+
+        dall_e_2_model = ImageModel(
+            name="dall-e-2",
+            value="dall-e-2",
+            provider=MultimediaProvider.OPENAI,
+            client_class=OpenAIImageClient,
+            parameter_schema={
+                "n": {"type": "integer", "default": 1, "description": "The number of images to generate."},
+                "size": {"type": "string", "default": "1024x1024", "allowed_values": ["256x256", "512x512", "1024x1024"], "description": "The size of the generated images."}
+            }
+        )
+
+        # Google Imagen Models (via Gemini API)
+        imagen_model = ImageModel(
+            name="imagen-4",
+            value="imagen-4.0-generate-001",
+            provider=MultimediaProvider.GOOGLE,
+            client_class=GeminiImageClient,
+            parameter_schema={} # The genai library doesn't expose these as simple params
+        )
+
+        models_to_register = [
+            gpt_image_1_model,
+            dall_e_2_model,
+            imagen_model,
+        ]
+        
+        for model in models_to_register:
+            ImageClientFactory.register_model(model)
+        
+        logger.info("Default API-based image models registered.")
+
+        # Discover models from remote Autobyteus servers
+        AutobyteusImageModelProvider.discover_and_register()
+
+    @staticmethod
+    def register_model(model: ImageModel):
+        """Registers a new image model."""
+        identifier = model.model_identifier
+        if identifier in ImageClientFactory._models_by_identifier:
+            logger.warning(f"Image model '{identifier}' is already registered. Overwriting.")
+        
+        if not isinstance(model.provider, MultimediaProvider):
+            try:
+                model.provider = MultimediaProvider(model.provider)
+            except ValueError:
+                logger.error(f"Cannot register model '{identifier}' with unknown provider '{model.provider}'.")
+                return
+
+        ImageClientFactory._models_by_identifier[identifier] = model
+
+    @staticmethod
+    def create_image_client(model_identifier: str, config_override: Optional[MultimediaConfig] = None) -> BaseImageClient:
+        """Creates an instance of a registered image client for a specific model."""
+        ImageClientFactory.ensure_initialized()
+        
+        model = ImageClientFactory._models_by_identifier.get(model_identifier)
+        if not model:
+            raise ValueError(f"No image model registered with the name '{model_identifier}'. "
+                             f"Available models: {list(ImageClientFactory._models_by_identifier.keys())}")
+        
+        logger.info(f"Creating instance of image client for model '{model_identifier}'.")
+        return model.create_client(config_override)
+
+image_client_factory = ImageClientFactory()

autobyteus/multimedia/image/image_model.py

@@ -0,0 +1,96 @@
+from __future__ import annotations
+import logging
+from typing import TYPE_CHECKING, Type, Optional, Iterator, Dict, Any
+from urllib.parse import urlparse
+
+from autobyteus.multimedia.providers import MultimediaProvider
+from autobyteus.multimedia.runtimes import MultimediaRuntime
+from autobyteus.multimedia.utils.multimedia_config import MultimediaConfig
+
+if TYPE_CHECKING:
+    from autobyteus.multimedia.image.base_image_client import BaseImageClient
+
+logger = logging.getLogger(__name__)
+
+class ImageModelMeta(type):
+    """
+    Metaclass for ImageModel to allow discovery and access like an Enum.
+    """
+    def __iter__(cls) -> Iterator[ImageModel]:
+        from autobyteus.multimedia.image.image_client_factory import ImageClientFactory
+        ImageClientFactory.ensure_initialized()
+        for model in ImageClientFactory._models_by_identifier.values():
+            yield model
+
+    def __getitem__(cls, name_or_identifier: str) -> ImageModel:
+        from autobyteus.multimedia.image.image_client_factory import ImageClientFactory
+        ImageClientFactory.ensure_initialized()
+        model = ImageClientFactory._models_by_identifier.get(name_or_identifier)
+        if model:
+            return model
+        raise KeyError(f"Image model '{name_or_identifier}' not found.")
+
+    def __len__(cls) -> int:
+        from autobyteus.multimedia.image.image_client_factory import ImageClientFactory
+        ImageClientFactory.ensure_initialized()
+        return len(ImageClientFactory._models_by_identifier)
+
+
+class ImageModel(metaclass=ImageModelMeta):
+    """
+    Represents a single image model's metadata.
+    """
+    def __init__(
+        self,
+        name: str,
+        value: str,
+        provider: MultimediaProvider,
+        client_class: Type["BaseImageClient"],
+        parameter_schema: Optional[Dict[str, Any]] = None,
+        runtime: MultimediaRuntime = MultimediaRuntime.API,
+        host_url: Optional[str] = None
+    ):
+        self.name = name
+        self.value = value
+        self.provider = provider
+        self.client_class = client_class
+        self.runtime = runtime
+        self.host_url = host_url
+        self.parameter_schema = parameter_schema if parameter_schema else {}
+
+        # Automatically build default_config from the schema's default values
+        default_params = {
+            key: meta.get("default")
+            for key, meta in self.parameter_schema.items()
+            if "default" in meta
+        }
+        self.default_config = MultimediaConfig(params=default_params)
+
+    @property
+    def model_identifier(self) -> str:
+        """Returns the unique identifier for the model."""
+        if self.runtime == MultimediaRuntime.AUTOBYTEUS and self.host_url:
+            try:
+                host = urlparse(self.host_url).hostname
+                return f"{self.name}@{host}"
+            except Exception:
+                return f"{self.name}@{self.host_url}" # Fallback
+        return self.name
+
+    def create_client(self, config_override: Optional[MultimediaConfig] = None) -> "BaseImageClient":
+        """
+        Instantiates the client class for this model.
+        """
+        config_to_use = self.default_config
+        if config_override:
+            from copy import deepcopy
+            config_to_use = deepcopy(self.default_config)
+            config_to_use.merge_with(config_override)
+        
+        return self.client_class(model=self, config=config_to_use)
+
+    def __repr__(self):
+        return (
+            f"ImageModel(identifier='{self.model_identifier}', "
+            f"provider='{self.provider.name}', runtime='{self.runtime.value}')"
+        )

autobyteus/multimedia/providers.py

@@ -0,0 +1,5 @@
+from enum import Enum
+
+class MultimediaProvider(Enum):
+    OPENAI = "OPENAI"
+    GOOGLE = "GOOGLE"

autobyteus/multimedia/runtimes.py

@@ -0,0 +1,8 @@
+from enum import Enum
+
+class MultimediaRuntime(Enum):
+    """
+    Specifies the execution environment for a multimedia model.
+    """
+    API = "api"
+    AUTOBYTEUS = "autobyteus"

autobyteus/multimedia/utils/__init__.py

@@ -0,0 +1,10 @@
+from .multimedia_config import MultimediaConfig
+from .response_types import ImageGenerationResponse, SpeechGenerationResponse
+from .api_utils import load_image_from_url
+
+__all__ = [
+    "MultimediaConfig",
+    "ImageGenerationResponse",
+    "SpeechGenerationResponse",
+    "load_image_from_url",
+]

autobyteus/multimedia/utils/api_utils.py

@@ -0,0 +1,19 @@
+import logging
+from PIL import Image
+import requests
+
+logger = logging.getLogger(__name__)
+
+def load_image_from_url(url: str) -> Image.Image:
+    """Loads an image from a URL (http, https, or file path)."""
+    try:
+        if url.startswith(('http://', 'https://')):
+            response = requests.get(url, stream=True)
+            response.raise_for_status()
+            return Image.open(response.raw)
+        else:
+            # Assume it's a local file path
+            return Image.open(url)
+    except Exception as e:
+        logger.error(f"Failed to load image from URL/path '{url}': {e}")
+        raise

autobyteus/multimedia/utils/multimedia_config.py

@@ -0,0 +1,29 @@
+from dataclasses import dataclass, field
+from typing import Optional, Dict, Any
+
+@dataclass
+class MultimediaConfig:
+    """
+    Configuration for multimedia generation, using a flexible dictionary for parameters.
+    """
+    params: Dict[str, Any] = field(default_factory=dict)
+
+    def merge_with(self, override_config: Optional['MultimediaConfig']):
+        """
+        Merges parameters from an override config into this one.
+        """
+        if override_config and override_config.params:
+            self.params.update(override_config.params)
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> 'MultimediaConfig':
+        """
+        Creates a MultimediaConfig instance from a dictionary of parameters.
+        """
+        return cls(params=data if data is not None else {})
+
+    def to_dict(self) -> Dict[str, Any]:
+        """
+        Returns the configuration parameters as a dictionary.
+        """
+        return self.params

autobyteus/multimedia/utils/response_types.py

@@ -0,0 +1,13 @@
+from dataclasses import dataclass
+from typing import Optional, List
+
+@dataclass
+class ImageGenerationResponse:
+    """Response for image generation or editing."""
+    image_urls: List[str]
+    revised_prompt: Optional[str] = None
+
+@dataclass
+class SpeechGenerationResponse:
+    """Response for speech generation (Text-to-Speech)."""
+    audio_urls: List[str]

autobyteus/tools/__init__.py

@@ -25,6 +25,7 @@ from .file.file_writer import file_writer
 # General Class-based tools
 from .image_downloader import ImageDownloader
 from .timer import Timer
+from .multimedia.image_tools import GenerateImageTool, EditImageTool
 
 # Standalone Browser tools
 from .browser.standalone.google_search_ui import GoogleSearch
@@ -62,6 +63,8 @@ __all__ = [
     # Re-exported general class-based tools
     "ImageDownloader",
     "Timer",
+    "GenerateImageTool",
+    "EditImageTool",
 
     # Re-exported Standalone Browser tools
     "GoogleSearch",

autobyteus/tools/multimedia/__init__.py

@@ -0,0 +1,8 @@
+from .image_tools import GenerateImageTool, EditImageTool
+from .audio_tools import GenerateSpeechTool
+
+__all__ = [
+    "GenerateImageTool",
+    "EditImageTool",
+    "GenerateSpeechTool",
+]