abstractcore 2.4.2__py3-none-any.whl → 2.4.4__py3-none-any.whl

abstractcore/media/capabilities.py

@@ -0,0 +1,335 @@
+"""
+Media capability detection and management for AbstractCore.
+
+This module provides comprehensive capability detection for multimodal models,
+leveraging the existing model_capabilities.json infrastructure to determine
+what media types and formats each model supports.
+"""
+
+from dataclasses import dataclass, field
+from typing import Dict, List, Any, Optional, Union
+from pathlib import Path
+import logging
+
+from ..architectures import get_model_capabilities
+from .types import MediaType
+
+
+@dataclass
+class MediaCapabilities:
+    """
+    Comprehensive media capabilities for a specific model.
+
+    This class aggregates all media-related capabilities from model_capabilities.json
+    and provides convenient methods for checking media support.
+    """
+    model_name: str
+
+    # Core media support
+    vision_support: bool = False
+    audio_support: bool = False
+    video_support: bool = False
+
+    # Image capabilities
+    max_images_per_message: int = 1
+    supported_image_formats: List[str] = field(default_factory=lambda: ['jpg', 'jpeg', 'png'])
+    image_resolutions: List[str] = field(default_factory=list)
+    max_image_size_bytes: int = 5 * 1024 * 1024  # 5MB default
+
+    # Document capabilities
+    document_support: bool = True  # Most models can handle text documents
+    max_document_size_bytes: int = 50 * 1024 * 1024  # 50MB default
+
+    # Provider-specific features
+    multimodal_message_support: bool = True
+    text_embedding_preferred: bool = False  # For local models
+    streaming_media_support: bool = False
+
+    # Advanced features
+    parallel_media_processing: bool = False
+    media_token_estimation: bool = True
+
+    @classmethod
+    def from_model_capabilities(cls, model: str, provider: str = None) -> 'MediaCapabilities':
+        """
+        Create MediaCapabilities from model_capabilities.json data.
+
+        Args:
+            model: Model name to look up capabilities for
+            provider: Optional provider name for provider-specific adjustments
+
+        Returns:
+            MediaCapabilities instance with detected capabilities
+        """
+        caps = get_model_capabilities(model)
+        if not caps:
+            caps = {}
+
+        # Base capabilities from JSON
+        instance = cls(
+            model_name=model,
+            vision_support=caps.get('vision_support', False),
+            audio_support=caps.get('audio_support', False),
+            video_support=caps.get('video_support', False),
+            image_resolutions=caps.get('image_resolutions', [])
+        )
+
+        # Provider-specific adjustments
+        if provider:
+            instance._apply_provider_adjustments(provider, caps)
+
+        # Model-specific adjustments based on model name patterns
+        instance._apply_model_adjustments(caps)
+
+        return instance
+
+    def _apply_provider_adjustments(self, provider: str, caps: Dict[str, Any]):
+        """Apply provider-specific capability adjustments."""
+        provider_lower = provider.lower()
+
+        if provider_lower == "openai":
+            self.max_images_per_message = 10 if "gpt-4o" in self.model_name.lower() else 1
+            self.max_image_size_bytes = 20 * 1024 * 1024  # 20MB for OpenAI
+            self.supported_image_formats = ['png', 'jpeg', 'jpg', 'gif', 'webp']
+            self.streaming_media_support = True
+
+        elif provider_lower == "anthropic":
+            self.max_images_per_message = 20  # Claude supports up to 20 images
+            self.max_image_size_bytes = 5 * 1024 * 1024  # 5MB for Anthropic
+            self.supported_image_formats = ['png', 'jpeg', 'jpg', 'gif', 'webp']
+            self.streaming_media_support = True
+
+        elif provider_lower in ["ollama", "mlx", "lmstudio"]:
+            self.text_embedding_preferred = True  # Local models often prefer text
+            self.multimodal_message_support = True
+            self.streaming_media_support = False
+            self.max_image_size_bytes = 10 * 1024 * 1024  # 10MB for local
+
+        elif provider_lower == "huggingface":
+            self.streaming_media_support = False
+            self.max_image_size_bytes = 15 * 1024 * 1024  # 15MB for HF
+
+    def _apply_model_adjustments(self, caps: Dict[str, Any]):
+        """Apply model-specific capability adjustments based on model patterns."""
+        model_lower = self.model_name.lower()
+
+        # Vision model patterns
+        if any(pattern in model_lower for pattern in ['vision', 'vl', 'visual']):
+            self.vision_support = True
+            if 'qwen' in model_lower:
+                self.max_images_per_message = 5  # Qwen-VL supports multiple images
+
+        # Multimodal model patterns
+        if any(pattern in model_lower for pattern in ['4o', 'multimodal', 'omni']):
+            self.vision_support = True
+            if 'audio' not in caps or caps.get('audio_support'):
+                self.audio_support = True
+
+        # Local model adjustments
+        if any(pattern in model_lower for pattern in ['llama', 'qwen', 'phi', 'gemma']):
+            self.text_embedding_preferred = True
+
+    def supports_media_type(self, media_type: MediaType) -> bool:
+        """Check if the model supports a specific media type."""
+        if media_type == MediaType.IMAGE:
+            return self.vision_support
+        elif media_type == MediaType.AUDIO:
+            return self.audio_support
+        elif media_type == MediaType.VIDEO:
+            return self.video_support
+        elif media_type in [MediaType.DOCUMENT, MediaType.TEXT]:
+            return self.document_support
+        return False
+
+    def get_image_limits(self) -> Dict[str, Any]:
+        """Get image-specific limits and capabilities."""
+        return {
+            'max_images_per_message': self.max_images_per_message,
+            'supported_formats': self.supported_image_formats,
+            'max_size_bytes': self.max_image_size_bytes,
+            'supported_resolutions': self.image_resolutions,
+            'vision_support': self.vision_support
+        }
+
+    def get_document_limits(self) -> Dict[str, Any]:
+        """Get document-specific limits and capabilities."""
+        return {
+            'max_size_bytes': self.max_document_size_bytes,
+            'document_support': self.document_support,
+            'text_embedding_preferred': self.text_embedding_preferred
+        }
+
+    def estimate_media_tokens(self, media_type: MediaType, content_size: int = 0) -> int:
+        """
+        Estimate token usage for media content.
+
+        Args:
+            media_type: Type of media
+            content_size: Size of content in bytes (optional)
+
+        Returns:
+            Estimated token count
+        """
+        if not self.media_token_estimation:
+            return 0
+
+        if media_type == MediaType.IMAGE and self.vision_support:
+            # Base token cost for images varies by model
+            model_lower = self.model_name.lower()
+            if 'gpt-4o' in model_lower:
+                return 85 + (170 * 4)  # Simplified GPT-4o calculation
+            elif 'claude' in model_lower:
+                return 1600  # Anthropic standard
+            else:
+                return 512  # Conservative estimate for local models
+
+        elif media_type in [MediaType.TEXT, MediaType.DOCUMENT]:
+            # Text content token estimation
+            if content_size > 0:
+                return content_size // 4  # ~4 chars per token
+            return 100  # Default estimate
+
+        return 0
+
+    def validate_media_content(self, media_type: MediaType, file_size: int = 0,
+                              format: str = None) -> tuple[bool, Optional[str]]:
+        """
+        Validate if media content meets model requirements.
+
+        Args:
+            media_type: Type of media
+            file_size: Size of file in bytes
+            format: File format/extension
+
+        Returns:
+            Tuple of (is_valid, error_message)
+        """
+        if not self.supports_media_type(media_type):
+            return False, f"Model {self.model_name} does not support {media_type.value} content"
+
+        if media_type == MediaType.IMAGE:
+            if file_size > self.max_image_size_bytes:
+                return False, f"Image size ({file_size} bytes) exceeds limit ({self.max_image_size_bytes} bytes)"
+
+            if format and format.lower() not in [f.lower() for f in self.supported_image_formats]:
+                return False, f"Image format '{format}' not supported. Supported: {self.supported_image_formats}"
+
+        elif media_type in [MediaType.DOCUMENT, MediaType.TEXT]:
+            if file_size > self.max_document_size_bytes:
+                return False, f"Document size ({file_size} bytes) exceeds limit ({self.max_document_size_bytes} bytes)"
+
+        return True, None
+
+    def get_processing_strategy(self, media_type: MediaType) -> str:
+        """
+        Get the recommended processing strategy for this media type.
+
+        Returns:
+            Processing strategy: 'multimodal', 'text_embedding', or 'unsupported'
+        """
+        if not self.supports_media_type(media_type):
+            return 'unsupported'
+
+        if media_type == MediaType.IMAGE and self.vision_support:
+            if self.text_embedding_preferred:
+                return 'text_embedding'  # Local models often prefer text description
+            else:
+                return 'multimodal'
+
+        elif media_type in [MediaType.DOCUMENT, MediaType.TEXT]:
+            return 'text_embedding'  # Always embed documents as text
+
+        return 'unsupported'
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert to dictionary for serialization."""
+        return {
+            'model_name': self.model_name,
+            'vision_support': self.vision_support,
+            'audio_support': self.audio_support,
+            'video_support': self.video_support,
+            'max_images_per_message': self.max_images_per_message,
+            'supported_image_formats': self.supported_image_formats,
+            'image_resolutions': self.image_resolutions,
+            'max_image_size_bytes': self.max_image_size_bytes,
+            'document_support': self.document_support,
+            'max_document_size_bytes': self.max_document_size_bytes,
+            'multimodal_message_support': self.multimodal_message_support,
+            'text_embedding_preferred': self.text_embedding_preferred,
+            'streaming_media_support': self.streaming_media_support,
+            'parallel_media_processing': self.parallel_media_processing,
+            'media_token_estimation': self.media_token_estimation
+        }
+
+
+def get_media_capabilities(model: str, provider: str = None) -> MediaCapabilities:
+    """
+    Get comprehensive media capabilities for a model.
+
+    Args:
+        model: Model name
+        provider: Optional provider name for provider-specific adjustments
+
+    Returns:
+        MediaCapabilities instance
+    """
+    return MediaCapabilities.from_model_capabilities(model, provider)
+
+
+def is_vision_model(model: str) -> bool:
+    """Quick check if a model supports vision."""
+    caps = get_media_capabilities(model)
+    return caps.vision_support
+
+
+def is_multimodal_model(model: str) -> bool:
+    """Quick check if a model supports any multimodal content."""
+    caps = get_media_capabilities(model)
+    return caps.vision_support or caps.audio_support or caps.video_support
+
+
+def get_supported_media_types(model: str, provider: str = None) -> List[MediaType]:
+    """
+    Get list of supported media types for a model.
+
+    Args:
+        model: Model name
+        provider: Optional provider name
+
+    Returns:
+        List of supported MediaType values
+    """
+    caps = get_media_capabilities(model, provider)
+    supported = []
+
+    if caps.vision_support:
+        supported.append(MediaType.IMAGE)
+    if caps.audio_support:
+        supported.append(MediaType.AUDIO)
+    if caps.video_support:
+        supported.append(MediaType.VIDEO)
+    if caps.document_support:
+        supported.extend([MediaType.DOCUMENT, MediaType.TEXT])
+
+    return supported
+
+
+# Convenience functions for common capability checks
+def supports_images(model: str, provider: str = None) -> bool:
+    """Check if model supports image processing."""
+    return get_media_capabilities(model, provider).vision_support
+
+
+def supports_documents(model: str, provider: str = None) -> bool:
+    """Check if model supports document processing."""
+    return get_media_capabilities(model, provider).document_support
+
+
+def get_max_images(model: str, provider: str = None) -> int:
+    """Get maximum images per message for model."""
+    return get_media_capabilities(model, provider).max_images_per_message
+
+
+def should_use_text_embedding(model: str, provider: str = None) -> bool:
+    """Check if model prefers text embedding over multimodal messages."""
+    return get_media_capabilities(model, provider).text_embedding_preferred

abstractcore/media/types.py

@@ -0,0 +1,300 @@
+"""
+Core media types and models for AbstractCore multimodal support.
+
+This module defines the fundamental data structures for handling various media types
+across different LLM providers, following AbstractCore's unified interface patterns.
+"""
+
+import base64
+import mimetypes
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Union, Dict, Any, Optional, List, Literal
+from enum import Enum
+
+from pydantic import BaseModel, Field, validator
+
+
+class MediaType(Enum):
+    """Supported media types for multimodal processing."""
+    IMAGE = "image"
+    DOCUMENT = "document"
+    AUDIO = "audio"
+    VIDEO = "video"
+    TEXT = "text"
+
+
+class ContentFormat(Enum):
+    """Different ways media content can be represented."""
+    BASE64 = "base64"
+    URL = "url"
+    FILE_PATH = "file_path"
+    TEXT = "text"
+    BINARY = "binary"
+    AUTO = "auto"
+
+
+@dataclass
+class MediaContent:
+    """
+    Represents a piece of media content with metadata.
+
+    This is the core data structure for all media handling in AbstractCore.
+    It provides a unified way to represent different types of content regardless
+    of the underlying provider.
+    """
+    media_type: MediaType
+    content: Union[str, bytes]
+    content_format: ContentFormat
+    mime_type: str
+    file_path: Optional[str] = None
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+    def __post_init__(self):
+        """Validate and normalize the MediaContent after initialization."""
+        # Auto-detect MIME type if not provided and we have a file path
+        if self.mime_type == "auto" and self.file_path:
+            detected_mime, _ = mimetypes.guess_type(self.file_path)
+            self.mime_type = detected_mime or "application/octet-stream"
+
+        # Ensure content format matches content type
+        if self.content_format == ContentFormat.BASE64 and isinstance(self.content, bytes):
+            self.content = base64.b64encode(self.content).decode('utf-8')
+        elif self.content_format == ContentFormat.TEXT and isinstance(self.content, bytes):
+            self.content = self.content.decode('utf-8')
+
+
+class MultimodalMessage(BaseModel):
+    """
+    A message that can contain both text and media content.
+
+    This follows the pattern of modern multimodal APIs where a single message
+    can contain multiple content elements of different types.
+    """
+    role: str = Field(..., description="Message role (user, assistant, system)")
+    content: List[Union[str, Dict[str, Any]]] = Field(
+        default_factory=list,
+        description="Mixed content list containing text strings and media objects"
+    )
+    metadata: Dict[str, Any] = Field(default_factory=dict)
+
+    @validator('role')
+    def validate_role(cls, v):
+        valid_roles = {'user', 'assistant', 'system', 'tool'}
+        if v not in valid_roles:
+            raise ValueError(f"Role must be one of {valid_roles}")
+        return v
+
+    def add_text(self, text: str) -> None:
+        """Add text content to the message."""
+        self.content.append(text)
+
+    def add_media(self, media: MediaContent) -> None:
+        """Add media content to the message."""
+        media_dict = {
+            "type": "media",
+            "media_type": media.media_type.value,
+            "content": media.content,
+            "content_format": media.content_format.value,
+            "mime_type": media.mime_type,
+            "metadata": media.metadata
+        }
+        if media.file_path:
+            media_dict["file_path"] = media.file_path
+
+        self.content.append(media_dict)
+
+    def has_media(self) -> bool:
+        """Check if this message contains any media content."""
+        return any(
+            isinstance(item, dict) and item.get("type") == "media"
+            for item in self.content
+        )
+
+    def get_text_content(self) -> str:
+        """Extract all text content from the message."""
+        text_parts = [
+            item for item in self.content
+            if isinstance(item, str)
+        ]
+        return " ".join(text_parts)
+
+    def get_media_content(self) -> List[Dict[str, Any]]:
+        """Extract all media content from the message."""
+        return [
+            item for item in self.content
+            if isinstance(item, dict) and item.get("type") == "media"
+        ]
+
+
+@dataclass
+class MediaCapabilities:
+    """
+    Represents what media capabilities a provider/model supports.
+
+    This is used for intelligent routing and validation of media content
+    based on the target provider and model capabilities.
+    """
+    vision_support: bool = False
+    audio_support: bool = False
+    video_support: bool = False
+    document_support: bool = False
+
+    # Image-specific capabilities
+    max_image_resolution: Optional[str] = None
+    supported_image_formats: List[str] = field(default_factory=lambda: ["jpg", "png"])
+
+    # Document-specific capabilities
+    supported_document_formats: List[str] = field(default_factory=lambda: ["pdf", "txt"])
+
+    # Audio/Video capabilities
+    max_audio_duration: Optional[int] = None  # in seconds
+    max_video_duration: Optional[int] = None  # in seconds
+
+    # Provider-specific limits
+    max_file_size: Optional[int] = None  # in bytes
+    max_concurrent_media: int = 1
+
+    def supports_media_type(self, media_type: MediaType) -> bool:
+        """Check if this provider supports the given media type."""
+        support_map = {
+            MediaType.IMAGE: self.vision_support,
+            MediaType.AUDIO: self.audio_support,
+            MediaType.VIDEO: self.video_support,
+            MediaType.DOCUMENT: self.document_support,
+            MediaType.TEXT: True  # All providers support text
+        }
+        return support_map.get(media_type, False)
+
+    def supports_format(self, media_type: MediaType, format_ext: str) -> bool:
+        """Check if this provider supports the specific format."""
+        format_ext = format_ext.lower().lstrip('.')
+
+        if media_type == MediaType.IMAGE:
+            return format_ext in self.supported_image_formats
+        elif media_type == MediaType.DOCUMENT:
+            return format_ext in self.supported_document_formats
+        elif media_type in [MediaType.AUDIO, MediaType.VIDEO]:
+            # For now, assume basic support if the media type is supported
+            return self.supports_media_type(media_type)
+        else:
+            return True
+
+
+class MediaProcessingResult(BaseModel):
+    """
+    Result of processing a media file.
+
+    Contains the processed content and metadata about the processing operation.
+    """
+    success: bool
+    media_content: Optional[MediaContent] = None
+    error_message: Optional[str] = None
+    processing_time: Optional[float] = None
+    extracted_text: Optional[str] = None
+    metadata: Dict[str, Any] = Field(default_factory=dict)
+
+    @property
+    def failed(self) -> bool:
+        """Check if processing failed."""
+        return not self.success
+
+
+# File extension mappings for quick media type detection
+FILE_TYPE_MAPPINGS = {
+    # Images
+    'jpg': MediaType.IMAGE, 'jpeg': MediaType.IMAGE, 'png': MediaType.IMAGE,
+    'gif': MediaType.IMAGE, 'bmp': MediaType.IMAGE, 'tif': MediaType.IMAGE,
+    'tiff': MediaType.IMAGE, 'webp': MediaType.IMAGE, 'ico': MediaType.IMAGE,
+
+    # Documents
+    'pdf': MediaType.DOCUMENT, 'doc': MediaType.DOCUMENT, 'docx': MediaType.DOCUMENT,
+    'xls': MediaType.DOCUMENT, 'xlsx': MediaType.DOCUMENT, 'ppt': MediaType.DOCUMENT,
+    'pptx': MediaType.DOCUMENT, 'odt': MediaType.DOCUMENT, 'rtf': MediaType.DOCUMENT,
+
+    # Text formats
+    'txt': MediaType.TEXT, 'md': MediaType.TEXT, 'csv': MediaType.TEXT,
+    'tsv': MediaType.TEXT, 'json': MediaType.TEXT, 'xml': MediaType.TEXT,
+    'html': MediaType.TEXT, 'htm': MediaType.TEXT,
+
+    # Audio
+    'mp3': MediaType.AUDIO, 'wav': MediaType.AUDIO, 'm4a': MediaType.AUDIO,
+    'ogg': MediaType.AUDIO, 'flac': MediaType.AUDIO, 'aac': MediaType.AUDIO,
+
+    # Video
+    'mp4': MediaType.VIDEO, 'avi': MediaType.VIDEO, 'mov': MediaType.VIDEO,
+    'mkv': MediaType.VIDEO, 'webm': MediaType.VIDEO, 'wmv': MediaType.VIDEO,
+}
+
+
+def detect_media_type(file_path: Union[str, Path]) -> MediaType:
+    """
+    Detect the media type of a file based on its extension.
+
+    Args:
+        file_path: Path to the file
+
+    Returns:
+        MediaType enum value
+    """
+    path = Path(file_path)
+    extension = path.suffix.lower().lstrip('.')
+
+    return FILE_TYPE_MAPPINGS.get(extension, MediaType.DOCUMENT)
+
+
+def create_media_content(
+    file_path: Union[str, Path],
+    content_format: ContentFormat = ContentFormat.AUTO,
+    mime_type: str = "auto"
+) -> MediaContent:
+    """
+    Create a MediaContent object from a file path.
+
+    Args:
+        file_path: Path to the media file
+        content_format: How to represent the content
+        mime_type: MIME type of the content (auto-detected if "auto")
+
+    Returns:
+        MediaContent object
+    """
+    path = Path(file_path)
+
+    if not path.exists():
+        raise FileNotFoundError(f"File not found: {file_path}")
+
+    media_type = detect_media_type(path)
+
+    # Auto-select content format based on media type
+    if content_format == ContentFormat.AUTO:
+        if media_type in [MediaType.IMAGE, MediaType.AUDIO, MediaType.VIDEO]:
+            content_format = ContentFormat.BASE64
+        else:
+            content_format = ContentFormat.TEXT
+
+    # Read and encode content based on format
+    if content_format == ContentFormat.BASE64:
+        with open(path, 'rb') as f:
+            content = base64.b64encode(f.read()).decode('utf-8')
+    elif content_format == ContentFormat.TEXT:
+        with open(path, 'r', encoding='utf-8') as f:
+            content = f.read()
+    elif content_format == ContentFormat.FILE_PATH:
+        content = str(path)
+    else:
+        with open(path, 'rb') as f:
+            content = f.read()
+
+    return MediaContent(
+        media_type=media_type,
+        content=content,
+        content_format=content_format,
+        mime_type=mime_type,
+        file_path=str(path),
+        metadata={
+            'file_size': path.stat().st_size,
+            'file_name': path.name,
+            'file_extension': path.suffix
+        }
+    )