kweaver-dolphin 0.1.0__py3-none-any.whl

dolphin/cli/multimodal/handler.py

@@ -0,0 +1,249 @@
+"""
+Multimodal input handler for CLI.
+
+Integrates parsing, image reading, and processing to convert user input
+with multimodal markers into proper multimodal content.
+"""
+
+import os
+import base64
+from typing import Union, List, Dict, Any, Optional
+
+from dolphin.cli.multimodal.input_parser import (
+    MultimodalInputParser,
+    ParsedMultimodalInput,
+    ImageSourceType,
+)
+from dolphin.cli.multimodal.clipboard import ClipboardImageReader
+from dolphin.cli.multimodal.image_processor import ImageProcessor, ImageProcessConfig
+from dolphin.core.common.multimodal import (
+    text_block,
+    image_url_block,
+    ClipboardEmptyError,
+)
+
+
+# Type alias for message content
+MessageContent = Union[str, List[Dict[str, Any]]]
+
+
+class MultimodalInputHandler:
+    """Handler for processing multimodal CLI input.
+    
+    Converts user input with @paste, @image:, @url: markers into
+    proper multimodal content compatible with LLM APIs.
+    
+    Usage:
+        handler = MultimodalInputHandler()
+        content = handler.process("@paste 请描述这张图片")
+        # Returns List[Dict] with image and text blocks
+    """
+    
+    def __init__(
+        self,
+        image_config: Optional[ImageProcessConfig] = None,
+        verbose: bool = False
+    ):
+        """Initialize the handler.
+        
+        Args:
+            image_config: Configuration for image processing
+            verbose: If True, print status messages
+        """
+        self.parser = MultimodalInputParser()
+        self.clipboard = ClipboardImageReader()
+        self.processor = ImageProcessor(image_config)
+        self.verbose = verbose
+    
+    def process(self, raw_input: str) -> MessageContent:
+        """Process user input and convert to message content.
+        
+        Args:
+            raw_input: Raw user input string
+            
+        Returns:
+            str if no multimodal markers, List[Dict] if multimodal
+            
+        Raises:
+            ClipboardEmptyError: If @paste used but clipboard empty
+            FileNotFoundError: If @image: file doesn't exist
+            Other exceptions from image processing
+        """
+        # Quick check - if no markers, return as plain text
+        if not self.parser.has_multimodal_markers(raw_input):
+            return raw_input
+        
+        # Parse the input
+        parsed = self.parser.parse(raw_input)
+        
+        if not parsed.has_images():
+            return raw_input
+        
+        # Build multimodal content
+        content: List[Dict[str, Any]] = []
+        
+        for i, text_part in enumerate(parsed.text_parts):
+            # Add text block if not empty
+            if text_part.strip():
+                content.append(text_block(text_part.strip()))
+            
+            # Add corresponding image if exists
+            if i < len(parsed.image_refs):
+                ref = parsed.image_refs[i]
+                image_url = self._resolve_image_ref(ref)
+                content.append(image_url_block(image_url, detail="auto"))
+        
+        # Ensure we have at least one block
+        if not content:
+            return raw_input
+        
+        return content
+    
+    def _resolve_image_ref(self, ref) -> str:
+        """Resolve an image reference to a usable URL.
+        
+        Args:
+            ref: ImageReference object
+            
+        Returns:
+            Image URL (data: URL for local/clipboard, https: for web)
+        """
+        if ref.source_type == ImageSourceType.CLIPBOARD:
+            return self._read_clipboard()
+        
+        elif ref.source_type == ImageSourceType.FILE:
+            return self._read_file(ref.source)
+        
+        elif ref.source_type == ImageSourceType.URL:
+            # URL is passed through directly
+            return ref.source
+        
+        raise ValueError(f"Unknown source type: {ref.source_type}")
+    
+    def _read_clipboard(self) -> str:
+        """Read and process clipboard image.
+        
+        Returns:
+            Base64 data URL
+        """
+        data = self.clipboard.read()
+        if data is None:
+            raise ClipboardEmptyError(
+                "No image found in clipboard. "
+                "Please copy an image first (Cmd/Ctrl+C on an image)."
+            )
+        
+        if self.verbose:
+            info = self.processor.get_image_info(data)
+            print(f"📎 Read clipboard image: {info.get('width', '?')}x{info.get('height', '?')}, "
+                  f"{info.get('size_bytes', 0) // 1024}KB")
+        
+        # Process the image (resize, compress if needed)
+        processed = self.processor.process(data)
+        
+        if self.verbose and len(processed) != len(data):
+            print(f"   Compressed to {len(processed) // 1024}KB")
+        
+        return self.clipboard.to_base64_url(processed)
+    
+    def _read_file(self, path: str) -> str:
+        """Read and process image file.
+        
+        Args:
+            path: File path (can be relative or use ~)
+            
+        Returns:
+            Base64 data URL
+        """
+        # Expand user home and resolve path
+        expanded_path = os.path.expanduser(path)
+        if not os.path.isabs(expanded_path):
+            expanded_path = os.path.abspath(expanded_path)
+        
+        if not os.path.exists(expanded_path):
+            raise FileNotFoundError(f"Image file not found: {path}")
+        
+        if self.verbose:
+            print(f"📁 Reading image file: {path}")
+        
+        with open(expanded_path, "rb") as f:
+            data = f.read()
+        
+        # Detect MIME type
+        mime_type = self._detect_mime_type(data)
+        
+        # Process the image
+        processed = self.processor.process(data)
+        
+        if self.verbose:
+            info = self.processor.get_image_info(processed)
+            print(f"   Image: {info.get('width', '?')}x{info.get('height', '?')}, "
+                  f"{len(processed) // 1024}KB")
+        
+        return self._to_base64_url(processed, mime_type)
+    
+    def _detect_mime_type(self, data: bytes) -> str:
+        """Detect MIME type from image data.
+        
+        Args:
+            data: Image bytes
+            
+        Returns:
+            MIME type string
+        """
+        # Check magic bytes
+        if data[:8] == b'\x89PNG\r\n\x1a\n':
+            return "image/png"
+        elif data[:2] == b'\xff\xd8':
+            return "image/jpeg"
+        elif data[:6] in (b'GIF87a', b'GIF89a'):
+            return "image/gif"
+        elif data[:4] == b'RIFF' and data[8:12] == b'WEBP':
+            return "image/webp"
+        else:
+            # Default to PNG (processor will convert)
+            return "image/png"
+    
+    def _to_base64_url(self, data: bytes, mime_type: str = "image/png") -> str:
+        """Convert image data to base64 data URL.
+        
+        Args:
+            data: Image bytes
+            mime_type: MIME type
+            
+        Returns:
+            Data URL string
+        """
+        b64 = base64.b64encode(data).decode('utf-8')
+        return f"data:{mime_type};base64,{b64}"
+    
+    def check_clipboard_status(self) -> dict:
+        """Check if clipboard contains an image.
+        
+        Returns:
+            Status dict with has_image and optional info
+        """
+        data = self.clipboard.read()
+        if data is None:
+            return {"has_image": False}
+        
+        info = self.processor.get_image_info(data)
+        return {
+            "has_image": True,
+            "info": info
+        }
+
+
+# Convenience function for quick processing
+def process_multimodal_input(raw_input: str, verbose: bool = False) -> MessageContent:
+    """Process user input and convert multimodal markers to content blocks.
+    
+    Args:
+        raw_input: Raw user input string
+        verbose: If True, print status messages
+        
+    Returns:
+        str if no multimodal markers, List[Dict] if multimodal
+    """
+    handler = MultimodalInputHandler(verbose=verbose)
+    return handler.process(raw_input)

dolphin/cli/multimodal/image_processor.py

@@ -0,0 +1,214 @@
+"""
+Image processor for multimodal CLI input.
+
+Handles image validation, format conversion, and compression.
+"""
+
+import io
+from dataclasses import dataclass, field
+from typing import Tuple, Optional
+
+from dolphin.core.common.multimodal import (
+    UnsupportedImageFormatError,
+    ImagePayloadTooLargeError,
+)
+
+
+@dataclass
+class ImageProcessConfig:
+    """Configuration for image processing."""
+    max_size_bytes: int = 4 * 1024 * 1024      # 4MB
+    max_dimension: int = 2048                   # Maximum edge length
+    quality: int = 85                           # JPEG compression quality
+    # Added MPO (Multi-Picture Object) - used by some cameras/phones, JPEG-based
+    allowed_formats: Tuple[str, ...] = ("PNG", "JPEG", "GIF", "WEBP", "MPO")
+    auto_compress: bool = True                  # Auto-compress oversized images
+
+
+class ImageProcessor:
+    """Image preprocessor for CLI multimodal input.
+    
+    Handles:
+    - Format validation
+    - Size checking and compression
+    - Format conversion
+    
+    Usage:
+        processor = ImageProcessor()
+        processed_data = processor.process(raw_image_data)
+    """
+    
+    def __init__(self, config: Optional[ImageProcessConfig] = None):
+        """Initialize the processor with configuration.
+        
+        Args:
+            config: Processing configuration (uses defaults if not provided)
+        """
+        self.config = config or ImageProcessConfig()
+    
+    def process(self, image_data: bytes) -> bytes:
+        """Process image data: validate, resize if needed, optimize.
+        
+        Args:
+            image_data: Raw image bytes
+            
+        Returns:
+            Processed image bytes
+            
+        Raises:
+            UnsupportedImageFormatError: If format not allowed
+            ImagePayloadTooLargeError: If size exceeds limit (after compression attempt)
+        """
+        try:
+            from PIL import Image
+        except ImportError:
+            # Pillow not installed, return data as-is but check size
+            if len(image_data) > self.config.max_size_bytes:
+                raise ImagePayloadTooLargeError(
+                    f"Image size {len(image_data)} exceeds limit {self.config.max_size_bytes}. "
+                    f"Install Pillow for automatic compression."
+                )
+            return image_data
+        
+        # Open and validate image
+        img = Image.open(io.BytesIO(image_data))
+        
+        # Check format - treat MPO as JPEG (MPO is JPEG-based multi-picture format)
+        img_format = img.format.upper() if img.format else None
+        if img_format and img_format not in self.config.allowed_formats:
+            raise UnsupportedImageFormatError(
+                f"Image format '{img.format}' not supported. "
+                f"Allowed formats: {', '.join(self.config.allowed_formats)}"
+            )
+        
+        # Check and resize if needed
+        if max(img.size) > self.config.max_dimension:
+            if self.config.auto_compress:
+                img = self._resize(img)
+            else:
+                raise ImagePayloadTooLargeError(
+                    f"Image dimensions {img.size} exceed limit {self.config.max_dimension}. "
+                    f"Enable auto_compress to automatically resize."
+                )
+        
+        # Convert to output format
+        output = io.BytesIO()
+        
+        # Choose format based on image mode
+        if img.mode in ('RGBA', 'LA', 'P'):
+            # Preserve transparency with PNG
+            if img.mode == 'P' and 'transparency' in img.info:
+                img = img.convert('RGBA')
+            output_format = "PNG"
+            img.save(output, format=output_format, optimize=True)
+        else:
+            # Use JPEG for photos (smaller size)
+            if img.mode != 'RGB':
+                img = img.convert('RGB')
+            output_format = "JPEG"
+            img.save(output, format=output_format, quality=self.config.quality, optimize=True)
+        
+        result = output.getvalue()
+        
+        # Final size check
+        if len(result) > self.config.max_size_bytes:
+            if self.config.auto_compress:
+                # Try more aggressive compression
+                result = self._aggressive_compress(img, output_format)
+            
+            if len(result) > self.config.max_size_bytes:
+                raise ImagePayloadTooLargeError(
+                    f"Image size {len(result)} exceeds limit {self.config.max_size_bytes} "
+                    f"even after compression."
+                )
+        
+        return result
+    
+    def _resize(self, img) -> "Image.Image":
+        """Resize image to fit within max_dimension while preserving aspect ratio.
+        
+        Args:
+            img: PIL Image object
+            
+        Returns:
+            Resized PIL Image
+        """
+        from PIL import Image
+        
+        ratio = self.config.max_dimension / max(img.size)
+        new_size = (int(img.width * ratio), int(img.height * ratio))
+        return img.resize(new_size, Image.Resampling.LANCZOS)
+    
+    def _aggressive_compress(self, img, output_format: str) -> bytes:
+        """Apply aggressive compression to reduce file size.
+        
+        Args:
+            img: PIL Image object
+            output_format: Target format
+            
+        Returns:
+            Compressed image bytes
+        """
+        output = io.BytesIO()
+        
+        # Reduce dimensions further
+        max_dim = min(self.config.max_dimension, 1024)
+        if max(img.size) > max_dim:
+            from PIL import Image
+            ratio = max_dim / max(img.size)
+            new_size = (int(img.width * ratio), int(img.height * ratio))
+            img = img.resize(new_size, Image.Resampling.LANCZOS)
+        
+        # Use lower quality
+        if output_format == "JPEG":
+            if img.mode != 'RGB':
+                img = img.convert('RGB')
+            img.save(output, format="JPEG", quality=60, optimize=True)
+        else:
+            img.save(output, format="PNG", optimize=True)
+        
+        return output.getvalue()
+    
+    def get_image_info(self, image_data: bytes) -> dict:
+        """Get information about an image.
+        
+        Args:
+            image_data: Raw image bytes
+            
+        Returns:
+            Dict with format, size, mode, dimensions
+        """
+        try:
+            from PIL import Image
+            
+            img = Image.open(io.BytesIO(image_data))
+            return {
+                "format": img.format,
+                "mode": img.mode,
+                "width": img.width,
+                "height": img.height,
+                "size_bytes": len(image_data),
+            }
+        except Exception as e:
+            return {
+                "error": str(e),
+                "size_bytes": len(image_data),
+            }
+    
+    def validate_file(self, file_path: str) -> bool:
+        """Validate that a file is a valid image.
+        
+        Args:
+            file_path: Path to image file
+            
+        Returns:
+            True if valid image
+        """
+        try:
+            from PIL import Image
+            
+            with Image.open(file_path) as img:
+                img.verify()
+            return True
+        except Exception:
+            return False

dolphin/cli/multimodal/input_parser.py

@@ -0,0 +1,149 @@
+"""
+Input parser for multimodal CLI input.
+
+Parses user input to extract multimodal references like @paste, @image:<path>, @url:<url>.
+"""
+
+import re
+from dataclasses import dataclass
+from enum import Enum
+from typing import List, Tuple
+
+
+class ImageSourceType(Enum):
+    """Type of image source."""
+    CLIPBOARD = "clipboard"      # @paste
+    FILE = "file"                # @image:<path>
+    URL = "url"                  # @url:<url>
+
+
+@dataclass
+class ImageReference:
+    """Reference to an image in user input."""
+    source_type: ImageSourceType
+    source: str                   # Path, URL, or "clipboard"
+    position: int                 # Position in original text
+    original_text: str            # Original matched text (e.g., "@paste", "@image:./foo.png")
+
+
+@dataclass
+class ParsedMultimodalInput:
+    """Result of parsing multimodal input."""
+    text_parts: List[str]         # Text fragments after removing image references
+    image_refs: List[ImageReference]  # List of image references
+    original_input: str           # Original unmodified input
+    
+    def has_images(self) -> bool:
+        """Check if input contains any image references."""
+        return len(self.image_refs) > 0
+    
+    def get_combined_text(self) -> str:
+        """Get all text parts combined (without image markers)."""
+        return " ".join(part.strip() for part in self.text_parts if part.strip())
+
+
+class MultimodalInputParser:
+    """Parser for multimodal CLI input.
+    
+    Supports:
+    - @paste: Read image from clipboard
+    - @image:<path>: Read image from local file
+    - @url:<url>: Reference image by URL (https only)
+    
+    Example:
+        parser = MultimodalInputParser()
+        result = parser.parse("@paste 请描述这张图片")
+        # result.has_images() == True
+        # result.image_refs[0].source_type == ImageSourceType.CLIPBOARD
+    """
+    
+    # Pattern definitions
+    PASTE_PATTERN = r"@paste"
+    IMAGE_PATTERN = r"@image:([^\s]+)"
+    URL_PATTERN = r"@url:(https://[^\s]+)"
+    
+    def __init__(self):
+        """Initialize the parser with compiled regex patterns."""
+        self._patterns = [
+            (re.compile(self.PASTE_PATTERN, re.IGNORECASE), ImageSourceType.CLIPBOARD),
+            (re.compile(self.IMAGE_PATTERN, re.IGNORECASE), ImageSourceType.FILE),
+            (re.compile(self.URL_PATTERN, re.IGNORECASE), ImageSourceType.URL),
+        ]
+    
+    def parse(self, raw_input: str) -> ParsedMultimodalInput:
+        """Parse raw input to extract multimodal references.
+        
+        Args:
+            raw_input: User's raw input string
+            
+        Returns:
+            ParsedMultimodalInput containing text parts and image references
+        """
+        if not raw_input:
+            return ParsedMultimodalInput(
+                text_parts=[""],
+                image_refs=[],
+                original_input=raw_input
+            )
+        
+        # Find all matches with their positions
+        matches: List[Tuple[int, int, ImageReference]] = []
+        
+        for pattern, source_type in self._patterns:
+            for match in pattern.finditer(raw_input):
+                start, end = match.span()
+                
+                if source_type == ImageSourceType.CLIPBOARD:
+                    source = "clipboard"
+                elif source_type == ImageSourceType.FILE:
+                    source = match.group(1)  # The path
+                elif source_type == ImageSourceType.URL:
+                    source = match.group(1)  # The URL
+                else:
+                    continue
+                
+                ref = ImageReference(
+                    source_type=source_type,
+                    source=source,
+                    position=start,
+                    original_text=match.group(0)
+                )
+                matches.append((start, end, ref))
+        
+        # Sort by position
+        matches.sort(key=lambda x: x[0])
+        
+        # Extract text parts and image references
+        text_parts: List[str] = []
+        image_refs: List[ImageReference] = []
+        last_end = 0
+        
+        for start, end, ref in matches:
+            # Add text before this match
+            text_before = raw_input[last_end:start]
+            text_parts.append(text_before)
+            image_refs.append(ref)
+            last_end = end
+        
+        # Add remaining text after last match
+        text_parts.append(raw_input[last_end:])
+        
+        return ParsedMultimodalInput(
+            text_parts=text_parts,
+            image_refs=image_refs,
+            original_input=raw_input
+        )
+    
+    def has_multimodal_markers(self, text: str) -> bool:
+        """Quick check if text contains any multimodal markers.
+        
+        Args:
+            text: Text to check
+            
+        Returns:
+            True if text contains @paste, @image:, or @url:
+        """
+        for pattern, _ in self._patterns:
+            if pattern.search(text):
+                return True
+        return False

dolphin/cli/runner/__init__.py

@@ -0,0 +1,8 @@
+# -*- coding: utf-8 -*-
+"""Runner 模块 - CLI 运行器"""
+
+from dolphin.cli.runner.runner import runDolphinAgent
+
+__all__ = [
+    "runDolphinAgent",
+]