abstractcore 2.4.2__py3-none-any.whl → 2.4.4__py3-none-any.whl

abstractcore/media/vision_fallback.py

@@ -0,0 +1,260 @@
+"""
+Vision Fallback System for Text-Only Models
+
+Implements two-stage pipeline: vision model → description → text-only model
+Uses unified AbstractCore configuration system.
+"""
+
+import logging
+from pathlib import Path
+from typing import Optional, Dict, Any
+
+logger = logging.getLogger(__name__)
+
+
+class VisionNotConfiguredError(Exception):
+    """Raised when vision fallback is requested but not configured."""
+    pass
+
+
+class VisionFallbackHandler:
+    """
+    Handles vision fallback for text-only models using two-stage pipeline.
+
+    When a text-only model receives an image:
+    1. Uses configured vision model to generate description
+    2. Provides description to text-only model for processing
+
+    Uses the unified AbstractCore configuration system.
+    """
+
+    def __init__(self, config_manager=None):
+        """Initialize with configuration manager."""
+        if config_manager is None:
+            from abstractcore.config import get_config_manager
+            self.config_manager = get_config_manager()
+        else:
+            self.config_manager = config_manager
+
+    @property
+    def vision_config(self):
+        """Get vision configuration from unified config system."""
+        return self.config_manager.config.vision
+
+    def create_description(self, image_path: str, user_prompt: str = None) -> str:
+        """
+        Generate description using configured vision model.
+
+        Args:
+            image_path: Path to the image file
+            user_prompt: Original user prompt for context
+
+        Returns:
+            Description string to be used by text-only model
+
+        Raises:
+            VisionNotConfiguredError: When vision fallback is not configured
+        """
+        if self.vision_config.strategy == "disabled":
+            raise VisionNotConfiguredError("Vision fallback is disabled")
+
+        if not self._has_vision_capability():
+            raise VisionNotConfiguredError("No vision capability configured")
+
+        try:
+            return self._generate_with_fallback(image_path)
+        except Exception as e:
+            logger.debug(f"Vision fallback failed: {e}")
+            raise VisionNotConfiguredError(f"Vision fallback generation failed: {e}")
+
+    def _has_vision_capability(self) -> bool:
+        """Check if any vision capability is configured."""
+        return (
+            (self.vision_config.caption_provider is not None and
+             self.vision_config.caption_model is not None) or
+            len(self.vision_config.fallback_chain) > 0 or
+            self._has_local_models()
+        )
+
+    def _has_local_models(self) -> bool:
+        """Check if any local vision models are available."""
+        models_dir = Path(self.vision_config.local_models_path).expanduser()
+        return models_dir.exists() and any(models_dir.iterdir())
+
+    def _generate_with_fallback(self, image_path: str) -> str:
+        """Try vision models in fallback chain order."""
+        # Try primary provider first
+        if self.vision_config.caption_provider and self.vision_config.caption_model:
+            try:
+                description = self._generate_description(
+                    self.vision_config.caption_provider,
+                    self.vision_config.caption_model,
+                    image_path
+                )
+                return description
+            except Exception as e:
+                logger.debug(f"Primary vision provider failed: {e}")
+
+        # Try fallback chain
+        for provider_config in self.vision_config.fallback_chain:
+            try:
+                description = self._generate_description(
+                    provider_config["provider"],
+                    provider_config["model"],
+                    image_path
+                )
+                return description
+            except Exception as e:
+                logger.debug(f"Vision provider {provider_config} failed: {e}")
+                continue
+
+        # Try local models
+        if self._has_local_models():
+            try:
+                description = self._generate_local_description(image_path)
+                return description
+            except Exception as e:
+                logger.debug(f"Local vision model failed: {e}")
+
+        raise Exception("All vision fallback providers failed")
+
+    def _generate_description(self, provider: str, model: str, image_path: str) -> str:
+        """Generate description using specified provider and model."""
+        try:
+            # Import here to avoid circular imports
+            from abstractcore import create_llm
+
+            vision_llm = create_llm(provider, model=model)
+            response = vision_llm.generate(
+                "Provide a detailed description of this image in 3-4 sentences. Be precise about specific landmarks, buildings, objects, and details. If you recognize specific places or things, name them accurately. Describe naturally without phrases like 'this image shows'.",
+                media=[image_path]
+            )
+            return response.content.strip()
+        except Exception as e:
+            logger.debug(f"Failed to generate description with {provider}/{model}: {e}")
+            raise
+
+    def _generate_local_description(self, image_path: str) -> str:
+        """Generate description using local vision model."""
+        try:
+            models_dir = Path(self.vision_config.local_models_path).expanduser()
+
+            # Look for downloaded vision models
+            for model_dir in models_dir.iterdir():
+                if model_dir.is_dir() and ("caption" in model_dir.name.lower() or "blip" in model_dir.name.lower() or "vit" in model_dir.name.lower() or "git" in model_dir.name.lower()):
+                    try:
+                        # Check if download is complete
+                        if not (model_dir / "download_complete.txt").exists():
+                            logger.debug(f"Model {model_dir.name} download incomplete")
+                            continue
+
+                        description = self._use_local_model(model_dir, image_path)
+                        if description:
+                            return description
+
+                    except Exception as e:
+                        logger.debug(f"Local model {model_dir} failed: {e}")
+                        continue
+
+            raise Exception("No working local models found")
+        except ImportError:
+            raise Exception("transformers library not available for local models")
+
+    def _use_local_model(self, model_dir: Path, image_path: str) -> str:
+        """Use a specific local model to generate description."""
+        from PIL import Image
+
+        model_name = model_dir.name
+
+        if "blip" in model_name:
+            from transformers import BlipProcessor, BlipForConditionalGeneration
+
+            # Load BLIP model and processor
+            processor = BlipProcessor.from_pretrained(model_dir / "processor", use_fast=False)
+            model = BlipForConditionalGeneration.from_pretrained(model_dir / "model")
+
+            # Process image
+            image = Image.open(image_path).convert('RGB')
+            inputs = processor(image, return_tensors="pt")
+
+            # Generate description
+            out = model.generate(**inputs, max_length=50, num_beams=5)
+            description = processor.decode(out[0], skip_special_tokens=True)
+            return description
+
+        elif "vit-gpt2" in model_name:
+            from transformers import VisionEncoderDecoderModel, ViTImageProcessor, AutoTokenizer
+
+            # Load ViT-GPT2 components
+            model = VisionEncoderDecoderModel.from_pretrained(model_dir / "model")
+            feature_extractor = ViTImageProcessor.from_pretrained(model_dir / "feature_extractor")
+            tokenizer = AutoTokenizer.from_pretrained(model_dir / "tokenizer")
+
+            # Process image
+            image = Image.open(image_path).convert('RGB')
+            pixel_values = feature_extractor(images=image, return_tensors="pt").pixel_values
+
+            # Generate description
+            output_ids = model.generate(pixel_values, max_length=50, num_beams=4)
+            description = tokenizer.decode(output_ids[0], skip_special_tokens=True)
+            return description
+
+        elif "git" in model_name:
+            from transformers import GitProcessor, GitForCausalLM
+
+            # Load GIT model and processor
+            processor = GitProcessor.from_pretrained(model_dir / "processor")
+            model = GitForCausalLM.from_pretrained(model_dir / "model")
+
+            # Process image
+            image = Image.open(image_path).convert('RGB')
+            inputs = processor(images=image, return_tensors="pt")
+
+            # Generate description
+            generated_ids = model.generate(pixel_values=inputs.pixel_values, max_length=50)
+            description = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
+            return description
+
+        else:
+            # Try generic image-to-text pipeline
+            from transformers import pipeline
+            captioner = pipeline("image-to-text", model=str(model_dir))
+            result = captioner(image_path)
+            if result and len(result) > 0:
+                return result[0]["generated_text"]
+
+        return None
+
+    def _show_setup_instructions(self) -> str:
+        """Return helpful setup instructions for users."""
+        return """⚠️  Vision capability not configured for text-only models.
+
+To enable image analysis with text-only models:
+1. Download local model: abstractcore --download-vision-model
+2. Use existing model: abstractcore --set-vision-caption qwen2.5vl:7b
+3. Use cloud API: abstractcore --set-vision-provider openai --model gpt-4o
+4. Interactive setup: abstractcore --configure
+
+Current status: abstractcore --status"""
+
+    def get_status(self) -> Dict[str, Any]:
+        """Get current vision configuration status using unified config."""
+        return self.config_manager.get_status()["vision"]
+
+    def is_enabled(self) -> bool:
+        """Check if vision fallback is enabled and configured."""
+        return (self.vision_config.strategy == "two_stage" and
+                self._has_vision_capability())
+
+
+# Convenience functions for easy integration
+def has_vision_capability() -> bool:
+    """Check if vision fallback is configured and enabled."""
+    handler = VisionFallbackHandler()
+    return handler.is_enabled()
+
+
+def create_image_description(image_path: str, user_prompt: str = None) -> str:
+    """Create image description for text-only models."""
+    handler = VisionFallbackHandler()
+    return handler.create_description(image_path, user_prompt)

abstractcore/providers/anthropic_provider.py

@@ -61,6 +61,7 @@ class AnthropicProvider(BaseProvider):
                           messages: Optional[List[Dict[str, str]]] = None,
                           system_prompt: Optional[str] = None,
                           tools: Optional[List[Dict[str, Any]]] = None,
+                          media: Optional[List['MediaContent']] = None,
                           stream: bool = False,
                           response_model: Optional[Type[BaseModel]] = None,
                           **kwargs) -> Union[GenerateResponse, Iterator[GenerateResponse]]:
@@ -89,7 +90,23 @@ class AnthropicProvider(BaseProvider):
 
         # Add current prompt as user message
         if prompt and prompt not in [msg.get("content") for msg in (messages or [])]:
-            api_messages.append({"role": "user", "content": prompt})
+            # Handle multimodal message with media content
+            if media:
+                try:
+                    from ..media.handlers import AnthropicMediaHandler
+                    media_handler = AnthropicMediaHandler(self.model_capabilities)
+
+                    # Create multimodal message combining text and media
+                    multimodal_message = media_handler.create_multimodal_message(prompt, media)
+                    api_messages.append(multimodal_message)
+                except ImportError:
+                    self.logger.warning("Media processing not available. Install with: pip install abstractcore[media]")
+                    api_messages.append({"role": "user", "content": prompt})
+                except Exception as e:
+                    self.logger.warning(f"Failed to process media content: {e}")
+                    api_messages.append({"role": "user", "content": prompt})
+            else:
+                api_messages.append({"role": "user", "content": prompt})
 
         # Prepare API call parameters using unified system
         generation_kwargs = self._prepare_generation_kwargs(**kwargs)

abstractcore/providers/base.py

@@ -204,6 +204,7 @@ class BaseProvider(AbstractCoreInterface, ABC):
                                messages: Optional[List[Dict[str, str]]] = None,
                                system_prompt: Optional[str] = None,
                                tools: Optional[List] = None,  # Accept both ToolDefinition and Dict
+                               media: Optional[List[Union[str, Dict[str, Any], 'MediaContent']]] = None,  # Media files
                                stream: bool = False,
                                response_model: Optional[Type[BaseModel]] = None,
                                retry_strategy=None,  # Custom retry strategy for structured output
@@ -215,6 +216,12 @@ class BaseProvider(AbstractCoreInterface, ABC):
         Providers should override _generate_internal instead of generate.
 
         Args:
+            prompt: The input prompt
+            messages: Optional conversation history
+            system_prompt: Optional system prompt
+            tools: Optional list of available tools
+            media: Optional list of media files (file paths, MediaContent objects, or dicts)
+            stream: Whether to stream the response
             response_model: Optional Pydantic model for structured output
             retry_strategy: Optional retry strategy for structured output validation
             tool_call_tags: Optional tool call tag format for rewriting
@@ -235,6 +242,7 @@ class BaseProvider(AbstractCoreInterface, ABC):
                     messages=messages,
                     system_prompt=system_prompt,
                     tools=tools,
+                    media=media,
                     response_model=response_model,
                     retry_strategy=retry_strategy,
                     tool_call_tags=tool_call_tags,
@@ -253,10 +261,16 @@ class BaseProvider(AbstractCoreInterface, ABC):
                 messages=messages,
                 system_prompt=system_prompt,
                 tools=None,  # No tools in this path
+                media=media,
                 stream=stream,
                 **kwargs
             )
 
+        # Process media content if provided
+        processed_media = None
+        if media:
+            processed_media = self._process_media_content(media)
+
         # Convert tools to ToolDefinition objects first (outside retry loop)
         converted_tools = None
         if tools:
@@ -308,6 +322,7 @@ class BaseProvider(AbstractCoreInterface, ABC):
                     messages=messages,
                     system_prompt=system_prompt,
                     tools=converted_tools,
+                    media=processed_media,
                     stream=stream,
                     execute_tools=should_execute_tools,
                     tool_call_tags=tool_call_tags,
@@ -391,6 +406,7 @@ class BaseProvider(AbstractCoreInterface, ABC):
                           messages: Optional[List[Dict[str, str]]] = None,
                           system_prompt: Optional[str] = None,
                           tools: Optional[List[Dict[str, Any]]] = None,
+                          media: Optional[List['MediaContent']] = None,
                           stream: bool = False,
                           response_model: Optional[Type[BaseModel]] = None,
                           execute_tools: Optional[bool] = None,
@@ -400,8 +416,15 @@ class BaseProvider(AbstractCoreInterface, ABC):
         This is called by generate_with_telemetry.
 
         Args:
+            prompt: The input prompt
+            messages: Optional conversation history
+            system_prompt: Optional system prompt
+            tools: Optional list of available tools
+            media: Optional list of processed MediaContent objects
+            stream: Whether to stream the response
             response_model: Optional Pydantic model for structured output
             execute_tools: Whether to execute tools automatically (True) or let agent handle execution (False)
+            **kwargs: Additional provider-specific parameters
         """
         raise NotImplementedError("Subclasses must implement _generate_internal")
 
@@ -757,6 +780,73 @@ class BaseProvider(AbstractCoreInterface, ABC):
         """Rough estimation of token count for given text"""
         return super().estimate_tokens(text)
 
+    def _process_media_content(self, media: List[Union[str, Dict[str, Any], 'MediaContent']]) -> List['MediaContent']:
+        """
+        Process media content from various input formats into standardized MediaContent objects.
+
+        Args:
+            media: List of media inputs (file paths, MediaContent objects, or dicts)
+
+        Returns:
+            List of processed MediaContent objects
+
+        Raises:
+            ImportError: If media processing dependencies are not available
+            ValueError: If media input format is invalid
+        """
+        if not media:
+            return []
+
+        try:
+            # Import media handler components
+            from ..media import AutoMediaHandler
+            from ..media.types import MediaContent
+        except ImportError as e:
+            raise ImportError(
+                f"Media processing requires additional dependencies. "
+                f"Install with: pip install abstractcore[media]. Error: {e}"
+            )
+
+        processed_media = []
+
+        for i, media_item in enumerate(media):
+            try:
+                if isinstance(media_item, str):
+                    # File path - process with auto media handler
+                    handler = AutoMediaHandler()
+                    result = handler.process_file(media_item)
+                    if result.success:
+                        processed_media.append(result.media_content)
+                    else:
+                        self.logger.warning(f"Failed to process media file {media_item}: {result.error_message}")
+                        continue
+
+                elif hasattr(media_item, 'media_type'):
+                    # Already a MediaContent object
+                    processed_media.append(media_item)
+
+                elif isinstance(media_item, dict):
+                    # Dictionary format - convert to MediaContent
+                    try:
+                        media_content = MediaContent.from_dict(media_item)
+                        processed_media.append(media_content)
+                    except Exception as e:
+                        self.logger.warning(f"Failed to convert media dict at index {i}: {e}")
+                        continue
+
+                else:
+                    self.logger.warning(f"Unsupported media type at index {i}: {type(media_item)}")
+                    continue
+
+            except Exception as e:
+                self.logger.warning(f"Failed to process media item at index {i}: {e}")
+                continue
+
+        if not processed_media and media:
+            self.logger.warning("No media items were successfully processed")
+
+        return processed_media
+
     @abstractmethod
     def list_available_models(self, **kwargs) -> List[str]:
         """
@@ -777,6 +867,103 @@ class BaseProvider(AbstractCoreInterface, ABC):
         """
         pass
 
+    def health(self, timeout: Optional[float] = 5.0) -> Dict[str, Any]:
+        """
+        Check provider health and connectivity.
+
+        This method tests if the provider is online and accessible by attempting
+        to list available models. A successful model listing indicates the provider
+        is healthy and ready to serve requests.
+
+        Args:
+            timeout: Maximum time in seconds to wait for health check (default: 5.0).
+                    None means unlimited timeout (not recommended for health checks).
+
+        Returns:
+            Dict with health status information:
+            {
+                "status": bool,              # True if provider is healthy/online
+                "provider": str,             # Provider class name (e.g., "OpenAIProvider")
+                "models": List[str] | None,  # Available models if online, None if offline
+                "model_count": int,          # Number of models available (0 if offline)
+                "error": str | None,         # Error message if offline, None if healthy
+                "latency_ms": float          # Time taken for health check in milliseconds
+            }
+
+        Example:
+            >>> provider = OllamaProvider(model="llama2")
+            >>> health = provider.health(timeout=3.0)
+            >>> if health["status"]:
+            >>>     print(f"Healthy! {health['model_count']} models available")
+            >>> else:
+            >>>     print(f"Offline: {health['error']}")
+
+        Note:
+            - This method never raises exceptions; errors are captured in the response
+            - Uses list_available_models() as the connectivity test
+            - Providers can override this method for custom health check logic
+        """
+        import time as time_module
+
+        start_time = time_module.time()
+        provider_name = self.__class__.__name__
+
+        try:
+            # Attempt to list models as connectivity test
+            # Store original timeout if provider has HTTP client
+            original_timeout = None
+            timeout_changed = False
+
+            if timeout is not None and hasattr(self, '_timeout'):
+                original_timeout = self._timeout
+                if original_timeout != timeout:
+                    self.set_timeout(timeout)
+                    timeout_changed = True
+
+            try:
+                models = self.list_available_models()
+
+                # Restore original timeout if changed
+                if timeout_changed and original_timeout is not None:
+                    self.set_timeout(original_timeout)
+
+                latency_ms = (time_module.time() - start_time) * 1000
+
+                return {
+                    "status": True,
+                    "provider": provider_name,
+                    "models": models,
+                    "model_count": len(models) if models else 0,
+                    "error": None,
+                    "latency_ms": round(latency_ms, 2)
+                }
+
+            except Exception as e:
+                # Restore original timeout on error
+                if timeout_changed and original_timeout is not None:
+                    try:
+                        self.set_timeout(original_timeout)
+                    except:
+                        pass  # Best effort restoration
+                raise  # Re-raise to be caught by outer handler
+
+        except Exception as e:
+            latency_ms = (time_module.time() - start_time) * 1000
+
+            # Extract meaningful error message
+            error_message = str(e)
+            if not error_message:
+                error_message = f"{type(e).__name__} occurred during health check"
+
+            return {
+                "status": False,
+                "provider": provider_name,
+                "models": None,
+                "model_count": 0,
+                "error": error_message,
+                "latency_ms": round(latency_ms, 2)
+            }
+
     def _needs_tag_rewriting(self, tool_call_tags) -> bool:
         """Check if tag rewriting is needed (tags are non-standard)"""
         try: