abstractcore 2.4.2__py3-none-any.whl → 2.4.4__py3-none-any.whl

abstractcore/providers/huggingface_provider.py

@@ -473,21 +473,23 @@ class HuggingFaceProvider(BaseProvider):
                           messages: Optional[List[Dict[str, str]]] = None,
                           system_prompt: Optional[str] = None,
                           tools: Optional[List[Dict[str, Any]]] = None,
+                          media: Optional[List['MediaContent']] = None,
                           stream: bool = False,
                           response_model: Optional[Type[BaseModel]] = None,
                           **kwargs) -> Union[GenerateResponse, Iterator[GenerateResponse]]:
         """Generate response using appropriate backend"""
 
         if self.model_type == "gguf":
-            return self._generate_gguf(prompt, messages, system_prompt, tools, stream, **kwargs)
+            return self._generate_gguf(prompt, messages, system_prompt, tools, media, stream, **kwargs)
         else:
-            return self._generate_transformers(prompt, messages, system_prompt, tools, stream, **kwargs)
+            return self._generate_transformers(prompt, messages, system_prompt, tools, media, stream, **kwargs)
 
     def _generate_transformers(self,
                                prompt: str,
                                messages: Optional[List[Dict[str, str]]] = None,
                                system_prompt: Optional[str] = None,
                                tools: Optional[List[Dict[str, Any]]] = None,
+                               media: Optional[List['MediaContent']] = None,
                                stream: bool = False,
                                **kwargs) -> Union[GenerateResponse, Iterator[GenerateResponse]]:
         """Generate using transformers backend (original implementation)"""
@@ -499,7 +501,37 @@ class HuggingFaceProvider(BaseProvider):
                 finish_reason="error"
             )
 
-        # Build input text with tool support
+        # Build input text with tool and media support
+        # Handle media content first if present
+        if media:
+            try:
+                from ..media.handlers import LocalMediaHandler
+                media_handler = LocalMediaHandler("huggingface", self.model_capabilities, model_name=self.model)
+
+                # Create multimodal message combining text and media
+                multimodal_message = media_handler.create_multimodal_message(prompt, media)
+
+                # For local providers, we get text-embedded content
+                if isinstance(multimodal_message, str):
+                    prompt = multimodal_message
+                else:
+                    # If we get a structured message, extract the content
+                    if isinstance(multimodal_message, dict) and "content" in multimodal_message:
+                        if isinstance(multimodal_message["content"], list):
+                            # Find text content in the structured message
+                            text_content = ""
+                            for item in multimodal_message["content"]:
+                                if item.get("type") == "text":
+                                    text_content = item.get("text", "")
+                                    break
+                            prompt = text_content or prompt
+                        else:
+                            prompt = str(multimodal_message["content"])
+            except ImportError:
+                self.logger.warning("Media processing not available. Install with: pip install abstractcore[media]")
+            except Exception as e:
+                self.logger.warning(f"Failed to process media content: {e}")
+
         input_text = self._build_input_text_transformers(prompt, messages, system_prompt, tools)
 
         # Generation parameters using unified system
@@ -532,6 +564,7 @@ class HuggingFaceProvider(BaseProvider):
                        messages: Optional[List[Dict[str, str]]] = None,
                        system_prompt: Optional[str] = None,
                        tools: Optional[List[Dict[str, Any]]] = None,
+                       media: Optional[List['MediaContent']] = None,
                        stream: bool = False,
                        **kwargs) -> Union[GenerateResponse, Iterator[GenerateResponse]]:
         """Generate using GGUF backend with llama-cpp-python"""
@@ -552,7 +585,64 @@ class HuggingFaceProvider(BaseProvider):
         if messages:
             chat_messages.extend(messages)
 
-        chat_messages.append({"role": "user", "content": prompt})
+        # Handle media content for the user message - use proper vision format for GGUF models
+        if media:
+            try:
+                from ..architectures.detection import supports_vision
+
+                # Check if this model supports vision natively
+                if supports_vision(self.model):
+                    # Use HuggingFace multimodal format for vision-capable GGUF models
+                    user_message_content = []
+
+                    # Add text content
+                    user_message_content.append({"type": "text", "text": prompt})
+
+                    # Add media content
+                    for media_item in media:
+                        if hasattr(media_item, 'file_path') and media_item.file_path:
+                            # Use file:// URL format as specified in HuggingFace docs
+                            file_path = str(media_item.file_path)
+                            if not file_path.startswith('file://'):
+                                file_path = f"file://{file_path}"
+                            user_message_content.append({
+                                "type": "image",
+                                "image": file_path
+                            })
+                        elif hasattr(media_item, 'content') and media_item.content:
+                            # For base64 or other content, we might need to save to temp file
+                            import tempfile
+                            import base64
+                            with tempfile.NamedTemporaryFile(suffix='.jpg', delete=False) as tmp_file:
+                                if isinstance(media_item.content, str) and media_item.content.startswith('data:'):
+                                    # Handle base64 data URLs
+                                    header, data = media_item.content.split(',', 1)
+                                    decoded_data = base64.b64decode(data)
+                                    tmp_file.write(decoded_data)
+                                else:
+                                    tmp_file.write(media_item.content)
+                                tmp_file.flush()
+                                user_message_content.append({
+                                    "type": "image",
+                                    "image": f"file://{tmp_file.name}"
+                                })
+                else:
+                    # Fallback to text-based media handling for non-vision models
+                    from ..media.handlers import LocalMediaHandler
+                    media_handler = LocalMediaHandler("huggingface", self.model_capabilities, model_name=self.model)
+                    multimodal_message = media_handler.create_multimodal_message(prompt, media)
+                    user_message_content = multimodal_message if isinstance(multimodal_message, str) else prompt
+
+            except ImportError:
+                self.logger.warning("Media processing not available. Install with: pip install abstractcore[media]")
+                user_message_content = prompt
+            except Exception as e:
+                self.logger.warning(f"Failed to process media content: {e}")
+                user_message_content = prompt
+        else:
+            user_message_content = prompt
+
+        chat_messages.append({"role": "user", "content": user_message_content})
 
         # Prepare parameters using unified system
         unified_kwargs = self._prepare_generation_kwargs(**kwargs)
@@ -774,19 +864,14 @@ class HuggingFaceProvider(BaseProvider):
             if outputs and len(outputs) > 0:
                 response_text = outputs[0]['generated_text'].strip()
 
-                # Calculate token usage
-                input_tokens = len(self.tokenizer.encode(input_text))
-                output_tokens = len(self.tokenizer.encode(response_text))
+                # Calculate token usage using centralized utilities
+                usage = self._calculate_usage(input_text, response_text)
 
                 return GenerateResponse(
                     content=response_text,
                     model=self.model,
                     finish_reason="stop",
-                    usage={
-                        "prompt_tokens": input_tokens,
-                        "completion_tokens": output_tokens,
-                        "total_tokens": input_tokens + output_tokens
-                    }
+                    usage=usage
                 )
             else:
                 return GenerateResponse(
@@ -802,6 +887,20 @@ class HuggingFaceProvider(BaseProvider):
                 finish_reason="error"
             )
 
+    def _calculate_usage(self, prompt: str, response: str) -> Dict[str, int]:
+        """Calculate token usage using centralized token utilities."""
+        from ..utils.token_utils import TokenUtils
+
+        prompt_tokens = TokenUtils.estimate_tokens(prompt, self.model)
+        completion_tokens = TokenUtils.estimate_tokens(response, self.model)
+        total_tokens = prompt_tokens + completion_tokens
+
+        return {
+            "prompt_tokens": prompt_tokens,
+            "completion_tokens": completion_tokens,
+            "total_tokens": total_tokens
+        }
+
     def _stream_generate_transformers(self, input_text: str, max_new_tokens: int,
                                      temperature: float, top_p: float, tool_call_tags: Optional[str] = None) -> Iterator[GenerateResponse]:
         """Stream response using transformers (simulated, original implementation) with tool tag rewriting support"""

abstractcore/providers/lmstudio_provider.py

@@ -99,6 +99,7 @@ class LMStudioProvider(BaseProvider):
                           messages: Optional[List[Dict[str, str]]] = None,
                           system_prompt: Optional[str] = None,
                           tools: Optional[List[Dict[str, Any]]] = None,
+                          media: Optional[List['MediaContent']] = None,
                           stream: bool = False,
                           response_model: Optional[Type[BaseModel]] = None,
                           execute_tools: Optional[bool] = None,
@@ -129,11 +130,63 @@ class LMStudioProvider(BaseProvider):
         if messages:
             chat_messages.extend(messages)
 
-        # Add current prompt
-        chat_messages.append({
-            "role": "user",
-            "content": prompt
-        })
+        # Handle media content regardless of prompt (media can be used with messages too)
+        if media:
+            # Get the last user message content to combine with media
+            user_message_text = prompt.strip() if prompt else ""
+            if not user_message_text and chat_messages:
+                # If no prompt, try to get text from the last user message
+                for msg in reversed(chat_messages):
+                    if msg.get("role") == "user" and msg.get("content"):
+                        user_message_text = msg["content"]
+                        break
+            try:
+                # CRITICAL FIX: Process media files into MediaContent objects first
+                processed_media = self._process_media_content(media)
+
+                # Use capability-based media handler selection
+                media_handler = self._get_media_handler_for_model(self.model)
+
+                # Create multimodal message combining text and processed media
+                multimodal_message = media_handler.create_multimodal_message(user_message_text, processed_media)
+
+                # For LMStudio (OpenAI-compatible), we might get a string (embedded text) or dict (structured)
+                if isinstance(multimodal_message, str):
+                    # Replace the last user message with the multimodal message, or add new one
+                    if chat_messages and chat_messages[-1].get("role") == "user":
+                        chat_messages[-1]["content"] = multimodal_message
+                    else:
+                        chat_messages.append({
+                            "role": "user",
+                            "content": multimodal_message
+                        })
+                else:
+                    if chat_messages and chat_messages[-1].get("role") == "user":
+                        # Replace last user message with structured multimodal message
+                        chat_messages[-1] = multimodal_message
+                    else:
+                        chat_messages.append(multimodal_message)
+            except ImportError:
+                self.logger.warning("Media processing not available. Install with: pip install abstractcore[media]")
+                if user_message_text:
+                    chat_messages.append({
+                        "role": "user",
+                        "content": user_message_text
+                    })
+            except Exception as e:
+                self.logger.warning(f"Failed to process media content: {e}")
+                if user_message_text:
+                    chat_messages.append({
+                        "role": "user",
+                        "content": user_message_text
+                    })
+
+        # Add prompt as separate message if provided (for backward compatibility)
+        elif prompt and prompt.strip():
+            chat_messages.append({
+                "role": "user",
+                "content": prompt
+            })
 
         # Build request payload using unified system
         generation_kwargs = self._prepare_generation_kwargs(**kwargs)
@@ -313,7 +366,37 @@ class LMStudioProvider(BaseProvider):
                 except Exception:
                     pass  # Best effort - don't fail the operation
 
+    def _normalize_model_name(self, model_name: str) -> str:
+        """Remove common provider prefixes from model name."""
+        for prefix in ["lmstudio/", "qwen/", "ollama/", "huggingface/"]:
+            if model_name.startswith(prefix):
+                model_name = model_name[len(prefix):]
+        return model_name
+
+    def _get_media_handler_for_model(self, model_name: str):
+        """Get appropriate media handler based on model vision capabilities."""
+        from ..media.handlers import OpenAIMediaHandler, LocalMediaHandler
+
+        # Normalize model name by removing provider prefixes
+        clean_model_name = self._normalize_model_name(model_name)
+
+        # Determine if model supports vision
+        try:
+            from ..architectures.detection import supports_vision
+            use_vision_handler = supports_vision(clean_model_name)
+        except Exception as e:
+            self.logger.debug(f"Vision detection failed: {e}, defaulting to LocalMediaHandler")
+            use_vision_handler = False
+
+        # Create appropriate handler
+        if use_vision_handler:
+            handler = OpenAIMediaHandler(self.model_capabilities, model_name=model_name)
+            self.logger.debug(f"Using OpenAIMediaHandler for vision model: {clean_model_name}")
+        else:
+            handler = LocalMediaHandler("lmstudio", self.model_capabilities, model_name=model_name)
+            self.logger.debug(f"Using LocalMediaHandler for model: {clean_model_name}")
 
+        return handler
 
     def list_available_models(self, **kwargs) -> List[str]:
         """List available models from LMStudio server."""

abstractcore/providers/mlx_provider.py

@@ -139,6 +139,7 @@ class MLXProvider(BaseProvider):
                           messages: Optional[List[Dict[str, str]]] = None,
                           system_prompt: Optional[str] = None,
                           tools: Optional[List[Dict[str, Any]]] = None,
+                          media: Optional[List['MediaContent']] = None,
                           stream: bool = False,
                           response_model: Optional[Type[BaseModel]] = None,
                           **kwargs) -> Union[GenerateResponse, Iterator[GenerateResponse]]:
@@ -151,8 +152,39 @@ class MLXProvider(BaseProvider):
                 finish_reason="error"
             )
 
+        # Handle media content first if present
+        processed_prompt = prompt
+        if media:
+            try:
+                from ..media.handlers import LocalMediaHandler
+                media_handler = LocalMediaHandler("mlx", self.model_capabilities, model_name=self.model)
+
+                # Create multimodal message combining text and media
+                multimodal_message = media_handler.create_multimodal_message(prompt, media)
+
+                # For MLX (local provider), we get text-embedded content
+                if isinstance(multimodal_message, str):
+                    processed_prompt = multimodal_message
+                else:
+                    # If we get a structured message, extract the content
+                    if isinstance(multimodal_message, dict) and "content" in multimodal_message:
+                        if isinstance(multimodal_message["content"], list):
+                            # Find text content in the structured message
+                            text_content = ""
+                            for item in multimodal_message["content"]:
+                                if item.get("type") == "text":
+                                    text_content = item.get("text", "")
+                                    break
+                            processed_prompt = text_content or prompt
+                        else:
+                            processed_prompt = str(multimodal_message["content"])
+            except ImportError:
+                self.logger.warning("Media processing not available. Install with: pip install abstractcore[media]")
+            except Exception as e:
+                self.logger.warning(f"Failed to process media content: {e}")
+
         # Build full prompt with tool support
-        full_prompt = self._build_prompt(prompt, messages, system_prompt, tools)
+        full_prompt = self._build_prompt(processed_prompt, messages, system_prompt, tools)
 
         # MLX generation parameters using unified system
         generation_kwargs = self._prepare_generation_kwargs(**kwargs)

abstractcore/providers/ollama_provider.py

@@ -109,6 +109,7 @@ class OllamaProvider(BaseProvider):
                           messages: Optional[List[Dict[str, str]]] = None,
                           system_prompt: Optional[str] = None,
                           tools: Optional[List[Dict[str, Any]]] = None,
+                          media: Optional[List['MediaContent']] = None,
                           stream: bool = False,
                           response_model: Optional[Type[BaseModel]] = None,
                           **kwargs) -> Union[GenerateResponse, Iterator[GenerateResponse]]:
@@ -160,9 +161,42 @@ class OllamaProvider(BaseProvider):
                 converted_messages = self._convert_messages_for_ollama(messages)
                 payload["messages"].extend(converted_messages)
 
-            # Add current prompt as user message (only if non-empty)
-            # When using messages array, prompt should be empty or already in messages
-            if prompt and prompt.strip():
+            # Handle media content regardless of prompt (media can be used with messages too)
+            if media:
+                # Get the text to combine with media
+                user_message_text = prompt.strip() if prompt else ""
+                try:
+                    from ..media.handlers import LocalMediaHandler
+                    media_handler = LocalMediaHandler("ollama", self.model_capabilities, model_name=self.model)
+
+                    # Create multimodal message combining text and media
+                    multimodal_message = media_handler.create_multimodal_message(user_message_text, media)
+
+                    # For local providers, we might get a string (embedded text) or dict (structured)
+                    if isinstance(multimodal_message, str):
+                        payload["messages"].append({
+                            "role": "user",
+                            "content": multimodal_message
+                        })
+                    else:
+                        payload["messages"].append(multimodal_message)
+                except ImportError:
+                    self.logger.warning("Media processing not available. Install with: pip install abstractcore[media]")
+                    if user_message_text:
+                        payload["messages"].append({
+                            "role": "user",
+                            "content": user_message_text
+                        })
+                except Exception as e:
+                    self.logger.warning(f"Failed to process media content: {e}")
+                    if user_message_text:
+                        payload["messages"].append({
+                            "role": "user",
+                            "content": user_message_text
+                        })
+
+            # Add prompt as separate message if provided (for backward compatibility)
+            elif prompt and prompt.strip():
                 payload["messages"].append({
                     "role": "user",
                     "content": prompt

abstractcore/providers/openai_provider.py

@@ -65,6 +65,7 @@ class OpenAIProvider(BaseProvider):
                           messages: Optional[List[Dict[str, str]]] = None,
                           system_prompt: Optional[str] = None,
                           tools: Optional[List[Dict[str, Any]]] = None,
+                          media: Optional[List['MediaContent']] = None,
                           stream: bool = False,
                           response_model: Optional[Type[BaseModel]] = None,
                           **kwargs) -> Union[GenerateResponse, Iterator[GenerateResponse]]:
@@ -89,7 +90,23 @@ class OpenAIProvider(BaseProvider):
 
         # Add current prompt as user message
         if prompt and prompt not in [msg.get("content") for msg in (messages or [])]:
-            api_messages.append({"role": "user", "content": prompt})
+            # Handle multimodal message with media content
+            if media:
+                try:
+                    from ..media.handlers import OpenAIMediaHandler
+                    media_handler = OpenAIMediaHandler(self.model_capabilities)
+
+                    # Create multimodal message combining text and media
+                    multimodal_message = media_handler.create_multimodal_message(prompt, media)
+                    api_messages.append(multimodal_message)
+                except ImportError:
+                    self.logger.warning("Media processing not available. Install with: pip install abstractcore[media]")
+                    api_messages.append({"role": "user", "content": prompt})
+                except Exception as e:
+                    self.logger.warning(f"Failed to process media content: {e}")
+                    api_messages.append({"role": "user", "content": prompt})
+            else:
+                api_messages.append({"role": "user", "content": prompt})
 
         # Prepare API call parameters using unified system
         generation_kwargs = self._prepare_generation_kwargs(**kwargs)