abstractcore 2.4.2__py3-none-any.whl → 2.4.4__py3-none-any.whl

abstractcore/assets/model_capabilities.json

@@ -13,16 +13,40 @@
       "aliases": [],
       "max_tokens": 128000
     },
-    "gpt-4o": {
+    "gpt-4-turbo-with-vision": {
       "max_output_tokens": 4096,
       "tool_support": "native",
       "structured_output": "native",
       "parallel_tools": true,
       "max_tools": -1,
       "vision_support": true,
+      "audio_support": false,
+      "image_resolutions": [
+        "variable"
+      ],
+      "notes": "GPT-4 Turbo with vision capabilities",
+      "source": "OpenAI official docs 2025",
+      "canonical_name": "gpt-4-turbo-with-vision",
+      "aliases": [
+        "gpt-4-turbo-vision",
+        "gpt-4-vision-preview"
+      ],
+      "max_tokens": 128000
+    },
+    "gpt-4o": {
+      "max_output_tokens": 16384,
+      "tool_support": "native",
+      "structured_output": "native",
+      "parallel_tools": true,
+      "max_tools": -1,
+      "vision_support": true,
       "audio_support": true,
-      "notes": "Multimodal, optimized for speed",
-      "source": "OpenAI official docs",
+      "video_support": true,
+      "image_resolutions": [
+        "variable"
+      ],
+      "notes": "Multimodal omni model, 2x faster, half price, 5x higher rate limits (updated Nov 2024)",
+      "source": "OpenAI official docs 2025",
       "canonical_name": "gpt-4o",
       "aliases": [],
       "max_tokens": 128000
@@ -168,7 +192,9 @@
       "notes": "More likely to call unnecessary tools",
       "source": "Anthropic official docs",
       "canonical_name": "claude-3.5-haiku",
-      "aliases": [],
+      "aliases": [
+        "claude-3-5-haiku-20241022"
+      ],
       "max_tokens": 200000
     },
     "claude-3-opus": {
@@ -790,6 +816,128 @@
       "aliases": [],
       "max_tokens": 32768
     },
+    "magistral-small-2509": {
+      "max_output_tokens": 8192,
+      "tool_support": "native",
+      "structured_output": "native",
+      "parallel_tools": true,
+      "vision_support": true,
+      "audio_support": false,
+      "video_support": false,
+      "image_resolutions": [
+        "variable"
+      ],
+      "max_image_resolution": "variable",
+      "notes": "Mistral vision model optimized for multimodal tasks",
+      "source": "Mistral AI 2025 release",
+      "canonical_name": "magistral-small-2509",
+      "aliases": [
+        "mistralai/magistral-small-2509"
+      ],
+      "max_tokens": 128000
+    },
+    "Qwen/Qwen3-VL-8B-Instruct-FP8": {
+      "max_output_tokens": 8192,
+      "tool_support": "native",
+      "structured_output": "native",
+      "parallel_tools": true,
+      "vision_support": true,
+      "audio_support": false,
+      "video_support": false,
+      "image_resolutions": [
+        "variable"
+      ],
+      "max_image_resolution": "variable",
+      "image_patch_size": 16,
+      "max_image_tokens": 24576,
+      "pixel_grouping": "32x32",
+      "notes": "Qwen3-VL 8B model with FP8 quantization for HuggingFace, optimized for efficient inference",
+      "source": "Qwen team 2025 HuggingFace release",
+      "canonical_name": "Qwen/Qwen3-VL-8B-Instruct-FP8",
+      "aliases": [
+        "qwen3-vl-8b-fp8",
+        "qwen3-vl-8b-instruct-fp8"
+      ],
+      "max_tokens": 262144
+    },
+    "llama3.2-vision:11b": {
+      "max_output_tokens": 4096,
+      "tool_support": "prompted",
+      "structured_output": "prompted",
+      "parallel_tools": false,
+      "vision_support": true,
+      "audio_support": false,
+      "video_support": false,
+      "image_resolutions": [
+        "560x560",
+        "1120x560",
+        "560x1120",
+        "1120x1120"
+      ],
+      "max_image_resolution": "1120x1120",
+      "image_patch_size": 14,
+      "max_image_tokens": 6400,
+      "notes": "Llama 3.2 Vision 11B model with multimodal capabilities for visual recognition and reasoning",
+      "source": "Meta AI Llama 3.2 release",
+      "canonical_name": "llama3.2-vision:11b",
+      "aliases": [
+        "llama3.2-vision-11b",
+        "llama-3.2-vision:11b"
+      ],
+      "max_tokens": 131072
+    },
+    "llama3.2-vision:70b": {
+      "max_output_tokens": 4096,
+      "tool_support": "prompted",
+      "structured_output": "prompted",
+      "parallel_tools": false,
+      "vision_support": true,
+      "audio_support": false,
+      "video_support": false,
+      "image_resolutions": [
+        "560x560",
+        "1120x560",
+        "560x1120",
+        "1120x1120"
+      ],
+      "max_image_resolution": "1120x1120",
+      "image_patch_size": 14,
+      "max_image_tokens": 6400,
+      "notes": "Llama 3.2 Vision 70B model with advanced multimodal capabilities for complex visual reasoning",
+      "source": "Meta AI Llama 3.2 release",
+      "canonical_name": "llama3.2-vision:70b",
+      "aliases": [
+        "llama3.2-vision-70b",
+        "llama-3.2-vision:70b"
+      ],
+      "max_tokens": 131072
+    },
+    "llama3.2-vision:90b": {
+      "max_output_tokens": 4096,
+      "tool_support": "prompted",
+      "structured_output": "prompted",
+      "parallel_tools": false,
+      "vision_support": true,
+      "audio_support": false,
+      "video_support": false,
+      "image_resolutions": [
+        "560x560",
+        "1120x560",
+        "560x1120",
+        "1120x1120"
+      ],
+      "max_image_resolution": "1120x1120",
+      "image_patch_size": 14,
+      "max_image_tokens": 6400,
+      "notes": "Llama 3.2 Vision 90B model with top-tier multimodal capabilities for advanced visual understanding",
+      "source": "Meta AI Llama 3.2 release",
+      "canonical_name": "llama3.2-vision:90b",
+      "aliases": [
+        "llama3.2-vision-90b",
+        "llama-3.2-vision:90b"
+      ],
+      "max_tokens": 131072
+    },
     "gemma-2b": {
       "max_output_tokens": 8192,
       "tool_support": "none",
@@ -1089,26 +1237,202 @@
       "image_resolutions": [
         "variable"
       ],
-      "notes": "Qwen3-VL multimodal model with vision and video support",
+      "image_patch_size": 16,
+      "max_image_tokens": 24576,
+      "pixel_grouping": "32x32",
+      "notes": "Qwen3-VL multimodal model with vision and video support, 32x32 pixel patches",
       "source": "Alibaba Qwen3-VL technical report",
       "canonical_name": "qwen3-vl",
       "aliases": [],
       "max_tokens": 131072
     },
-    "gemma3n": {
+    "qwen3-vl-4b": {
+      "max_output_tokens": 8192,
+      "tool_support": "prompted",
+      "structured_output": "prompted",
+      "parallel_tools": false,
+      "vision_support": true,
+      "video_support": true,
+      "audio_support": false,
+      "image_resolutions": [
+        "variable"
+      ],
+      "max_image_resolution": "variable",
+      "image_patch_size": 16,
+      "max_image_tokens": 24576,
+      "pixel_grouping": "32x32",
+      "notes": "Qwen3-VL 4B dense model with 256K context, optimized for LMStudio",
+      "source": "Alibaba Qwen3-VL technical report 2025",
+      "canonical_name": "qwen3-vl-4b",
+      "aliases": [
+        "qwen/qwen3-vl-4b"
+      ],
+      "max_tokens": 262144
+    },
+    "qwen3-vl-8b": {
+      "max_output_tokens": 8192,
+      "tool_support": "prompted",
+      "structured_output": "prompted",
+      "parallel_tools": false,
+      "vision_support": true,
+      "video_support": true,
+      "audio_support": false,
+      "image_resolutions": [
+        "variable"
+      ],
+      "max_image_resolution": "variable",
+      "image_patch_size": 16,
+      "max_image_tokens": 24576,
+      "pixel_grouping": "32x32",
+      "notes": "Qwen3-VL 8B dense model with 256K context, optimized for LMStudio",
+      "source": "Alibaba Qwen3-VL technical report 2025",
+      "canonical_name": "qwen3-vl-8b",
+      "aliases": [
+        "qwen/qwen3-vl-8b"
+      ],
+      "max_tokens": 262144
+    },
+    "qwen3-vl-30b": {
+      "max_output_tokens": 8192,
+      "tool_support": "prompted",
+      "structured_output": "prompted",
+      "parallel_tools": false,
+      "vision_support": true,
+      "video_support": true,
+      "audio_support": false,
+      "image_resolutions": [
+        "variable"
+      ],
+      "max_image_resolution": "variable",
+      "image_patch_size": 16,
+      "max_image_tokens": 24576,
+      "pixel_grouping": "32x32",
+      "notes": "Qwen3-VL 30B MoE model (30.5B total/3.3B active), best performing vision model, 256K context",
+      "source": "Alibaba Qwen3-VL technical report 2025",
+      "canonical_name": "qwen3-vl-30b",
+      "aliases": [
+        "qwen/qwen3-vl-30b"
+      ],
+      "max_tokens": 262144
+    },
+    "qwen2.5-vl-7b": {
+      "max_output_tokens": 8192,
+      "tool_support": "prompted",
+      "structured_output": "native",
+      "parallel_tools": false,
+      "vision_support": true,
+      "audio_support": false,
+      "image_resolutions": [
+        "56x56 to 3584x3584"
+      ],
+      "max_image_resolution": "3584x3584",
+      "image_patch_size": 14,
+      "max_image_tokens": 16384,
+      "pixel_grouping": "28x28",
+      "notes": "Qwen2.5-VL 7B parameter vision model, 28x28 pixel patches, max 3584x3584 resolution",
+      "source": "Alibaba official docs",
+      "canonical_name": "qwen2.5-vl-7b",
+      "aliases": [
+        "qwen/qwen2.5-vl-7b",
+        "unsloth/Qwen2.5-VL-7B-Instruct-GGUF"
+      ],
+      "max_tokens": 128000
+    },    
+    "gemma3-4b": {
       "max_output_tokens": 8192,
       "tool_support": "native",
       "structured_output": "native",
       "parallel_tools": true,
       "vision_support": true,
-      "audio_support": true,
-      "video_support": true,
-      "notes": "Gemma3n device-optimized multimodal model",
-      "source": "Google Gemma3n announcement",
-      "canonical_name": "gemma3n",
+      "audio_support": false,
+      "video_support": false,
+      "image_resolutions": [
+        "896x896"
+      ],
+      "max_image_resolution": "896x896",
+      "vision_encoder": "SigLIP-400M",
+      "image_tokens_per_image": 256,
+      "adaptive_windowing": true,
+      "notes": "Gemma3 4B parameter model with vision support, 896x896 fixed resolution with adaptive windowing",
+      "source": "Google Gemma3 documentation 2025",
+      "canonical_name": "gemma3-4b",
+      "aliases": [
+        "gemma3:4b"
+      ],
+      "max_tokens": 128000
+    },
+    "qwen2.5vl:7b": {
+      "max_output_tokens": 8192,
+      "tool_support": "prompted",
+      "structured_output": "native",
+      "parallel_tools": false,
+      "vision_support": true,
+      "audio_support": false,
+      "image_resolutions": [
+        "56x56 to 3584x3584"
+      ],
+      "max_image_resolution": "3584x3584",
+      "image_patch_size": 14,
+      "max_image_tokens": 16384,
+      "pixel_grouping": "28x28",
+      "notes": "Qwen2.5-VL 7B Ollama variant, 28x28 pixel patches, max 3584x3584 resolution",
+      "source": "Ollama model library",
+      "canonical_name": "qwen2.5vl:7b",
+      "aliases": [
+        "qwen2.5vl"
+      ],
+      "max_tokens": 128000
+    },
+    "gemma3:4b-it-qat": {
+      "max_output_tokens": 8192,
+      "tool_support": "native",
+      "structured_output": "native",
+      "parallel_tools": true,
+      "vision_support": true,
+      "audio_support": false,
+      "video_support": false,
+      "image_resolutions": [
+        "896x896"
+      ],
+      "max_image_resolution": "896x896",
+      "vision_encoder": "SigLIP-400M",
+      "image_tokens_per_image": 256,
+      "adaptive_windowing": true,
+      "notes": "Gemma3 4B instruct-tuned quantized model for Ollama, 896x896 fixed resolution",
+      "source": "Ollama model library",
+      "canonical_name": "gemma3:4b-it-qat",
       "aliases": [],
       "max_tokens": 128000
     },
+    "gemma3n:e4b": {
+      "max_output_tokens": 8192,
+      "tool_support": "native",
+      "structured_output": "native",
+      "parallel_tools": true,
+      "vision_support": true,
+      "audio_support": true,
+      "video_support": true,
+      "image_resolutions": [
+        "896x896"
+      ],
+      "max_image_resolution": "896x896",
+      "vision_encoder": "SigLIP-400M",
+      "image_tokens_per_image": 256,
+      "adaptive_windowing": true,
+      "memory_footprint": "3GB",
+      "notes": "Gemma3n device-optimized multimodal model, 896x896 fixed resolution",
+      "source": "Google Gemma3n documentation 2025",
+      "canonical_name": "gemma3n:e4b",
+      "aliases": [
+        "gemma3n:e4b:latest",
+        "gemma-3n-e4b",
+        "google/gemma-3n-e4b",
+        "gemma3n",
+        "gemma3n:e2b:latest",
+        "gemma3n:e2b"
+      ],
+      "max_tokens": 32768
+    },
     "seed-oss": {
       "max_output_tokens": 8192,
       "tool_support": "prompted",
@@ -1178,6 +1502,91 @@
       "canonical_name": "llama-4-109b",
       "aliases": [],
       "max_tokens": 10000000
+    },
+    "granite3.2:2b": {
+      "max_output_tokens": 8192,
+      "tool_support": "prompted",
+      "structured_output": "prompted",
+      "parallel_tools": false,
+      "vision_support": false,
+      "audio_support": false,
+      "notes": "IBM Granite 3.2 2B text-only model with reasoning capabilities",
+      "source": "IBM Granite 3.2 technical report",
+      "canonical_name": "granite3.2:2b",
+      "aliases": [
+        "granite3.2-2b"
+      ],
+      "max_tokens": 32768
+    },
+    "granite3.2:8b": {
+      "max_output_tokens": 8192,
+      "tool_support": "prompted",
+      "structured_output": "prompted",
+      "parallel_tools": false,
+      "vision_support": false,
+      "audio_support": false,
+      "notes": "IBM Granite 3.2 8B text-only model with reasoning capabilities",
+      "source": "IBM Granite 3.2 technical report",
+      "canonical_name": "granite3.2:8b",
+      "aliases": [
+        "granite3.2-8b"
+      ],
+      "max_tokens": 32768
+    },
+    "granite3.2-vision:2b": {
+      "max_output_tokens": 8192,
+      "tool_support": "prompted",
+      "structured_output": "prompted",
+      "parallel_tools": false,
+      "vision_support": true,
+      "audio_support": false,
+      "video_support": false,
+      "image_resolutions": [
+        "768x768"
+      ],
+      "max_image_resolution": "768x768",
+      "vision_encoder": "SigLIP2-so400m-patch14-384",
+      "image_patch_size": 14,
+      "notes": "IBM Granite 3.2-Vision 2B model with SigLIP2 encoder, optimized for visual document understanding",
+      "source": "IBM Granite 3.2 technical report arXiv:2502.09927",
+      "canonical_name": "granite3.2-vision:2b",
+      "aliases": [
+        "granite3.2-vision:latest",
+        "granite3.2-vision",
+        "granite-vision",
+        "ibm-granite-vision"
+      ],
+      "max_tokens": 32768
+    },
+    "granite3.3:2b": {
+      "max_output_tokens": 8192,
+      "tool_support": "prompted",
+      "structured_output": "prompted",
+      "parallel_tools": false,
+      "vision_support": false,
+      "audio_support": false,
+      "notes": "IBM Granite 3.3 2B text-only model with enhanced reasoning capabilities",
+      "source": "IBM Granite 3.3 release announcement",
+      "canonical_name": "granite3.3:2b",
+      "aliases": [
+        "granite3.3-2b"
+      ],
+      "max_tokens": 32768
+    },
+    "granite3.3:8b": {
+      "max_output_tokens": 8192,
+      "tool_support": "prompted",
+      "structured_output": "prompted",
+      "parallel_tools": false,
+      "vision_support": false,
+      "audio_support": false,
+      "notes": "IBM Granite 3.3 8B text-only model with enhanced reasoning capabilities",
+      "source": "IBM Granite 3.3 release announcement",
+      "canonical_name": "granite3.3:8b",
+      "aliases": [
+        "granite3.3-8b"
+      ],
+      "max_tokens": 32768
     }
   },
   "tool_support_levels": {

abstractcore/core/interface.py

@@ -90,6 +90,7 @@ class AbstractCoreInterface(ABC):
                 messages: Optional[List[Dict[str, str]]] = None,
                 system_prompt: Optional[str] = None,
                 tools: Optional[List[Dict[str, Any]]] = None,
+                media: Optional[List[Union[str, Dict[str, Any], 'MediaContent']]] = None,
                 stream: bool = False,
                 **kwargs) -> Union[GenerateResponse, Iterator[GenerateResponse]]:
         """
@@ -100,6 +101,7 @@ class AbstractCoreInterface(ABC):
             messages: Optional conversation history
             system_prompt: Optional system prompt
             tools: Optional list of available tools
+            media: Optional list of media files (file paths, MediaContent objects, or dicts)
             stream: Whether to stream the response
             **kwargs: Additional provider-specific parameters
 

abstractcore/core/session.py

@@ -186,11 +186,15 @@ class BasicSession:
         if hasattr(self, 'tool_call_tags') and self.tool_call_tags is not None and 'tool_call_tags' not in kwargs:
             kwargs['tool_call_tags'] = self.tool_call_tags
 
+        # Extract media parameter explicitly (fix for media parameter passing)
+        media = kwargs.pop('media', None)
+
         # Call provider
         response = self.provider.generate(
             prompt=prompt,
             messages=messages,
             system_prompt=self.system_prompt,
+            media=media,
             **kwargs
         )
 

abstractcore/embeddings/manager.py

@@ -119,7 +119,7 @@ class EmbeddingManager:
     def __init__(
         self,
         model: str = None,
-        provider: str = "huggingface",
+        provider: str = None,
         backend: Union[str, EmbeddingBackend] = "auto",
         cache_dir: Optional[Path] = None,
         cache_size: int = 1000,
@@ -130,15 +130,20 @@ class EmbeddingManager:
 
         Args:
             model: Model identifier (HuggingFace model ID for HF provider, model name for others).
-            provider: Embedding provider ('huggingface', 'ollama', 'lmstudio'). Defaults to 'huggingface'.
+                  If None, uses configured default from AbstractCore config system.
+            provider: Embedding provider ('huggingface', 'ollama', 'lmstudio').
+                     If None, uses configured default from AbstractCore config system.
             backend: Inference backend for HuggingFace ('auto', 'pytorch', 'onnx', 'openvino')
             cache_dir: Directory for persistent cache. Defaults to ~/.abstractcore/embeddings
             cache_size: Maximum number of embeddings to cache in memory
             output_dims: Output dimensions for Matryoshka truncation (if supported by provider)
             trust_remote_code: Whether to trust remote code (HuggingFace only)
         """
-        # Store provider
-        self.provider = provider.lower()
+        # Load configuration defaults, but ONLY if parameters weren't explicitly provided
+        self._load_config_defaults(model, provider)
+
+        # Store provider (after config loading)
+        self.provider = self._resolved_provider.lower()
         
         # Validate provider
         if self.provider not in ["huggingface", "ollama", "lmstudio"]:
@@ -151,16 +156,16 @@ class EmbeddingManager:
         # Set up model identifier
         if self.provider == "huggingface":
             # Model configuration - HuggingFace only
-            if model is None:
-                model = get_default_model()  # Returns alias "all-minilm-l6-v2"
+            # Use resolved model (which includes config defaults if not explicitly provided)
+            resolved_model = self._resolved_model
 
             # Handle model aliases from our favored models config
-            if model in list_available_models():
-                self.model_config = get_model_config(model)
+            if resolved_model in list_available_models():
+                self.model_config = get_model_config(resolved_model)
                 self.model_id = self.model_config.model_id
             else:
                 # Direct HuggingFace model ID
-                self.model_id = model
+                self.model_id = resolved_model
                 self.model_config = None
 
             self.backend = EmbeddingBackend(backend) if backend != "auto" else None
@@ -175,22 +180,22 @@ class EmbeddingManager:
                     logger.warning(f"Dimension {output_dims} not in supported dims {self.model_config.matryoshka_dims}")
         else:
             # Ollama or LMStudio provider
-            if model is None:
+            if self._resolved_model is None:
                 raise ValueError(f"Model name is required for {self.provider} provider")
-            
-            self.model_id = model
+
+            self.model_id = self._resolved_model
             self.backend = None
             self.trust_remote_code = False
             
             # Create provider instance for delegation
             if self.provider == "ollama":
                 from ..providers.ollama_provider import OllamaProvider
-                self._provider_instance = OllamaProvider(model=model)
-                logger.info(f"Initialized Ollama embedding provider with model: {model}")
+                self._provider_instance = OllamaProvider(model=self._resolved_model)
+                logger.info(f"Initialized Ollama embedding provider with model: {self._resolved_model}")
             elif self.provider == "lmstudio":
                 from ..providers.lmstudio_provider import LMStudioProvider
-                self._provider_instance = LMStudioProvider(model=model)
-                logger.info(f"Initialized LMStudio embedding provider with model: {model}")
+                self._provider_instance = LMStudioProvider(model=self._resolved_model)
+                logger.info(f"Initialized LMStudio embedding provider with model: {self._resolved_model}")
 
         # Common setup for all providers
         self.cache_dir = Path(cache_dir) if cache_dir else Path.home() / ".abstractcore" / "embeddings"
@@ -235,6 +240,39 @@ class EmbeddingManager:
         else:
             self.has_events = False
 
+    def _load_config_defaults(self, model: Optional[str], provider: Optional[str]) -> None:
+        """Load configuration defaults, but ONLY for parameters not explicitly provided.
+
+        This ensures that direct parameters always take precedence over config defaults.
+        """
+        try:
+            # Import config manager - use lazy import to avoid circular dependencies
+            from ..config import get_config_manager
+            config_manager = get_config_manager()
+            embeddings_config = config_manager.config.embeddings
+
+            # Apply defaults ONLY if not explicitly provided
+            if provider is None:
+                self._resolved_provider = embeddings_config.provider or "huggingface"
+                logger.debug(f"Using configured default provider: {self._resolved_provider}")
+            else:
+                self._resolved_provider = provider
+                logger.debug(f"Using explicit provider parameter: {self._resolved_provider}")
+
+            if model is None:
+                # Use config default, or fallback to EmbeddingManager's original default
+                self._resolved_model = embeddings_config.model or "all-minilm-l6-v2"
+                logger.debug(f"Using configured default model: {self._resolved_model}")
+            else:
+                self._resolved_model = model
+                logger.debug(f"Using explicit model parameter: {self._resolved_model}")
+
+        except Exception as e:
+            # Fallback to hardcoded defaults if config system fails
+            logger.debug(f"Config system unavailable, using fallback defaults: {e}")
+            self._resolved_provider = provider or "huggingface"
+            self._resolved_model = model or "all-minilm-l6-v2"
+
     def _load_model(self):
         """Load the HuggingFace embedding model with optimal backend and reduced warnings."""
         try: