abstractcore 2.5.0__py3-none-any.whl → 2.5.3__py3-none-any.whl

abstractcore/providers/__init__.py

@@ -22,6 +22,16 @@ from .registry import (
     get_available_models_for_provider
 )
 
+# Model capability filtering (new system)
+from .model_capabilities import (
+    ModelInputCapability,
+    ModelOutputCapability,
+    get_model_input_capabilities,
+    get_model_output_capabilities,
+    filter_models_by_capabilities,
+    get_capability_summary
+)
+
 __all__ = [
     # Provider classes
     'BaseProvider',
@@ -43,4 +53,12 @@ __all__ = [
     'get_all_providers_status',
     'create_provider',
     'get_available_models_for_provider',
+
+    # Model capability filtering (new system)
+    'ModelInputCapability',
+    'ModelOutputCapability',
+    'get_model_input_capabilities',
+    'get_model_output_capabilities',
+    'filter_models_by_capabilities',
+    'get_capability_summary',
 ]

abstractcore/providers/anthropic_provider.py

@@ -32,6 +32,7 @@ class AnthropicProvider(BaseProvider):
 
     def __init__(self, model: str = "claude-3-haiku-20240307", api_key: Optional[str] = None, **kwargs):
         super().__init__(model, **kwargs)
+        self.provider = "anthropic"
 
         if not ANTHROPIC_AVAILABLE:
             raise ImportError("Anthropic package not installed. Install with: pip install anthropic")
@@ -454,9 +455,21 @@ class AnthropicProvider(BaseProvider):
         # Create new client with updated timeout
         self.client = anthropic.Anthropic(api_key=self.api_key, timeout=self._timeout)
     def list_available_models(self, **kwargs) -> List[str]:
-        """List available models from Anthropic API."""
+        """
+        List available models from Anthropic API.
+
+        Args:
+            **kwargs: Optional parameters including:
+                - api_key: Anthropic API key
+                - input_capabilities: List of ModelInputCapability enums to filter by input capability
+                - output_capabilities: List of ModelOutputCapability enums to filter by output capability
+
+        Returns:
+            List of model names, optionally filtered by capabilities
+        """
         try:
             import httpx
+            from .model_capabilities import filter_models_by_capabilities
 
             # Use provided API key or instance API key
             api_key = kwargs.get('api_key', self.api_key)
@@ -480,7 +493,21 @@ class AnthropicProvider(BaseProvider):
                 data = response.json()
                 models = [model["id"] for model in data.get("data", [])]
                 self.logger.debug(f"Retrieved {len(models)} models from Anthropic API")
-                return sorted(models, reverse=True)  # Latest models first
+                models = sorted(models, reverse=True)  # Latest models first
+
+                # Apply new capability filtering if provided
+                input_capabilities = kwargs.get('input_capabilities')
+                output_capabilities = kwargs.get('output_capabilities')
+                
+                if input_capabilities or output_capabilities:
+                    models = filter_models_by_capabilities(
+                        models, 
+                        input_capabilities=input_capabilities,
+                        output_capabilities=output_capabilities
+                    )
+
+
+                return models
             else:
                 self.logger.warning(f"Anthropic API returned status {response.status_code}")
                 return []

abstractcore/providers/base.py

@@ -3,6 +3,8 @@ Base provider with integrated telemetry, events, and exception handling.
 """
 
 import time
+import uuid
+from collections import deque
 from typing import List, Dict, Any, Optional, Union, Iterator, Type
 from abc import ABC, abstractmethod
 
@@ -38,6 +40,7 @@ class BaseProvider(AbstractCoreInterface, ABC):
 
     def __init__(self, model: str, **kwargs):
         AbstractCoreInterface.__init__(self, model, **kwargs)
+        self.provider = None
 
         # Setup structured logging
         self.logger = get_logger(self.__class__.__name__)
@@ -66,6 +69,13 @@ class BaseProvider(AbstractCoreInterface, ABC):
 
         # Create provider key for circuit breaker tracking
         self.provider_key = f"{self.__class__.__name__}:{self.model}"
+        
+        # Setup Glyph compression configuration
+        self.glyph_config = kwargs.get('glyph_config', None)
+
+        # Setup interaction tracing
+        self.enable_tracing = kwargs.get('enable_tracing', False)
+        self._traces = deque(maxlen=kwargs.get('max_traces', 100))  # Ring buffer for memory efficiency
 
         # Provider created successfully - no event emission needed
         # (The simplified event system focuses on generation and tool events only)
@@ -172,6 +182,97 @@ class BaseProvider(AbstractCoreInterface, ABC):
             result_info = f" (result length: {len(str(result))})" if result else ""
             self.logger.info(f"Tool call completed: {tool_name}{result_info}")
 
+    def _capture_trace(self, prompt: str, messages: Optional[List[Dict[str, str]]],
+                       system_prompt: Optional[str], tools: Optional[List[Dict[str, Any]]],
+                       response: GenerateResponse, kwargs: Dict[str, Any]) -> str:
+        """
+        Capture interaction trace for observability.
+
+        Args:
+            prompt: Input prompt
+            messages: Conversation history
+            system_prompt: System prompt
+            tools: Available tools
+            response: Generated response
+            kwargs: Additional generation parameters
+
+        Returns:
+            Trace ID (UUID string)
+        """
+        trace_id = str(uuid.uuid4())
+
+        # Extract generation parameters
+        temperature = kwargs.get('temperature', self.temperature)
+        max_tokens = kwargs.get('max_tokens', self.max_tokens)
+        max_output_tokens = kwargs.get('max_output_tokens', self.max_output_tokens)
+        seed = kwargs.get('seed', self.seed)
+        top_p = kwargs.get('top_p', getattr(self, 'top_p', None))
+        top_k = kwargs.get('top_k', getattr(self, 'top_k', None))
+
+        # Build parameters dict
+        parameters = {
+            'temperature': temperature,
+            'max_tokens': max_tokens,
+            'max_output_tokens': max_output_tokens,
+        }
+        if seed is not None:
+            parameters['seed'] = seed
+        if top_p is not None:
+            parameters['top_p'] = top_p
+        if top_k is not None:
+            parameters['top_k'] = top_k
+
+        # Create trace record
+        trace = {
+            'trace_id': trace_id,
+            'timestamp': datetime.now().isoformat(),
+            'provider': self.__class__.__name__,
+            'model': self.model,
+            'system_prompt': system_prompt,
+            'prompt': prompt,
+            'messages': messages,
+            'tools': tools,
+            'parameters': parameters,
+            'response': {
+                'content': response.content,
+                'raw_response': None,  # Omit raw_response to save memory and avoid logging sensitive data
+                'tool_calls': response.tool_calls,
+                'finish_reason': response.finish_reason,
+                'usage': response.usage,
+                'generation_time_ms': response.gen_time,
+            },
+            'metadata': kwargs.get('trace_metadata', {})
+        }
+
+        # Store trace in ring buffer
+        self._traces.append(trace)
+
+        return trace_id
+
+    def get_traces(self, trace_id: Optional[str] = None, last_n: Optional[int] = None) -> Union[Dict[str, Any], List[Dict[str, Any]]]:
+        """
+        Retrieve interaction traces.
+
+        Args:
+            trace_id: Optional specific trace ID to retrieve
+            last_n: Optional number of most recent traces to retrieve
+
+        Returns:
+            Single trace dict if trace_id provided, list of traces otherwise
+        """
+        if trace_id:
+            # Find specific trace by ID
+            for trace in self._traces:
+                if trace['trace_id'] == trace_id:
+                    return trace
+            return None
+
+        if last_n:
+            # Return last N traces
+            return list(self._traces)[-last_n:] if len(self._traces) >= last_n else list(self._traces)
+
+        # Return all traces
+        return list(self._traces)
 
     def _handle_api_error(self, error: Exception) -> Exception:
         """
@@ -210,6 +311,7 @@ class BaseProvider(AbstractCoreInterface, ABC):
                                retry_strategy=None,  # Custom retry strategy for structured output
                                tool_call_tags: Optional[str] = None,  # Tool call tag rewriting
                                execute_tools: Optional[bool] = None,  # Tool execution control
+                               glyph_compression: Optional[str] = None,  # Glyph compression preference
                                **kwargs) -> Union[GenerateResponse, Iterator[GenerateResponse], BaseModel]:
         """
         Generate with integrated telemetry and error handling.
@@ -226,6 +328,7 @@ class BaseProvider(AbstractCoreInterface, ABC):
             retry_strategy: Optional retry strategy for structured output validation
             tool_call_tags: Optional tool call tag format for rewriting
             execute_tools: Whether to execute tools automatically (True) or let agent handle execution (False)
+            glyph_compression: Glyph compression preference ("auto", "always", "never")
         """
         # Handle structured output request
         if response_model is not None:
@@ -268,8 +371,17 @@ class BaseProvider(AbstractCoreInterface, ABC):
 
         # Process media content if provided
         processed_media = None
+        media_metadata = None
         if media:
-            processed_media = self._process_media_content(media)
+            compression_pref = glyph_compression or kwargs.get('glyph_compression', 'auto')
+            processed_media = self._process_media_content(media, compression_pref)
+            
+            # Extract metadata from processed media for response
+            if processed_media:
+                media_metadata = []
+                for media_content in processed_media:
+                    if hasattr(media_content, 'metadata') and media_content.metadata:
+                        media_metadata.append(media_content.metadata)
 
         # Convert tools to ToolDefinition objects first (outside retry loop)
         converted_tools = None
@@ -326,6 +438,7 @@ class BaseProvider(AbstractCoreInterface, ABC):
                     stream=stream,
                     execute_tools=should_execute_tools,
                     tool_call_tags=tool_call_tags,
+                    media_metadata=media_metadata,
                     **kwargs
                 )
 
@@ -379,6 +492,26 @@ class BaseProvider(AbstractCoreInterface, ABC):
                     # Apply default qwen3 rewriting for non-streaming responses
                     response = self._apply_non_streaming_tag_rewriting(response, tool_call_tags)
 
+                # Add visual token calculation if media metadata is available
+                if media_metadata and response:
+                    self.logger.debug(f"Enhancing response with visual tokens from {len(media_metadata)} media items")
+                    response = self._enhance_response_with_visual_tokens(response, media_metadata)
+
+                # Capture interaction trace if enabled
+                if self.enable_tracing and response:
+                    trace_id = self._capture_trace(
+                        prompt=prompt,
+                        messages=messages,
+                        system_prompt=system_prompt,
+                        tools=converted_tools,
+                        response=response,
+                        kwargs=kwargs
+                    )
+                    # Attach trace_id to response metadata
+                    if not response.metadata:
+                        response.metadata = {}
+                    response.metadata['trace_id'] = trace_id
+
                 self._track_generation(prompt, response, start_time, success=True, stream=False)
                 return response
 
@@ -410,6 +543,7 @@ class BaseProvider(AbstractCoreInterface, ABC):
                           stream: bool = False,
                           response_model: Optional[Type[BaseModel]] = None,
                           execute_tools: Optional[bool] = None,
+                          media_metadata: Optional[List[Dict[str, Any]]] = None,
                           **kwargs) -> Union[GenerateResponse, Iterator[GenerateResponse]]:
         """
         Internal generation method to be implemented by subclasses.
@@ -428,6 +562,102 @@ class BaseProvider(AbstractCoreInterface, ABC):
         """
         raise NotImplementedError("Subclasses must implement _generate_internal")
 
+    def _enhance_response_with_visual_tokens(self, response: GenerateResponse, media_metadata: List[Dict[str, Any]]) -> GenerateResponse:
+        """
+        Enhance the response with visual token calculations for Glyph compression.
+        This method is called automatically by BaseProvider for all providers.
+        """
+        try:
+            # Calculate visual tokens using VLM token calculator
+            provider_name = self.provider or self.__class__.__name__.lower().replace('provider', '')
+            self.logger.debug(f"Calculating visual tokens for provider={provider_name}, model={self.model}")
+            
+            visual_tokens = self._calculate_visual_tokens(media_metadata, provider_name, self.model)
+            self.logger.debug(f"Calculated visual tokens: {visual_tokens}")
+            
+            if visual_tokens > 0:
+                # Ensure response has metadata
+                if not response.metadata:
+                    response.metadata = {}
+                
+                # Add visual token information to metadata
+                response.metadata['visual_tokens'] = visual_tokens
+                
+                # Ensure response has usage dict
+                if not response.usage:
+                    response.usage = {}
+                
+                # Add visual tokens to usage
+                response.usage['visual_tokens'] = visual_tokens
+                
+                # Update total tokens to include visual tokens
+                original_total = response.usage.get('total_tokens', 0)
+                response.usage['total_tokens'] = original_total + visual_tokens
+                
+                self.logger.info(f"Enhanced response with {visual_tokens} visual tokens (new total: {response.usage['total_tokens']})")
+            else:
+                self.logger.debug("No visual tokens calculated - skipping enhancement")
+                
+        except Exception as e:
+            self.logger.warning(f"Failed to enhance response with visual tokens: {e}")
+        
+        return response
+
+    def _calculate_visual_tokens(self, media_metadata: List[Dict[str, Any]], provider: str, model: str) -> int:
+        """Calculate visual tokens from media metadata using VLM token calculator."""
+        try:
+            from ..utils.vlm_token_calculator import VLMTokenCalculator
+            from pathlib import Path
+            
+            calculator = VLMTokenCalculator()
+            total_visual_tokens = 0
+            
+            self.logger.debug(f"Processing {len(media_metadata)} media metadata items")
+            
+            for i, metadata in enumerate(media_metadata):
+                self.logger.debug(f"Metadata {i}: processing_method={metadata.get('processing_method')}")
+                
+                # Check if this is Glyph compression
+                if metadata.get('processing_method') == 'direct_pdf_conversion':
+                    glyph_cache_dir = metadata.get('glyph_cache_dir')
+                    total_images = metadata.get('total_images', 0)
+                    
+                    self.logger.debug(f"Glyph metadata found: cache_dir={glyph_cache_dir}, total_images={total_images}")
+                    
+                    if glyph_cache_dir and Path(glyph_cache_dir).exists():
+                        # Get actual image paths
+                        cache_dir = Path(glyph_cache_dir)
+                        image_paths = list(cache_dir.glob("image_*.png"))
+                        
+                        self.logger.debug(f"Found {len(image_paths)} images in cache directory")
+                        
+                        if image_paths:
+                            # Calculate tokens for all images
+                            token_analysis = calculator.calculate_tokens_for_images(
+                                image_paths=image_paths,
+                                provider=provider,
+                                model=model
+                            )
+                            total_visual_tokens += token_analysis['total_tokens']
+                            
+                            self.logger.debug(f"Calculated {token_analysis['total_tokens']} visual tokens for {len(image_paths)} Glyph images")
+                        else:
+                            # Fallback: estimate based on total_images
+                            base_tokens = calculator.PROVIDER_CONFIGS.get(provider, {}).get('base_tokens', 512)
+                            estimated_tokens = total_images * base_tokens
+                            total_visual_tokens += estimated_tokens
+                            
+                            self.logger.debug(f"Estimated {estimated_tokens} visual tokens for {total_images} Glyph images (fallback)")
+                    else:
+                        self.logger.debug(f"Cache directory not found or doesn't exist: {glyph_cache_dir}")
+            
+            self.logger.debug(f"Total visual tokens calculated: {total_visual_tokens}")
+            return total_visual_tokens
+            
+        except Exception as e:
+            self.logger.warning(f"Failed to calculate visual tokens: {e}")
+            return 0
+
     def _initialize_token_limits(self):
         """Initialize default token limits based on model capabilities"""
         # Set default max_tokens if not provided
@@ -804,12 +1034,14 @@ class BaseProvider(AbstractCoreInterface, ABC):
         """Rough estimation of token count for given text"""
         return super().estimate_tokens(text)
 
-    def _process_media_content(self, media: List[Union[str, Dict[str, Any], 'MediaContent']]) -> List['MediaContent']:
+    def _process_media_content(self, media: List[Union[str, Dict[str, Any], 'MediaContent']], 
+                              glyph_compression: str = "auto") -> List['MediaContent']:
         """
         Process media content from various input formats into standardized MediaContent objects.
 
         Args:
             media: List of media inputs (file paths, MediaContent objects, or dicts)
+            glyph_compression: Glyph compression preference (auto, always, never)
 
         Returns:
             List of processed MediaContent objects
@@ -837,8 +1069,16 @@ class BaseProvider(AbstractCoreInterface, ABC):
             try:
                 if isinstance(media_item, str):
                     # File path - process with auto media handler
-                    handler = AutoMediaHandler()
-                    result = handler.process_file(media_item)
+                    handler = AutoMediaHandler(
+                        enable_glyph_compression=True,
+                        glyph_config=getattr(self, 'glyph_config', None)
+                    )
+                    result = handler.process_file(
+                        media_item,
+                        provider=self.provider,
+                        model=self.model,
+                        glyph_compression=glyph_compression
+                    )
                     if result.success:
                         processed_media.append(result.media_content)
                     else:
@@ -880,14 +1120,47 @@ class BaseProvider(AbstractCoreInterface, ABC):
         The server will use this method to aggregate models across all providers.
 
         Args:
-            **kwargs: Provider-specific parameters (e.g., api_key, base_url)
+            **kwargs: Provider-specific parameters including:
+                - api_key: API key for authentication (if required)
+                - base_url: Base URL for API endpoint (if applicable)
+                - input_capabilities: Optional list of ModelInputCapability enums to filter by input capability
+                    (e.g., [ModelInputCapability.IMAGE] for vision models)
+                - output_capabilities: Optional list of ModelOutputCapability enums to filter by output capability
+                    (e.g., [ModelOutputCapability.EMBEDDINGS] for embedding models)
 
         Returns:
-            List of model names available for this provider
+            List of model names available for this provider, optionally filtered by capabilities
+
+        Examples:
+            >>> from abstractcore.providers import OpenAIProvider
+            >>> from abstractcore.providers.model_capabilities import ModelInputCapability, ModelOutputCapability
+            >>>
+            >>> # Get all models
+            >>> all_models = OpenAIProvider.list_available_models(api_key="...")
+            >>>
+            >>> # Get models that can analyze images
+            >>> vision_models = OpenAIProvider.list_available_models(
+            ...     api_key="...",
+            ...     input_capabilities=[ModelInputCapability.IMAGE]
+            ... )
+            >>>
+            >>> # Get embedding models
+            >>> embedding_models = OpenAIProvider.list_available_models(
+            ...     api_key="...",
+            ...     output_capabilities=[ModelOutputCapability.EMBEDDINGS]
+            ... )
+            >>>
+            >>> # Get vision models that generate text (most common case)
+            >>> vision_text_models = OpenAIProvider.list_available_models(
+            ...     api_key="...",
+            ...     input_capabilities=[ModelInputCapability.TEXT, ModelInputCapability.IMAGE],
+            ...     output_capabilities=[ModelOutputCapability.TEXT]
+            ... )
 
         Note:
             This is an abstract method that MUST be implemented by all provider subclasses.
             Each provider should implement its own discovery logic (API calls, local scanning, etc.).
+            Providers should apply the capability filters if provided in kwargs.
         """
         pass