abstractcore 2.9.1__py3-none-any.whl → 2.11.4__py3-none-any.whl

abstractcore/media/auto_handler.py

@@ -12,7 +12,7 @@ from typing import Dict, Any, Optional, List
 
 from .base import BaseMediaHandler
 from .types import MediaContent, MediaType, ContentFormat, detect_media_type
-from .processors import ImageProcessor, TextProcessor, PDFProcessor, OfficeProcessor
+from .processors import ImageProcessor, TextProcessor, PDFProcessor, OfficeProcessor, AudioProcessor, VideoProcessor
 from ..exceptions import UnsupportedFeatureError
 
 # Import Glyph compression support
@@ -60,6 +60,7 @@ class AutoMediaHandler(BaseMediaHandler):
         self._text_processor = None
         self._pdf_processor = None
         self._office_processor = None
+        self._video_processor = None
         
         # Initialize Glyph compression support
         self._compression_orchestrator = None
@@ -98,20 +99,17 @@ class AutoMediaHandler(BaseMediaHandler):
             availability['office'] = True
         except ImportError:
             availability['office'] = False
+
+        # AudioProcessor (dependency-free)
+        availability['audio'] = True
+
+        # VideoProcessor (dependency-free)
+        availability['video'] = True
         
-        # GlyphProcessor (requires reportlab and pdf2image)
-        glyph_deps_available = True
-        if GLYPH_AVAILABLE and self.enable_compression:
-            # Check actual dependencies
-            try:
-                import reportlab
-                import pdf2image
-            except ImportError:
-                glyph_deps_available = False
-        else:
-            glyph_deps_available = False
-            
-        availability['glyph'] = glyph_deps_available
+        # GlyphProcessor (PIL renderer). Requires Pillow at runtime.
+        availability['glyph'] = bool(
+            GLYPH_AVAILABLE and self.enable_compression and availability.get('image', False)
+        )
 
         return availability
 
@@ -138,6 +136,12 @@ class AutoMediaHandler(BaseMediaHandler):
         if self._office_processor is None:
             self._office_processor = OfficeProcessor(**self.processor_config)
         return self._office_processor
+
+    def _get_video_processor(self) -> VideoProcessor:
+        """Get or create VideoProcessor instance."""
+        if self._video_processor is None:
+            self._video_processor = VideoProcessor(**self.processor_config)
+        return self._video_processor
     
     def _get_compression_orchestrator(self) -> 'CompressionOrchestrator':
         """Get or create CompressionOrchestrator instance."""
@@ -195,7 +199,14 @@ class AutoMediaHandler(BaseMediaHandler):
             else:
                 return self._get_text_processor()
 
-        # Handle other media types (audio, video) - not yet implemented
+        # Handle audio
+        elif media_type == MediaType.AUDIO:
+            return AudioProcessor(**self.processor_config)
+        # Handle video
+        elif media_type == MediaType.VIDEO:
+            if self._available_processors.get('video', False):
+                return self._get_video_processor()
+            return None
         else:
             self.logger.warning(f"Media type {media_type.value} not yet supported")
             return None
@@ -368,19 +379,21 @@ class AutoMediaHandler(BaseMediaHandler):
         # Check dependencies
         missing_deps = []
         try:
-            import reportlab
+            from PIL import Image  # noqa: F401
         except ImportError:
-            missing_deps.append("reportlab")
+            missing_deps.append("Pillow")
             
         try:
             import pdf2image
         except ImportError:
-            missing_deps.append("pdf2image")
+            # Only required for the experimental direct PDF→image path.
+            missing_deps.append("pdf2image (optional)")
             
         if missing_deps:
             deps_str = ", ".join(missing_deps)
             self.logger.warning(f"Missing Glyph dependencies: {deps_str}")
-            self.logger.warning(f"Install with: pip install {' '.join(missing_deps)}")
+            self.logger.warning("Install with: pip install \"abstractcore[compression]\" (Pillow renderer)")
+            self.logger.warning("Optional (PDF→image): pip install pdf2image (+ Poppler installed on your system)")
         
         if not self.enable_compression:
             self.logger.warning("Glyph compression is disabled in AutoMediaHandler configuration")
@@ -510,9 +523,9 @@ class AutoMediaHandler(BaseMediaHandler):
         elif media_type == MediaType.DOCUMENT:
             return True  # Always supported via text processor at minimum
         elif media_type == MediaType.AUDIO:
-            return False  # Not yet implemented
+            return self._available_processors.get('audio', False)
         elif media_type == MediaType.VIDEO:
-            return False  # Not yet implemented
+            return self._available_processors.get('video', False)
         return False
 
     def supports_format(self, media_type: MediaType, format_ext: str) -> bool:
@@ -550,6 +563,13 @@ class AutoMediaHandler(BaseMediaHandler):
             # This allows processing of unknown document formats
             return True
 
+        elif media_type == MediaType.AUDIO:
+            # AudioProcessor is dependency-free in v0; accept common audio containers.
+            return format_ext.lower() in {'mp3', 'wav', 'm4a', 'ogg', 'flac', 'aac', 'webm'}
+        elif media_type == MediaType.VIDEO:
+            # VideoProcessor is dependency-free in v0; frame extraction fallback may require ffmpeg.
+            return format_ext.lower() in {'mp4', 'mov', 'mkv', 'webm', 'avi', 'wmv', 'm4v'}
+
         return False
 
     def get_supported_formats(self) -> Dict[str, List[str]]:
@@ -654,4 +674,4 @@ class AutoMediaHandler(BaseMediaHandler):
         else:
             # Basic estimation based on file size
             file_size_mb = file_path.stat().st_size / (1024 * 1024)
-            return max(0.1, file_size_mb / 10.0)  # ~10MB/second processing rate
+            return max(0.1, file_size_mb / 10.0)  # ~10MB/second processing rate

abstractcore/media/base.py

@@ -453,4 +453,47 @@ class BaseProviderMediaHandler(BaseMediaHandler):
         Returns:
             True if provider can handle this content
         """
-        return self.supports_media_type(media_content.media_type)
+        return self.supports_media_type(media_content.media_type)
+
+    def estimate_tokens_for_media(self, media_content: MediaContent) -> int:
+        """
+        Estimate token usage for media content.
+
+        Base implementation that uses pre-computed estimates when available.
+        Subclasses can override _estimate_image_tokens() for provider-specific
+        image token calculations.
+
+        Args:
+            media_content: MediaContent to estimate
+
+        Returns:
+            Estimated token count
+        """
+        if media_content.media_type == MediaType.IMAGE:
+            return self._estimate_image_tokens(media_content)
+
+        elif media_content.media_type in [MediaType.TEXT, MediaType.DOCUMENT]:
+            # Use pre-computed estimate from processor if available (uses TokenUtils)
+            if 'estimated_tokens' in media_content.metadata:
+                return media_content.metadata['estimated_tokens']
+            # Fallback: rough estimation (~4 chars per token)
+            content_length = len(str(media_content.content))
+            return content_length // 4
+
+        return 0
+
+    def _estimate_image_tokens(self, media_content: MediaContent) -> int:
+        """
+        Estimate token usage for image content.
+
+        Override in subclasses for provider-specific image token calculations.
+        Default implementation returns a conservative estimate.
+
+        Args:
+            media_content: Image MediaContent to estimate
+
+        Returns:
+            Estimated token count for the image
+        """
+        # Conservative default - subclasses should override with provider-specific logic
+        return 512

abstractcore/media/capabilities.py

@@ -66,11 +66,18 @@ class MediaCapabilities:
             caps = {}
 
         # Base capabilities from JSON
+        video_mode = caps.get('video_input_mode')
+        if isinstance(video_mode, str):
+            vm = video_mode.strip().lower()
+            video_support = vm in {'frames', 'native'}
+        else:
+            video_support = caps.get('video_support', False)
+
         instance = cls(
             model_name=model,
             vision_support=caps.get('vision_support', False),
             audio_support=caps.get('audio_support', False),
-            video_support=caps.get('video_support', False),
+            video_support=video_support,
             image_resolutions=caps.get('image_resolutions', [])
         )
 
@@ -159,37 +166,9 @@ class MediaCapabilities:
             'text_embedding_preferred': self.text_embedding_preferred
         }
 
-    def estimate_media_tokens(self, media_type: MediaType, content_size: int = 0) -> int:
-        """
-        Estimate token usage for media content.
-
-        Args:
-            media_type: Type of media
-            content_size: Size of content in bytes (optional)
-
-        Returns:
-            Estimated token count
-        """
-        if not self.media_token_estimation:
-            return 0
-
-        if media_type == MediaType.IMAGE and self.vision_support:
-            # Base token cost for images varies by model
-            model_lower = self.model_name.lower()
-            if 'gpt-4o' in model_lower:
-                return 85 + (170 * 4)  # Simplified GPT-4o calculation
-            elif 'claude' in model_lower:
-                return 1600  # Anthropic standard
-            else:
-                return 512  # Conservative estimate for local models
-
-        elif media_type in [MediaType.TEXT, MediaType.DOCUMENT]:
-            # Text content token estimation
-            if content_size > 0:
-                return content_size // 4  # ~4 chars per token
-            return 100  # Default estimate
-
-        return 0
+    # Note: Token estimation is now handled by processors (TextProcessor, PDFProcessor, etc.)
+    # which add 'estimated_tokens' to MediaContent.metadata using TokenUtils.
+    # Handlers use BaseProviderMediaHandler.estimate_tokens_for_media() to retrieve it.
 
     def validate_media_content(self, media_type: MediaType, file_size: int = 0,
                               format: str = None) -> tuple[bool, Optional[str]]:
@@ -332,4 +311,4 @@ def get_max_images(model: str, provider: str = None) -> int:
 
 def should_use_text_embedding(model: str, provider: str = None) -> bool:
     """Check if model prefers text embedding over multimodal messages."""
-    return get_media_capabilities(model, provider).text_embedding_preferred
+    return get_media_capabilities(model, provider).text_embedding_preferred

abstractcore/media/enrichment.py

@@ -0,0 +1,105 @@
+"""
+Media enrichment metadata contract (v0).
+
+This module defines a provider-agnostic, response-level transparency hook for
+"input enrichment fallbacks" where non-text inputs (image/audio/video) are
+converted into short grounded text context (caption/transcript/frames) so a
+text-only model can proceed.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Literal, Optional, TypedDict
+
+
+MEDIA_ENRICHMENT_KEY = "media_enrichment"
+
+EnrichmentStatus = Literal["used", "skipped", "error"]
+BackendKind = Literal["llm", "plugin", "local_model", "unknown"]
+
+
+class EnrichmentBackend(TypedDict, total=False):
+    kind: BackendKind
+    provider: str
+    model: str
+    backend_id: str
+    source: str
+
+
+class MediaEnrichmentItem(TypedDict, total=False):
+    status: EnrichmentStatus
+    input_modality: str  # image|audio|video (string for forward-compat)
+    input_index: int
+    input_name: str
+    policy: str
+    summary_kind: str  # caption|transcript|frames (string for forward-compat)
+    backend: EnrichmentBackend
+    injected_text: str
+    injected_chars: int
+    artifact: Dict[str, Any]
+    error: str
+
+
+def build_enrichment_item(
+    *,
+    status: EnrichmentStatus,
+    input_modality: str,
+    summary_kind: str,
+    policy: Optional[str] = None,
+    backend: Optional[Dict[str, Any]] = None,
+    input_index: Optional[int] = None,
+    input_name: Optional[str] = None,
+    injected_text: Optional[str] = None,
+    artifact: Optional[Dict[str, Any]] = None,
+    error: Optional[str] = None,
+) -> MediaEnrichmentItem:
+    item: MediaEnrichmentItem = {
+        "status": status,
+        "input_modality": str(input_modality),
+        "summary_kind": str(summary_kind),
+    }
+    if isinstance(policy, str) and policy.strip():
+        item["policy"] = policy.strip()
+    if isinstance(backend, dict) and backend:
+        # Preserve only JSON-serializable primitives; callers should keep this small.
+        cleaned: Dict[str, Any] = {}
+        for k in ("kind", "provider", "model", "backend_id", "source"):
+            v = backend.get(k)
+            if isinstance(v, str) and v.strip():
+                cleaned[str(k)] = v.strip()
+        if cleaned:
+            item["backend"] = cleaned  # type: ignore[assignment]
+    if isinstance(input_index, int):
+        item["input_index"] = int(input_index)
+    if isinstance(input_name, str) and input_name.strip():
+        item["input_name"] = input_name.strip()
+    if isinstance(injected_text, str) and injected_text.strip():
+        txt = injected_text.strip()
+        item["injected_text"] = txt
+        item["injected_chars"] = len(txt)
+    if isinstance(artifact, dict) and artifact:
+        item["artifact"] = artifact
+    if isinstance(error, str) and error.strip():
+        item["error"] = error.strip()
+    return item
+
+
+def merge_enrichment_metadata(
+    metadata: Optional[Dict[str, Any]],
+    enrichments: Optional[List[Dict[str, Any]]],
+) -> Dict[str, Any]:
+    out: Dict[str, Any] = metadata if isinstance(metadata, dict) else {}
+    if not enrichments:
+        return out
+
+    existing = out.get(MEDIA_ENRICHMENT_KEY)
+    if not isinstance(existing, list):
+        existing = []
+        out[MEDIA_ENRICHMENT_KEY] = existing
+
+    for item in enrichments:
+        if isinstance(item, dict) and item:
+            existing.append(item)
+
+    return out
+

abstractcore/media/handlers/anthropic_handler.py

@@ -192,7 +192,16 @@ class AnthropicMediaHandler(BaseProviderMediaHandler):
         if media_content.media_type == MediaType.IMAGE:
             # Check if model supports vision
             if not self.model_capabilities.get('vision_support', False):
-                return False
+                # Fallback: consult the centralized media capability database.
+                # This keeps validation in sync with `abstractcore.media.capabilities.is_vision_model`
+                # and avoids hard-coding model-name heuristics here.
+                try:
+                    from ..capabilities import is_vision_model
+
+                    if not is_vision_model(model):
+                        return False
+                except Exception:
+                    return False
 
             # Check image size
             if hasattr(media_content, 'metadata'):
@@ -200,13 +209,9 @@ class AnthropicMediaHandler(BaseProviderMediaHandler):
                 if file_size > self.max_image_size:
                     return False
 
-            # Model-specific checks
-            if 'claude-3' in model_lower:
-                return True  # All Claude 3 models support vision
-            elif 'claude-3.5' in model_lower:
-                return True  # All Claude 3.5 models support vision
-            elif 'claude-4' in model_lower:
-                return True  # Future Claude 4 models
+            # If vision is supported (either by injected model capabilities or the centralized lookup),
+            # accept the image.
+            return True
 
         # Text/document validation
         elif media_content.media_type in [MediaType.TEXT, MediaType.DOCUMENT]:
@@ -219,28 +224,14 @@ class AnthropicMediaHandler(BaseProviderMediaHandler):
 
         return False
 
-    def estimate_tokens_for_media(self, media_content: MediaContent) -> int:
+    def _estimate_image_tokens(self, media_content: MediaContent) -> int:
         """
-        Estimate token usage for media content.
+        Anthropic-specific image token estimation.
 
-        Args:
-            media_content: MediaContent to estimate
-
-        Returns:
-            Estimated token count
+        Anthropic uses roughly ~1600 tokens per image for most cases.
+        This varies based on image content and complexity.
         """
-        if media_content.media_type == MediaType.IMAGE:
-            # Anthropic image token estimation
-            # Roughly ~1600 tokens per image for most cases
-            # This varies based on image content and complexity
-            return 1600
-
-        elif media_content.media_type in [MediaType.TEXT, MediaType.DOCUMENT]:
-            # Rough estimation: 3.5 characters per token (slightly better than GPT)
-            content_length = len(str(media_content.content))
-            return int(content_length / 3.5)
-
-        return 0
+        return 1600
 
     def get_model_media_limits(self, model: str) -> Dict[str, Any]:
         """
@@ -323,4 +314,4 @@ class AnthropicMediaHandler(BaseProviderMediaHandler):
             3. Any notable patterns, relationships, or conclusions
             4. Suggestions for how this information might be used or what actions might be taken
 
-            Be thorough but concise in your analysis."""
+            Be thorough but concise in your analysis."""