abstractcore 2.9.1__py3-none-any.whl → 2.11.2__py3-none-any.whl

abstractcore/media/processors/video_processor.py

@@ -0,0 +1,58 @@
+"""
+Video processor for AbstractCore media handling.
+
+v0 goals:
+- Treat video as a first-class media type (MediaType.VIDEO) in the media pipeline.
+- Keep processing lightweight and dependency-free by default (store as a file ref).
+
+Higher-level semantic handling (native video models, frame sampling, captioning)
+is handled by policy and capability layers (see planned video policy backlog).
+"""
+
+from __future__ import annotations
+
+import mimetypes
+from pathlib import Path
+
+from ..base import BaseMediaHandler, MediaProcessingError
+from ..types import ContentFormat, MediaCapabilities, MediaContent, MediaType
+
+
+class VideoProcessor(BaseMediaHandler):
+    """Lightweight video processor that stores a video file reference."""
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        self.capabilities = MediaCapabilities(
+            vision_support=False,
+            audio_support=False,
+            video_support=True,
+            document_support=False,
+            max_file_size=self.max_file_size,
+        )
+
+    def _process_internal(self, file_path: Path, media_type: MediaType, **kwargs) -> MediaContent:
+        if media_type != MediaType.VIDEO:
+            raise MediaProcessingError(f"VideoProcessor only handles video, got {media_type}")
+
+        mime_type, _enc = mimetypes.guess_type(str(file_path))
+        mime_type = mime_type or "application/octet-stream"
+
+        metadata = {
+            "file_name": file_path.name,
+            "file_path": str(file_path),
+            "file_size": file_path.stat().st_size if file_path.exists() else None,
+            "processor": self.__class__.__name__,
+        }
+        metadata.update(kwargs.get("metadata", {}) if isinstance(kwargs.get("metadata"), dict) else {})
+
+        return MediaContent(
+            media_type=MediaType.VIDEO,
+            content=str(file_path),
+            content_format=ContentFormat.FILE_PATH,
+            mime_type=mime_type,
+            file_path=str(file_path),
+            metadata=metadata,
+        )
+

abstractcore/media/types.py

@@ -12,7 +12,7 @@ from pathlib import Path
 from typing import Union, Dict, Any, Optional, List, Literal
 from enum import Enum
 
-from pydantic import BaseModel, Field, validator
+from pydantic import BaseModel, Field, field_validator
 
 
 class MediaType(Enum):
@@ -63,6 +63,98 @@ class MediaContent:
         elif self.content_format == ContentFormat.TEXT and isinstance(self.content, bytes):
             self.content = self.content.decode('utf-8')
 
+    def to_dict(self) -> Dict[str, Any]:
+        """Return a JSON-safe dict representation (best-effort)."""
+        content: Any = self.content
+        if isinstance(content, (bytes, bytearray)):
+            content = base64.b64encode(bytes(content)).decode("utf-8")
+        return {
+            "media_type": self.media_type.value if isinstance(self.media_type, MediaType) else str(self.media_type),
+            "content": content,
+            "content_format": self.content_format.value
+            if isinstance(self.content_format, ContentFormat)
+            else str(self.content_format),
+            "mime_type": str(self.mime_type or "application/octet-stream"),
+            "file_path": self.file_path,
+            "metadata": dict(self.metadata or {}),
+        }
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "MediaContent":
+        """Parse a dict into a MediaContent (best-effort, tolerant of common aliases)."""
+        if not isinstance(data, dict):
+            raise TypeError("MediaContent.from_dict expects a dict")
+
+        media_type_raw = data.get("media_type")
+        if media_type_raw is None:
+            media_type_raw = data.get("mediaType")
+
+        mime_type_raw = data.get("mime_type")
+        if mime_type_raw is None:
+            mime_type_raw = data.get("mimeType")
+        if mime_type_raw is None:
+            mime_type_raw = data.get("mime")
+
+        mime_type = str(mime_type_raw or "application/octet-stream")
+
+        if isinstance(media_type_raw, MediaType):
+            media_type = media_type_raw
+        elif isinstance(media_type_raw, str) and media_type_raw.strip():
+            media_type = MediaType(media_type_raw.strip())
+        else:
+            # Infer from MIME type when missing.
+            mt = mime_type.lower()
+            if mt.startswith("image/"):
+                media_type = MediaType.IMAGE
+            elif mt.startswith("audio/"):
+                media_type = MediaType.AUDIO
+            elif mt.startswith("video/"):
+                media_type = MediaType.VIDEO
+            elif mt.startswith("text/"):
+                media_type = MediaType.TEXT
+            else:
+                media_type = MediaType.DOCUMENT
+
+        content_format_raw = data.get("content_format")
+        if content_format_raw is None:
+            content_format_raw = data.get("contentFormat")
+        if content_format_raw is None:
+            content_format_raw = data.get("format")
+
+        file_path_raw = data.get("file_path")
+        if file_path_raw is None:
+            file_path_raw = data.get("filePath")
+
+        content = data.get("content")
+
+        if isinstance(content_format_raw, ContentFormat):
+            content_format = content_format_raw
+        elif isinstance(content_format_raw, str) and content_format_raw.strip():
+            content_format = ContentFormat(content_format_raw.strip())
+        else:
+            if isinstance(file_path_raw, str) and file_path_raw.strip():
+                content_format = ContentFormat.FILE_PATH
+            elif isinstance(content, (bytes, bytearray)):
+                content_format = ContentFormat.BINARY
+            elif isinstance(content, str):
+                content_format = ContentFormat.TEXT
+            else:
+                content_format = ContentFormat.AUTO
+
+        metadata_raw = data.get("metadata")
+        metadata = dict(metadata_raw) if isinstance(metadata_raw, dict) else {}
+
+        file_path = str(file_path_raw).strip() if isinstance(file_path_raw, str) and file_path_raw.strip() else None
+
+        return cls(
+            media_type=media_type,
+            content=content,
+            content_format=content_format,
+            mime_type=mime_type,
+            file_path=file_path,
+            metadata=metadata,
+        )
+
 
 class MultimodalMessage(BaseModel):
     """
@@ -78,8 +170,9 @@ class MultimodalMessage(BaseModel):
     )
     metadata: Dict[str, Any] = Field(default_factory=dict)
 
-    @validator('role')
-    def validate_role(cls, v):
+    @field_validator("role")
+    @classmethod
+    def validate_role(cls, v: str) -> str:
         valid_roles = {'user', 'assistant', 'system', 'tool'}
         if v not in valid_roles:
             raise ValueError(f"Role must be one of {valid_roles}")
@@ -454,4 +547,4 @@ def create_media_content(
             'file_name': path.name,
             'file_extension': path.suffix
         }
-    )
+    )

abstractcore/media/utils/image_scaler.py

@@ -5,11 +5,19 @@ Provides intelligent image scaling based on model-specific requirements
 and capabilities for vision models.
 """
 
+from __future__ import annotations  # PEP 563 - avoid hard PIL dependency at import time
+
 from typing import Tuple, Optional, Union, Dict, Any
 from enum import Enum
 from pathlib import Path
 
-from PIL import Image, ImageOps
+try:
+    from PIL import Image, ImageOps
+    PIL_AVAILABLE = True
+except ImportError:  # pragma: no cover
+    Image = None
+    ImageOps = None
+    PIL_AVAILABLE = False
 
 from ..base import MediaProcessingError
 from ...utils.structured_logging import get_logger
@@ -132,6 +140,11 @@ class ModelOptimizedScaler:
         Returns:
             Scaled PIL Image
         """
+        if not PIL_AVAILABLE:
+            raise MediaProcessingError(
+                "PIL/Pillow is required for image scaling. "
+                "Install with: pip install \"abstractcore[media]\""
+            )
         target_width, target_height = target_size
 
         if mode == ScalingMode.FIT:
@@ -278,6 +291,11 @@ def scale_image_for_model(image: Union[Image.Image, str, Path],
     Returns:
         Optimally scaled PIL Image
     """
+    if not PIL_AVAILABLE:
+        raise MediaProcessingError(
+            "PIL/Pillow is required for image scaling. "
+            "Install with: pip install \"abstractcore[media]\""
+        )
     if isinstance(image, (str, Path)):
         image = Image.open(image)
 
@@ -296,4 +314,4 @@ def get_optimal_size_for_model(model_name: str, original_size: Tuple[int, int])
         Optimal target size (width, height)
     """
     scaler = get_scaler()
-    return scaler.get_optimal_resolution(model_name, original_size)
+    return scaler.get_optimal_resolution(model_name, original_size)

abstractcore/media/utils/video_frames.py

@@ -0,0 +1,219 @@
+"""
+Video frame extraction utilities (v0).
+
+This module provides a small, dependency-light wrapper around ffmpeg/ffprobe
+to sample a bounded number of frames from a video for downstream analysis.
+
+Design goals:
+- deterministic sampling (timestamp-based)
+- bounded output (max_frames)
+- actionable errors when ffmpeg/ffprobe are unavailable
+"""
+
+from __future__ import annotations
+
+import shutil
+import subprocess
+import tempfile
+from pathlib import Path
+from typing import List, Optional, Tuple
+
+
+class VideoToolUnavailableError(RuntimeError):
+    pass
+
+
+def _which(cmd: str) -> Optional[str]:
+    try:
+        return shutil.which(cmd)
+    except Exception:
+        return None
+
+
+def probe_duration_s(video_path: Path) -> Optional[float]:
+    """Return best-effort duration (seconds) using ffprobe, or None."""
+    ffprobe = _which("ffprobe")
+    if not ffprobe:
+        return None
+
+    try:
+        out = subprocess.check_output(
+            [
+                ffprobe,
+                "-v",
+                "error",
+                "-show_entries",
+                "format=duration",
+                "-of",
+                "default=nk=1:nw=1",
+                str(video_path),
+            ],
+            text=True,
+            stderr=subprocess.STDOUT,
+        ).strip()
+        if not out:
+            return None
+        return float(out)
+    except Exception:
+        return None
+
+
+def probe_keyframe_timestamps_s(video_path: Path) -> List[float]:
+    """Return keyframe timestamps (seconds) using ffprobe when available."""
+    ffprobe = _which("ffprobe")
+    if not ffprobe:
+        return []
+
+    try:
+        out = subprocess.check_output(
+            [
+                ffprobe,
+                "-v",
+                "error",
+                "-skip_frame",
+                "nokey",
+                "-select_streams",
+                "v:0",
+                "-show_entries",
+                "frame=pkt_pts_time",
+                "-of",
+                "csv=p=0",
+                str(video_path),
+            ],
+            text=True,
+            stderr=subprocess.STDOUT,
+        )
+    except Exception:
+        return []
+
+    timestamps: List[float] = []
+    for line in (out or "").splitlines():
+        s = line.strip()
+        if not s:
+            continue
+        try:
+            t = float(s)
+        except Exception:
+            continue
+        if t < 0:
+            continue
+        timestamps.append(t)
+
+    # Deduplicate while preserving order (ffprobe can sometimes repeat values).
+    seen = set()
+    uniq: List[float] = []
+    for t in timestamps:
+        if t in seen:
+            continue
+        seen.add(t)
+        uniq.append(t)
+    return uniq
+
+
+def _build_timestamps(duration_s: Optional[float], max_frames: int) -> List[float]:
+    n = max(1, int(max_frames))
+    if duration_s is None or duration_s <= 0:
+        return [0.0]
+    # Sample away from the extreme endpoints to avoid decode edge-cases.
+    return [duration_s * (i + 1) / (n + 1) for i in range(n)]
+
+
+def _pick_evenly_spaced(values: List[float], k: int) -> List[float]:
+    if not values:
+        return []
+    n = len(values)
+    k = max(1, int(k))
+    if n <= k:
+        return list(values)
+    # Evenly spaced indices (include ends).
+    idxs = [round(i * (n - 1) / (k - 1)) for i in range(k)] if k > 1 else [round((n - 1) / 2)]
+    out: List[float] = []
+    last = None
+    for i in idxs:
+        i = max(0, min(n - 1, int(i)))
+        v = values[i]
+        if last is not None and v == last:
+            continue
+        out.append(v)
+        last = v
+    return out
+
+
+def extract_video_frames(
+    video_path: Path,
+    *,
+    max_frames: int = 3,
+    frame_format: str = "jpg",
+    sampling_strategy: str = "uniform",
+    max_side: Optional[int] = None,
+    output_dir: Optional[Path] = None,
+) -> Tuple[List[Path], List[float]]:
+    """
+    Extract up to max_frames as image files and return (frame_paths, timestamps_s).
+
+    Uses ffmpeg for extraction and ffprobe for duration (best-effort).
+    """
+    ffmpeg = _which("ffmpeg")
+    if not ffmpeg:
+        raise VideoToolUnavailableError("ffmpeg is required for video frame extraction. Install ffmpeg and ensure it is on PATH.")
+
+    if not isinstance(video_path, Path):
+        video_path = Path(video_path)
+    if not video_path.exists():
+        raise FileNotFoundError(str(video_path))
+
+    fmt = str(frame_format or "jpg").strip().lower()
+    if fmt == "jpeg":
+        fmt = "jpg"
+    if fmt not in {"jpg", "png"}:
+        fmt = "jpg"
+
+    out_dir = Path(output_dir) if output_dir is not None else Path(tempfile.mkdtemp(prefix="abstractcore_video_frames_"))
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    duration_s = probe_duration_s(video_path)
+    strategy = str(sampling_strategy or "uniform").strip().lower()
+    if strategy == "keyframes":
+        keyframes = probe_keyframe_timestamps_s(video_path)
+        timestamps = _pick_evenly_spaced(keyframes, int(max_frames)) if keyframes else _build_timestamps(duration_s, max_frames=max_frames)
+    else:
+        timestamps = _build_timestamps(duration_s, max_frames=max_frames)
+
+    frames: List[Path] = []
+    for idx, ts in enumerate(timestamps):
+        out_path = out_dir / f"frame_{idx+1:02d}.{fmt}"
+        cmd = [
+            ffmpeg,
+            "-hide_banner",
+            "-loglevel",
+            "error",
+            "-ss",
+            f"{ts:.3f}",
+            "-i",
+            str(video_path),
+            "-frames:v",
+            "1",
+        ]
+        if isinstance(max_side, int) and max_side > 0:
+            ms = int(max_side)
+            # Preserve aspect ratio, never upscale.
+            # If width >= height: clamp width to ms, derive height. Else clamp height.
+            vf = (
+                f"scale="
+                f"if(gt(iw\\,ih)\\,min(iw\\,{ms})\\,-2):"
+                f"if(gt(iw\\,ih)\\,-2\\,min(ih\\,{ms}))"
+            )
+            cmd.extend(["-vf", vf])
+        if fmt == "jpg":
+            cmd.extend(["-q:v", "2"])
+        cmd.append(str(out_path))
+
+        try:
+            subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        except subprocess.CalledProcessError:
+            continue
+
+        if out_path.exists() and out_path.stat().st_size > 0:
+            frames.append(out_path)
+
+    return frames, timestamps