videopython 0.55.1__tar.gz → 0.55.3__tar.gz

{videopython-0.55.1 → videopython-0.55.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: videopython
-Version: 0.55.1
+Version: 0.55.3
 Summary: Minimal video generation and processing library.
 Project-URL: Homepage, https://videopython.com
 Project-URL: Repository, https://github.com/bartwojtowicz/videopython/

{videopython-0.55.1 → videopython-0.55.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "videopython"
-version = "0.55.1"
+version = "0.55.3"
 description = "Minimal video generation and processing library."
 authors = [
     { name = "Bartosz Wójtowicz", email = "bartoszwojtowicz@outlook.com" },

{videopython-0.55.1 → videopython-0.55.3}/src/videopython/ai/_ollama.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import json
+import re
 from typing import Any
 
 import numpy as np
@@ -16,6 +17,45 @@ class OllamaError(AiError, RuntimeError):
     """Ollama returned unusable output (non-JSON or an unexpected shape)."""
 
 
+# Ollama's own default context window is 4096 tokens, and an oversized request
+# *fails* (``exceed_context_size_error``) instead of being truncated -- so a call
+# carrying images has to size the window before sending. A vision model spends
+# roughly this many tokens per image. Deliberately generous: underestimating
+# fails the request outright, overestimating only costs KV-cache memory.
+_TOKENS_PER_IMAGE = 1024
+# Headroom for the system prompt, the user text, and the generated answer.
+_TEXT_TOKEN_ALLOWANCE = 2048
+# Never ask for less than Ollama's own default.
+_MIN_NUM_CTX = 4096
+_CONTEXT_OVERFLOW_RE = re.compile(r"request \((\d+) tokens\) exceeds the available context size")
+
+
+def _round_num_ctx(token_count: int) -> int:
+    return max(_MIN_NUM_CTX, 1 << (token_count - 1).bit_length())
+
+
+def _num_ctx_for_images(image_count: int) -> int:
+    """Estimate the initial context window for ``image_count`` images.
+
+    Rounded up to a power of two so a run over scenes with differing frame counts
+    reuses a handful of window sizes instead of a new one per call -- ``num_ctx``
+    is a runner-level setting, so varying it every call risks reloading the model
+    between scenes.
+    """
+    needed = image_count * _TOKENS_PER_IMAGE + _TEXT_TOKEN_ALLOWANCE
+    return _round_num_ctx(needed)
+
+
+def _num_ctx_after_overflow(error: str, current: int) -> int | None:
+    """Return a larger context after Ollama reports a context overflow."""
+    match = _CONTEXT_OVERFLOW_RE.search(error)
+    if match:
+        return _round_num_ctx(int(match.group(1)) + _TEXT_TOKEN_ALLOWANCE)
+    if "exceed_context_size_error" in error:
+        return current * 2
+    return None
+
+
 class OllamaStructuredClient:
     """Generate schema-constrained JSON from text + optional images via Ollama.
 
@@ -30,6 +70,10 @@ class OllamaStructuredClient:
     budget thinking, stops on ``length``, and returns empty content. None of these
     callers want the chain-of-thought, so thinking is disabled on models that
     support it.
+
+    Image requests start with a context estimate based on the image count. If a
+    model uses more visual tokens, the request retries once with the token count
+    reported by Ollama. An explicit ``num_ctx`` in ``options`` always wins.
     """
 
     def __init__(self, model: str, *, host: str | None = None, options: dict[str, Any] | None = None) -> None:
@@ -69,12 +113,23 @@ class OllamaStructuredClient:
         if images:
             user["images"] = [encode_png_b64(image) for image in images]
         messages = [{"role": "system", "content": system}, user]
+        options = self.options
+        image_count = len(images) if images else 0
+        auto_num_ctx = image_count > 0 and "num_ctx" not in options
+        if auto_num_ctx:
+            options = {**options, "num_ctx": _num_ctx_for_images(image_count)}
         kwargs: dict[str, Any] = {}
         if self._supports_thinking():
             kwargs["think"] = False
-        response = self._get_client().chat(
-            model=self.model, messages=messages, format=schema, options=self.options, **kwargs
-        )
+        client = self._get_client()
+        try:
+            response = client.chat(model=self.model, messages=messages, format=schema, options=options, **kwargs)
+        except Exception as exc:
+            retry_num_ctx = _num_ctx_after_overflow(str(exc), options["num_ctx"]) if auto_num_ctx else None
+            if retry_num_ctx is None:
+                raise
+            options = {**options, "num_ctx": retry_num_ctx}
+            response = client.chat(model=self.model, messages=messages, format=schema, options=options, **kwargs)
         content = response.message.content
         try:
             data = json.loads(content)

{videopython-0.55.1 → videopython-0.55.3}/src/videopython/ai/effects.py

@@ -43,7 +43,11 @@ class ObjectDetectionOverlay(Effect):
     confidence_threshold: float = Field(0.5, ge=0, le=1, description="Minimum detection confidence to draw a box, 0-1.")
     class_filter: list[str] | None = Field(
         None,
-        description='Only draw these COCO class names, e.g. ["person", "car", "dog"]. Null draws all classes.',
+        description=(
+            'Only draw these COCO class names, e.g. ["person", "car", "motorcycle"]. '
+            "Null draws all classes. Standard COCO names and D-FINE's alternate spellings "
+            "(motorbike, aeroplane, sofa, pottedplant, diningtable, tvmonitor) are accepted."
+        ),
     )
     show_confidence: bool = Field(True, description="Append the detection confidence as a percentage to each label.")
     box_color: tuple[int, int, int] | None = Field(

{videopython-0.55.1 → videopython-0.55.3}/src/videopython/ai/understanding/audio.py

@@ -333,7 +333,16 @@ class AudioToText(ManagedPredictor):
         )
 
         all_words = self._assign_speakers_to_words(all_words, diarization_result)
-        return Transcription(words=all_words, language=transcription.language)
+
+        # Rebuilding from words regroups by speaker and drops the per-segment
+        # confidence the supplied transcription carried, exactly as it does on the
+        # combined path -- so re-attach it the same way. Without this, splitting
+        # transcription and diarization into two calls silently loses confidence
+        # that running them as one keeps.
+        source_segments = transcription.segments
+        rebuilt = Transcription(words=all_words, language=transcription.language)
+        _attach_confidence_by_overlap(rebuilt.segments, source_segments)
+        return rebuilt
 
     def _run_vad(self, audio_mono: Audio) -> list[tuple[float, float]]:
         """Return voiced spans in seconds using Silero VAD.

{videopython-0.55.1 → videopython-0.55.3}/src/videopython/ai/understanding/image.py

@@ -41,6 +41,11 @@ class SceneVLM(ManagedPredictor):
     The model must be vision-capable and support Ollama's structured-output
     ``format``; ``ollama pull <model>`` first. ``options`` are extra Ollama
     generation options merged over ``temperature=0``.
+
+    A scene's frames are sent as one multi-image request, so the context window
+    is sized to the frame count automatically -- Ollama's 4096-token default
+    fits only one or two frames and *fails* anything larger. Pass an explicit
+    ``num_ctx`` in ``options`` to override that sizing.
     """
 
     def __init__(

{videopython-0.55.1 → videopython-0.55.3}/src/videopython/ai/understanding/objects.py

@@ -10,8 +10,12 @@ two stay one mental model. Consumed by
 per-frame object analysis.
 
 D-FINE (Apache-2.0) replaced the AGPL-licensed Ultralytics YOLO weights. Its COCO
-labels use VOC-style spellings (``motorbike``, ``aeroplane``, ``sofa``, ``pottedplant``,
-``diningtable``, ``tvmonitor``) -- ``class_filter`` must use the model's exact names.
+labels use VOC-style spellings for six classes (``motorbike``, ``aeroplane``,
+``sofa``, ``pottedplant``, ``diningtable``, ``tvmonitor``) where the standard COCO
+names are ``motorcycle``, ``airplane``, ``couch``, ``potted plant``,
+``dining table`` and ``tv``. ``class_filter`` accepts either spelling: names are
+normalized through :data:`CLASS_ALIASES` before matching, and any name the model
+does not emit is logged once the class list is known.
 """
 
 from __future__ import annotations
@@ -19,6 +23,9 @@ from __future__ import annotations
 import logging
 from typing import TYPE_CHECKING, Any
 
+if TYPE_CHECKING:
+    from collections.abc import Iterable
+
 from videopython.ai._revisions import pinned
 from videopython.ai.understanding._detector import Backend, DetectorBase
 from videopython.base.description import BoundingBox, DetectedObject
@@ -28,7 +35,31 @@ if TYPE_CHECKING:
 
 logger = logging.getLogger(__name__)
 
-__all__ = ["ObjectDetector", "MODEL_SIZES"]
+__all__ = ["ObjectDetector", "MODEL_SIZES", "CLASS_ALIASES", "normalize_class_names"]
+
+# D-FINE emits VOC-style spellings for six COCO classes. Callers -- and LLMs asked
+# to name a class -- reach for the standard COCO spelling, which would match
+# nothing and draw nothing, with no error to explain the silence. Accept both.
+CLASS_ALIASES: dict[str, str] = {
+    "motorcycle": "motorbike",
+    "airplane": "aeroplane",
+    "couch": "sofa",
+    "potted plant": "pottedplant",
+    "dining table": "diningtable",
+    "tv": "tvmonitor",
+}
+
+
+def normalize_class_names(names: "Iterable[str]") -> tuple[str, ...]:
+    """Map COCO-standard class names onto the VOC-style spellings D-FINE emits.
+
+    Case and surrounding whitespace are normalized first, so ``"Potted Plant"``
+    and ``"potted  plant"`` both reach ``pottedplant``. A name with no alias is
+    passed through unchanged (lowercased) -- this translates spellings, it does
+    not validate them; see :meth:`ObjectDetector.unknown_filter_classes`.
+    """
+    return tuple(CLASS_ALIASES.get(key, key) for key in (" ".join(n.lower().split()) for n in names))
+
 
 # D-FINE COCO checkpoints (Apache-2.0), in ascending size/quality. ``model_size``
 # on ``ObjectDetectionOverlay`` maps onto these; pinned in ``_revisions.py``.
@@ -46,7 +77,9 @@ class ObjectDetector(DetectorBase[DetectedObject]):
     The D-FINE weights (default ``ustc-community/dfine-nano-coco``) download from
     HuggingFace on first real use; class names come from the model config.
     Detection is gated by ``confidence_threshold`` and optionally restricted to
-    ``class_filter`` (COCO class names; YOLO-style spellings are normalized).
+    ``class_filter``, which accepts either D-FINE's VOC-style spellings or the
+    standard COCO ones (``motorcycle``, ``airplane``, ``couch``, ``potted plant``,
+    ``dining table``, ``tv``) -- see :func:`normalize_class_names`.
     """
 
     DEFAULT_CONFIDENCE_THRESHOLD = 0.5
@@ -68,14 +101,18 @@ class ObjectDetector(DetectorBase[DetectedObject]):
                 ``ustc-community/dfine-nano-coco``, ``...-small-coco``,
                 ``...-medium-coco``, ``...-large-coco``). Downloaded on first use.
             confidence_threshold: Minimum detection confidence in ``[0, 1]``.
-            class_filter: If non-empty, only these COCO class names are kept
-                (D-FINE's VOC-style spelling, e.g. ``motorbike``/``tvmonitor``).
+            class_filter: If non-empty, only these COCO class names are kept.
+                Either spelling works -- ``motorcycle`` and ``motorbike`` both
+                match -- and names are normalized via
+                :func:`normalize_class_names`. A name the model never emits is
+                logged as a warning once the model loads, since it would
+                otherwise just silently match nothing.
             backend: Detection device - ``"cpu"``, ``"gpu"``, or ``"auto"``.
         """
         super().__init__(backend=backend)
         self.model_name = model_name
         self.confidence_threshold = confidence_threshold
-        self.class_filter = tuple(class_filter)
+        self.class_filter = normalize_class_names(class_filter)
         self._model: Any = None
         self._processor: Any = None
         self._class_names: dict[int, str] = {}
@@ -93,6 +130,26 @@ class ObjectDetector(DetectorBase[DetectedObject]):
             model = model.to("cuda")
         self._model = model
         self._class_names = {int(k): v for k, v in model.config.id2label.items()}
+        self._warn_unknown_filter_classes()
+
+    def unknown_filter_classes(self) -> tuple[str, ...]:
+        """``class_filter`` names this model never emits (empty until it loads)."""
+        if not self._class_names:
+            return ()
+        known = set(self._class_names.values())
+        return tuple(name for name in self.class_filter if name not in known)
+
+    def _warn_unknown_filter_classes(self) -> None:
+        """Log filter names that cannot match, which would otherwise draw nothing."""
+        unknown = self.unknown_filter_classes()
+        if unknown:
+            logger.warning(
+                "class_filter names not emitted by %s: %s. Nothing will be detected for them; "
+                "the model's classes are %s.",
+                self.model_name,
+                ", ".join(unknown),
+                ", ".join(sorted(set(self._class_names.values()))),
+            )
 
     def _infer(self, images: list[np.ndarray]) -> list[list[DetectedObject]]:
         import torch

{videopython-0.55.1 → videopython-0.55.3}/src/videopython/audio/audio.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import io
 import subprocess
+import tempfile
 import wave
 from dataclasses import dataclass
 from pathlib import Path
@@ -16,6 +17,17 @@ from videopython.base.exceptions import AudioLoadError, FFmpegProbeError
 if TYPE_CHECKING:
     from videopython.audio.analysis import AudioLevels, AudioSegment, AudioSegmentType, SilentSegment
 
+# What `Audio.from_path` asks ffmpeg for. Piped WAV comes back as pcm_s16le whatever
+# the source's bit depth, so requesting raw s16le loses no fidelity that the WAV path
+# was preserving, and there is no header to parse back.
+_PCM_FORMAT = "s16le"
+_PCM_DTYPE = np.int16
+_PCM_SAMPLE_WIDTH = 2
+
+# Read size when draining ffmpeg's stdout: big enough that a multi-GB decode is not
+# millions of round trips, small enough to be irrelevant for a short clip.
+_DECODE_CHUNK_BYTES = 8 << 20
+
 
 def atempo_chain(speed: float) -> list[str]:
     """Build the ``atempo`` filter chain that time-stretches audio by ``speed``.
@@ -163,100 +175,131 @@ class Audio:
         return cls(data, metadata)
 
     @classmethod
-    def from_path(cls, file_path: str | Path) -> Audio:
+    def from_path(
+        cls,
+        file_path: str | Path,
+        *,
+        sample_rate: int | None = None,
+        channels: int | None = None,
+    ) -> Audio:
         """
-        Load audio from a file using ffmpeg
+        Load audio from a file using ffmpeg.
+
+        ``sample_rate`` and ``channels`` ask ffmpeg to convert *while decoding*
+        rather than loading the source in full and converting afterwards. For a
+        caller that only wants 16kHz mono -- speech recognition, diarization,
+        speaker embeddings -- that is the difference between holding the whole
+        source in memory and holding a twelfth of it: a 12-hour 48kHz stereo
+        recording is 16.5GB of float32 at source rate and 1.4GB at 16kHz mono.
+        Resampling uses soxr, the engine :meth:`resample` uses, so the result
+        tracks ``Audio.from_path(p).to_mono().resample(r)`` sample for sample at
+        an error RMS around one 16-bit LSB -- the two quantize at different points
+        in the chain, and neither is the more faithful for it.
 
         Args:
             file_path: Path to the audio file
+            sample_rate: Decode at this rate instead of the source's.
+            channels: Decode to this many channels instead of the source's.
+                ``1`` downmixes to mono.
 
         Returns:
             Audio: New Audio instance
 
         Raises:
             FileNotFoundError: If the file doesn't exist
+            ValueError: If ``sample_rate`` or ``channels`` is not positive
             AudioLoadError: If there's an error loading the audio
         """
         file_path = Path(file_path)
         if not file_path.exists():
             raise FileNotFoundError(f"File not found: {file_path}")
+        if sample_rate is not None and sample_rate <= 0:
+            raise ValueError("Sample rate must be positive")
+        if channels is not None and channels <= 0:
+            raise ValueError("Channel count must be positive")
 
-        # Get audio info
         info = cls._get_ffmpeg_info(file_path)
-
-        # Convert to WAV using ffmpeg
+        target_rate = info["sample_rate"] if sample_rate is None else sample_rate
+        target_channels = info["channels"] if channels is None else channels
+
+        # Raw PCM rather than a WAV round-trip. Piped WAV comes back as pcm_s16le
+        # whatever the source's bit depth -- `-bits_per_raw_sample` is a hint the
+        # WAV muxer does not act on -- and its header carries a placeholder length
+        # because a pipe is not seekable. So parsing it back told us only what we
+        # had already asked for, and cost two more full copies of the audio on the
+        # way: one for `BytesIO`, one for `readframes`.
         cmd = [
             "ffmpeg",
+            "-v",
+            "error",
             "-i",
             str(file_path),
             "-f",
-            "wav",
+            _PCM_FORMAT,
             "-ar",
-            str(info["sample_rate"]),  # sample rate
+            str(target_rate),
             "-ac",
-            str(info["channels"]),  # channels
-            "-bits_per_raw_sample",
-            str(info["bit_depth"]),
+            str(target_channels),
+            # soxr rather than ffmpeg's default resampler, so that decoding at a
+            # rate and resampling to it afterwards agree.
+            "-af",
+            "aresample=resampler=soxr",
             "-",  # Output to stdout
         ]
 
         try:
-            process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-            wav_data, stderr = process.communicate()
+            # stderr to a file, not a pipe: stdout is drained to completion before
+            # stderr is read, and a full stderr pipe would deadlock ffmpeg partway
+            # through the audio.
+            with tempfile.TemporaryFile() as errors:
+                process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=errors)
+                assert process.stdout is not None
+                # A bytearray rather than `communicate()`, which accumulates chunks
+                # in a list and joins them at the end -- holding the whole of a long
+                # decode twice at the moment it completes.
+                raw = bytearray()
+                try:
+                    while chunk := process.stdout.read(_DECODE_CHUNK_BYTES):
+                        raw += chunk
+                finally:
+                    process.stdout.close()
+                if process.wait() != 0:
+                    errors.seek(0)
+                    raise AudioLoadError(f"FFmpeg error: {errors.read().decode(errors='replace')}")
+        except subprocess.CalledProcessError as e:
+            raise AudioLoadError(f"Error running ffmpeg: {e}")
 
-            if process.returncode != 0:
-                raise AudioLoadError(f"FFmpeg error: {stderr.decode()}")
-
-            # Read WAV data
-            with io.BytesIO(wav_data) as wav_io:
-                with wave.open(wav_io, "rb") as wav_file:
-                    # Get WAV metadata
-                    sample_width = wav_file.getsampwidth()
-                    channels = wav_file.getnchannels()
-                    sample_rate = wav_file.getframerate()
-                    n_frames = wav_file.getnframes()
-
-                    # Read raw audio data
-                    raw_data = wav_file.readframes(n_frames)
-
-                    # Convert bytes to numpy array based on sample width
-                    dtype_map = {1: np.int8, 2: np.int16, 4: np.int32}
-                    dtype = dtype_map.get(sample_width)
-                    if dtype is None:
-                        raise AudioLoadError(f"Unsupported sample width: {sample_width}")
-
-                    # Explicitly annotated: numpy>=2.5 shape-types ndarray, so the
-                    # 1-D frombuffer result cannot be rebound to a 2-D view below.
-                    data: np.ndarray[Any, np.dtype[np.float32]]
-                    data = np.frombuffer(raw_data, dtype=dtype).astype(np.float32)
-
-                    # Reshape to (frames, channels) if stereo
-                    if channels == 2:
-                        data = data.reshape(-1, 2)
-
-                    # Normalize to float between -1 and 1
-                    max_value = float(np.iinfo(dtype).max)  # type: ignore
-                    data = data / max_value
-
-                    # Ensure normalization is within bounds due to floating point precision
-                    data = np.clip(data, -1.0, 1.0)
-
-                    # Calculate frame count from actual data length
-                    # For stereo, len(data) is already correct after reshape
-                    frame_count = len(data)
-
-                    metadata = AudioMetadata(
-                        sample_rate=sample_rate,
-                        channels=channels,
-                        sample_width=sample_width,
-                        duration_seconds=info["duration"],
-                        frame_count=frame_count,
-                    )
+        # A truncated final frame would otherwise make `frombuffer` raise on a file
+        # that is entirely usable up to that point.
+        frame_bytes = _PCM_SAMPLE_WIDTH * target_channels
+        usable = len(raw) - (len(raw) % frame_bytes)
 
-                    return cls(data, metadata)
+        # Explicitly annotated: numpy>=2.5 shape-types ndarray, so the
+        # 1-D frombuffer result cannot be rebound to a 2-D view below.
+        data: np.ndarray[Any, np.dtype[np.float32]]
+        data = np.frombuffer(memoryview(raw)[:usable], dtype=_PCM_DTYPE).astype(np.float32)
 
-        except subprocess.CalledProcessError as e:
-            raise AudioLoadError(f"Error running ffmpeg: {e}")
+        # Reshape to (frames, channels) if stereo
+        if target_channels == 2:
+            data = data.reshape(-1, 2)
+
+        # Normalize to float between -1 and 1, and clamp for floating-point
+        # precision. Both in place: at these sizes a copy per step is most of what
+        # makes decoding a long file expensive.
+        data /= float(np.iinfo(_PCM_DTYPE).max)
+        np.clip(data, -1.0, 1.0, out=data)
+
+        # Calculate frame count from actual data length
+        # For stereo, len(data) is already correct after reshape
+        metadata = AudioMetadata(
+            sample_rate=target_rate,
+            channels=target_channels,
+            sample_width=_PCM_SAMPLE_WIDTH,
+            duration_seconds=info["duration"],
+            frame_count=len(data),
+        )
+
+        return cls(data, metadata)
 
     @classmethod
     def from_file(cls, file_path: str | Path) -> Audio:
@@ -653,8 +696,6 @@ class Audio:
         filter_str = ",".join(filters) if filters else "anull"
 
         # Save current audio to temp WAV, process with ffmpeg, read back
-        import tempfile
-
         with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as input_file:
             input_path = input_file.name