vision-agents-plugins-local 0.4.4__tar.gz

vision_agents_plugins_local-0.4.4/.gitignore

@@ -0,0 +1,98 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.cursor/*
+# Distribution / packaging
+.Python
+build/
+dist/
+downloads/
+develop-eggs/
+eggs/
+.eggs/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+pip-wheel-metadata/
+MANIFEST
+*.egg-info/
+*.egg
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+coverage.xml
+nosetests.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Type checker / lint caches
+.mypy_cache/
+.dmypy.json
+dmypy.json
+.pytype/
+.pyre/
+.ruff_cache/
+
+# Environments
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+.env
+.env.local
+.env.*.local
+.env.bak
+pyvenv.cfg
+.python-version
+
+# Editors / IDEs
+.vscode/
+.idea/
+
+# Jupyter Notebook
+.ipynb_checkpoints/
+
+# OS / Misc
+.DS_Store
+*.log
+
+# Tooling & repo-specific
+pyrightconfig.json
+shell.nix
+bin/*
+lib/*
+stream-py/
+
+# Example lock files (regenerated by uv sync)
+examples/*/uv.lock
+plugins/*/example/uv.lock
+
+# Artifacts / assets
+*.pt
+*.kef
+*.onnx
+profile.html
+
+/opencode.json
+.ralph-tui/
+.claude/
+
+.uv-cache/

vision_agents_plugins_local-0.4.4/PKG-INFO

@@ -0,0 +1,14 @@
+Metadata-Version: 2.4
+Name: vision-agents-plugins-local
+Version: 0.4.4
+Summary: Local audio & video integration for Vision Agents
+Project-URL: Documentation, https://visionagents.ai/
+Project-URL: Website, https://visionagents.ai/
+Project-URL: Source, https://github.com/GetStream/Vision-Agents
+License-Expression: MIT
+Keywords: AI,agents,local,voice agents
+Requires-Python: >=3.10
+Requires-Dist: aiortc<1.15.0,>=1.14.0
+Requires-Dist: av<17,>=14.2.0
+Requires-Dist: sounddevice>=0.5.0
+Requires-Dist: vision-agents

vision_agents_plugins_local-0.4.4/pyproject.toml

@@ -0,0 +1,42 @@
+[build-system]
+requires = ["hatchling", "hatch-vcs"]
+build-backend = "hatchling.build"
+
+[project]
+name = "vision-agents-plugins-local"
+dynamic = ["version"]
+description = "Local audio & video integration for Vision Agents"
+readme = "README.md"
+keywords = ["local", "AI", "voice agents", "agents"]
+requires-python = ">=3.10"
+license = "MIT"
+dependencies = [
+    "vision-agents",
+    "sounddevice>=0.5.0",
+    "aiortc>=1.14.0, <1.15.0",
+    "av>=14.2.0, <17",
+]
+
+[project.urls]
+Documentation = "https://visionagents.ai/"
+Website = "https://visionagents.ai/"
+Source = "https://github.com/GetStream/Vision-Agents"
+
+[tool.hatch.version]
+source = "vcs"
+raw-options = { root = "..", search_parent_directories = true, fallback_version = "0.0.0" }
+
+[tool.hatch.build.targets.wheel]
+packages = ["."]
+
+[tool.hatch.build.targets.sdist]
+include = ["/vision_agents"]
+
+[tool.uv.sources]
+vision-agents = { workspace = true }
+
+[dependency-groups]
+dev = [
+    "pytest>=8.4.1",
+    "pytest-asyncio>=1.0.0",
+]

vision_agents_plugins_local-0.4.4/vision_agents/plugins/local/__init__.py

@@ -0,0 +1,10 @@
+from .devices import AudioInputDevice, AudioOutputDevice, CameraDevice
+from .edge import LocalCall, LocalEdge
+
+__all__ = [
+    "AudioInputDevice",
+    "AudioOutputDevice",
+    "CameraDevice",
+    "LocalCall",
+    "LocalEdge",
+]

vision_agents_plugins_local-0.4.4/vision_agents/plugins/local/devices.py

@@ -0,0 +1,370 @@
+"""
+Device enumeration and selection utilities for LocalTransport.
+
+Provides typed device representations and interactive prompts for
+selecting audio and video devices when running agents locally.
+"""
+
+import glob
+import logging
+import platform
+import queue
+import re
+import subprocess
+from dataclasses import dataclass
+
+import numpy as np
+import sounddevice as sd
+
+from .utils import prompt_selection
+
+logger = logging.getLogger(__name__)
+
+
+_AVFOUNDATION_RE = re.compile(r"\[AVFoundation.*?\]\s*\[(\d+)\]\s*(.+)")
+_DSHOW_DEVICE_RE = re.compile(r'"(.+?)"')
+
+
+class AudioInputDevice:
+    """Audio input device (microphone).
+
+    Combines device metadata with stream capture. Subclass to implement
+    custom audio backends (e.g. GStreamer).
+    """
+
+    def __init__(
+        self,
+        index: int,
+        name: str,
+        sample_rate: int = 48000,
+        channels: int = 1,
+        is_default: bool = False,
+        blocksize: int | None = None,
+    ):
+        self.index = index
+        self.name = name
+        self._sample_rate = sample_rate
+        self._channels = channels
+        self.is_default = is_default
+        self._blocksize = (
+            blocksize if blocksize is not None else int(sample_rate * 0.02)
+        )
+        self._stream: sd.InputStream | None = None
+        self._buffer: queue.Queue[np.ndarray] = queue.Queue(maxsize=200)
+
+    @property
+    def sample_rate(self) -> int:
+        return self._sample_rate
+
+    @property
+    def channels(self) -> int:
+        return self._channels
+
+    def _callback(
+        self,
+        indata: np.ndarray,
+        frames: int,
+        time_info: object,
+        status: object,
+    ) -> None:
+        if status:
+            logger.warning("Audio input status: %s", status)
+        try:
+            self._buffer.put_nowait(indata.copy())
+        except queue.Full:
+            pass
+
+    def start(self) -> None:
+        """Open and start the audio input stream."""
+        self._stream = sd.InputStream(
+            samplerate=self._sample_rate,
+            channels=self._channels,
+            dtype="int16",
+            blocksize=self._blocksize,
+            device=self.index,
+            callback=self._callback,
+        )
+        self._stream.start()
+        logger.info(
+            "Started audio input: %dHz, %d channels",
+            self._sample_rate,
+            self._channels,
+        )
+
+    def read(self) -> np.ndarray | None:
+        """Block until audio data is available (up to 100ms timeout)."""
+        try:
+            return self._buffer.get(timeout=0.1)
+        except queue.Empty:
+            return None
+
+    def stop(self) -> None:
+        """Stop and close the audio input stream."""
+        if self._stream is not None:
+            self._stream.stop()
+            self._stream.close()
+            self._stream = None
+            logger.info("Stopped audio input")
+
+
+class AudioOutputDevice:
+    """Audio output device (speaker/headphones).
+
+    Combines device metadata with stream playback. Subclass to implement
+    custom audio backends (e.g. GStreamer).
+    """
+
+    def __init__(
+        self,
+        index: int,
+        name: str,
+        sample_rate: int = 48000,
+        channels: int = 2,
+        is_default: bool = False,
+        blocksize: int = 2048,
+    ):
+        self.index = index
+        self.name = name
+        self._sample_rate = sample_rate
+        self._channels = channels
+        self.is_default = is_default
+        self._blocksize = blocksize
+        self._stream: sd.OutputStream | None = None
+
+    @property
+    def sample_rate(self) -> int:
+        return self._sample_rate
+
+    @property
+    def channels(self) -> int:
+        return self._channels
+
+    def start(self) -> None:
+        """Open and start the audio output stream."""
+        self._stream = sd.OutputStream(
+            samplerate=self._sample_rate,
+            channels=self._channels,
+            dtype="int16",
+            blocksize=self._blocksize,
+            device=self.index,
+        )
+        self._stream.start()
+        logger.info(
+            "Started audio output: %dHz, %d channels",
+            self._sample_rate,
+            self._channels,
+        )
+
+    def write(self, samples: np.ndarray) -> None:
+        """Write flat int16 samples to the device."""
+        if self._stream is None:
+            return
+        frames = len(samples) // self._channels
+        audio = samples.reshape(frames, self._channels)
+        self._stream.write(audio)
+
+    def flush(self) -> None:
+        """Abort current playback and restart the stream."""
+        if self._stream is not None:
+            self._stream.abort()
+            self._stream.start()
+
+    def stop(self) -> None:
+        """Stop and close the audio output stream."""
+        if self._stream is not None:
+            self._stream.stop()
+            self._stream.close()
+            self._stream = None
+            logger.info("Stopped audio output")
+
+
+@dataclass(frozen=True)
+class CameraDevice:
+    """A detected camera."""
+
+    index: int
+    name: str
+    device: str
+
+
+def list_audio_input_devices() -> list[AudioInputDevice]:
+    """Return all audio input devices."""
+    raw = sd.query_devices()
+    default_in = sd.default.device[0]
+    return [
+        AudioInputDevice(
+            index=i,
+            name=dev["name"],
+            sample_rate=int(dev["default_samplerate"]),
+            channels=dev["max_input_channels"],
+            is_default=(i == default_in),
+        )
+        for i, dev in enumerate(raw)
+        if dev["max_input_channels"] > 0
+    ]
+
+
+def list_audio_output_devices() -> list[AudioOutputDevice]:
+    """Return all audio output devices."""
+    raw = sd.query_devices()
+    default_out = sd.default.device[1]
+    return [
+        AudioOutputDevice(
+            index=i,
+            name=dev["name"],
+            sample_rate=int(dev["default_samplerate"]),
+            channels=dev["max_output_channels"],
+            is_default=(i == default_out),
+        )
+        for i, dev in enumerate(raw)
+        if dev["max_output_channels"] > 0
+    ]
+
+
+def select_audio_input_device() -> AudioInputDevice | None:
+    """Interactive prompt to select an audio input device."""
+    devices = list_audio_input_devices()
+    default = next((d for d in devices if d.is_default), None)
+    return prompt_selection(
+        items=devices,
+        formatter=_format_audio_device,
+        header="INPUT DEVICES (Microphones)",
+        default=default,
+    )
+
+
+def select_audio_output_device() -> AudioOutputDevice | None:
+    """Interactive prompt to select an audio output device."""
+    devices = list_audio_output_devices()
+    default = next((d for d in devices if d.is_default), None)
+    return prompt_selection(
+        items=devices,
+        formatter=_format_audio_device,
+        header="OUTPUT DEVICES (Speakers)",
+        default=default,
+    )
+
+
+def select_video_device() -> CameraDevice | None:
+    """Interactive prompt to select a camera or skip.
+
+    Returns:
+        The selected camera device, or None if skipped.
+    """
+    cameras = list_cameras()
+
+    return prompt_selection(
+        items=cameras,
+        formatter=lambda c: c.name,
+        header="VIDEO DEVICES (Cameras)",
+        allow_skip=True,
+        empty_message="No cameras detected\n  (Camera support requires ffmpeg to be installed)",
+    )
+
+
+def list_cameras() -> list[CameraDevice]:
+    """List available cameras on the system."""
+    system = platform.system()
+
+    if system == "Darwin":
+        return _list_cameras_darwin()
+    if system == "Linux":
+        return _list_cameras_linux()
+    if system == "Windows":
+        return _list_cameras_windows()
+
+    return []
+
+
+def _format_audio_device(dev: AudioInputDevice | AudioOutputDevice) -> str:
+    """Format an audio device for display."""
+    default = " [DEFAULT]" if dev.is_default else ""
+    return f"{dev.name} ({dev.sample_rate}Hz){default}"
+
+
+def _list_cameras_darwin() -> list[CameraDevice]:
+    """List cameras on macOS via ffmpeg/AVFoundation."""
+    try:
+        result = subprocess.run(
+            ["ffmpeg", "-f", "avfoundation", "-list_devices", "true", "-i", ""],
+            capture_output=True,
+            text=True,
+            timeout=5,
+        )
+    except (subprocess.TimeoutExpired, FileNotFoundError):
+        logger.warning("Failed to list cameras (is ffmpeg installed?)")
+        return []
+
+    cameras: list[CameraDevice] = []
+    in_video_section = False
+
+    for line in result.stderr.splitlines():
+        if "AVFoundation video devices:" in line:
+            in_video_section = True
+            continue
+        if "AVFoundation audio devices:" in line:
+            break
+        if in_video_section:
+            match = _AVFOUNDATION_RE.search(line)
+            if match:
+                cam_idx = int(match.group(1))
+                cameras.append(
+                    CameraDevice(
+                        index=cam_idx, name=match.group(2), device=str(cam_idx)
+                    )
+                )
+
+    return cameras
+
+
+def _list_cameras_linux() -> list[CameraDevice]:
+    """List cameras on Linux via /dev/video* and sysfs."""
+    cameras: list[CameraDevice] = []
+
+    for i, dev_path in enumerate(sorted(glob.glob("/dev/video*"))):
+        name_path = f"/sys/class/video4linux/{dev_path.split('/')[-1]}/name"
+        try:
+            with open(name_path) as f:
+                name = f.read().strip()
+        except OSError:
+            name = dev_path
+        cameras.append(CameraDevice(index=i, name=name, device=dev_path))
+
+    return cameras
+
+
+def _list_cameras_windows() -> list[CameraDevice]:
+    """List cameras on Windows via ffmpeg/DirectShow."""
+    try:
+        result = subprocess.run(
+            ["ffmpeg", "-f", "dshow", "-list_devices", "true", "-i", "dummy"],
+            capture_output=True,
+            text=True,
+            timeout=5,
+        )
+    except (subprocess.TimeoutExpired, FileNotFoundError):
+        logger.warning("Failed to list cameras (is ffmpeg installed?)")
+        return []
+
+    cameras: list[CameraDevice] = []
+    in_video_section = False
+
+    for line in result.stderr.splitlines():
+        if "DirectShow video devices" in line:
+            in_video_section = True
+            continue
+        if "DirectShow audio devices" in line:
+            break
+        if in_video_section:
+            match = _DSHOW_DEVICE_RE.search(line)
+            if match:
+                name = match.group(1)
+                cameras.append(
+                    CameraDevice(
+                        index=len(cameras),
+                        name=name,
+                        device=f'video="{name}"',
+                    )
+                )
+
+    return cameras

vision_agents_plugins_local-0.4.4/vision_agents/plugins/local/display.py

@@ -0,0 +1,181 @@
+"""Tkinter-based video display for LocalEdge.
+
+Shows the agent's outbound video track in a tkinter window.
+Gracefully handles environments where tkinter is not available.
+"""
+
+import asyncio
+import contextlib
+import logging
+import signal
+import threading
+import warnings
+from typing import cast
+
+import av
+from aiortc import MediaStreamError, MediaStreamTrack
+
+logger = logging.getLogger(__name__)
+
+try:
+    import tkinter
+
+    _TKINTER_AVAILABLE = True
+except ImportError:
+    _TKINTER_AVAILABLE = False
+
+
+def _fit_size(src_w: int, src_h: int, dst_w: int, dst_h: int) -> tuple[int, int]:
+    """Compute the largest size that fits dst while preserving src aspect ratio."""
+    scale = min(dst_w / src_w, dst_h / src_h)
+    return max(2, int(src_w * scale)) & ~1, max(2, int(src_h * scale)) & ~1
+
+
+def _frame_to_ppm(frame: av.VideoFrame, width: int, height: int) -> bytes:
+    """Convert an av.VideoFrame to PPM bytes, scaled to fit width x height."""
+    fit_w, fit_h = _fit_size(frame.width, frame.height, width, height)
+    rgb = frame.reformat(width=fit_w, height=fit_h, format="rgb24")
+    pixels = rgb.to_ndarray()
+    header = f"P6 {fit_w} {fit_h} 255 ".encode()
+    return header + pixels.tobytes()
+
+
+class VideoDisplay:
+    """Displays video frames from a MediaStreamTrack in a tkinter window.
+
+    Tkinter events are pumped from the asyncio event loop (main thread)
+    to satisfy macOS Cocoa requirements. A thread-safe queue bridges the
+    async frame receiver and the display update loop.
+    """
+
+    def __init__(
+        self,
+        title: str = "Agent Video Output",
+        width: int = 640,
+        height: int = 480,
+        fps: int = 30,
+    ):
+        if fps <= 0:
+            raise ValueError("fps must be > 0")
+        if width <= 0:
+            raise ValueError("width must be > 0")
+        if height <= 0:
+            raise ValueError("height must be > 0")
+        self._title = title
+        self._width = width
+        self._height = height
+        self._frame_interval = 1.0 / fps
+        self._running = False
+        self._latest_frame: av.VideoFrame | None = None
+        self._frame_lock = threading.Lock()
+        self._recv_task: asyncio.Task[None] | None = None
+        self._tk_task: asyncio.Task[None] | None = None
+        self._root: tkinter.Tk | None = None
+
+    async def start(self, video_track: MediaStreamTrack) -> None:
+        """Start displaying frames from the given video track.
+
+        If tkinter is not available, emits an ImportWarning and returns
+        without starting.
+        """
+        if not _TKINTER_AVAILABLE:
+            warnings.warn(
+                "tkinter is not available. Install python3-tk or equivalent "
+                "for your platform to use the video display.",
+                ImportWarning,
+            )
+            return
+
+        self._running = True
+        self._recv_task = asyncio.create_task(self._recv_loop(video_track))
+        self._tk_task = asyncio.create_task(self._tk_loop())
+
+    async def stop(self) -> None:
+        """Stop the display and clean up tasks."""
+        self._running = False
+
+        for task in (self._recv_task, self._tk_task):
+            if task is not None:
+                task.cancel()
+                try:
+                    await task
+                except asyncio.CancelledError:
+                    pass
+
+        self._recv_task = None
+        self._tk_task = None
+
+    async def _recv_loop(self, track: MediaStreamTrack) -> None:
+        """Consume frames from the video track and store the latest."""
+        try:
+            while self._running:
+                frame = cast(av.VideoFrame, await track.recv())
+                with self._frame_lock:
+                    self._latest_frame = frame
+        except asyncio.CancelledError:
+            raise
+        except MediaStreamError:
+            logger.debug("Video track ended")
+        except RuntimeError:
+            logger.debug("Video track stopped")
+
+    async def _tk_loop(self) -> None:
+        """Pump Tkinter events from the asyncio event loop (main thread)."""
+        root: tkinter.Tk | None = None
+        prev_sigint = signal.getsignal(signal.SIGINT)
+        try:
+            root = tkinter.Tk()
+            # Tk() overrides SIGINT — restore the previous handler (typically
+            # asyncio's) so the first Ctrl+C gracefully cancels the main task
+            # instead of raising KeyboardInterrupt inside this task.
+            signal.signal(signal.SIGINT, prev_sigint)
+            root.title(self._title)
+            root.geometry(f"{self._width}x{self._height}")
+            root.protocol("WM_DELETE_WINDOW", self._on_window_close)
+            self._root = root
+
+            gray = bytes([128] * (self._width * self._height * 3))
+            header = f"P6 {self._width} {self._height} 255 ".encode()
+            self._photo = tkinter.PhotoImage(data=header + gray)
+
+            self._label = tkinter.Label(root, image=self._photo)
+            self._label.pack(fill="both", expand=True)
+
+            while self._running:
+                with self._frame_lock:
+                    frame = self._latest_frame
+                    self._latest_frame = None
+
+                if frame is not None:
+                    ppm = await asyncio.to_thread(
+                        _frame_to_ppm,
+                        frame,
+                        self._width,
+                        self._height,
+                    )
+                    self._photo = tkinter.PhotoImage(data=ppm)
+                    self._label.configure(image=self._photo)
+
+                try:
+                    root.update()
+                except (tkinter.TclError, KeyboardInterrupt):
+                    break
+
+                await asyncio.sleep(self._frame_interval)
+        except asyncio.CancelledError:
+            raise
+        finally:
+            if root is not None:
+                with contextlib.suppress(tkinter.TclError, KeyboardInterrupt):
+                    root.destroy()
+            self._root = None
+
+    def _on_window_close(self) -> None:
+        """Handle the user closing the tkinter window."""
+        self._running = False
+        if self._root is not None:
+            try:
+                self._root.destroy()
+            except tkinter.TclError:
+                pass
+            self._root = None

vision_agents_plugins_local-0.4.4/vision_agents/plugins/local/edge.py

@@ -0,0 +1,306 @@
+import asyncio
+import logging
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any, cast
+
+import aiortc
+import av
+import numpy as np
+from getstream.video.rtc.track_util import AudioFormat, PcmData
+from vision_agents.core.agents.conversation import InMemoryConversation
+from vision_agents.core.edge.edge_transport import EdgeTransport
+from vision_agents.core.edge.events import AudioReceivedEvent, TrackAddedEvent
+from vision_agents.core.edge.types import Connection, Participant, TrackType, User
+from vision_agents.core.utils.utils import cancel_and_wait
+from vision_agents.core.utils.video_track import QueuedVideoTrack
+
+from .devices import AudioInputDevice, AudioOutputDevice, CameraDevice
+from .display import VideoDisplay
+from .tracks import LocalOutputAudioTrack, LocalVideoTrack
+
+if TYPE_CHECKING:
+    from vision_agents.core.agents.agents import Agent
+
+PLUGIN_NAME = "local"
+LOCAL_VIDEO_TRACK_ID = "local-video-track"
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class LocalCall:
+    """Minimal Call-compatible object for local transport."""
+
+    id: str
+
+
+class LocalEdge(EdgeTransport):
+    """EdgeTransport implementation for local audio/video I/O.
+
+    Captures microphone audio via AudioInputDevice and plays agent audio
+    through AudioOutputDevice (both default to sounddevice). Optionally
+    captures camera video via CameraDevice and displays agent video output
+    in a tkinter window. Subclass the device classes to use alternative
+    backends (e.g. GStreamer).
+    """
+
+    def __init__(
+        self,
+        audio_input: AudioInputDevice,
+        audio_output: AudioOutputDevice,
+        video_input: CameraDevice | None = None,
+        video_width: int = 640,
+        video_height: int = 480,
+        video_fps: int = 30,
+    ):
+        """Create a local edge transport.
+
+        Args:
+            audio_input: Microphone device for capturing user audio.
+            audio_output: Speaker device for playing agent audio.
+            video_input: Camera device for capturing user video. None disables video.
+            video_width: Width of the video frame in pixels.
+            video_height: Height of the video frame in pixels.
+            video_fps: Video frame rate.
+        """
+        super().__init__()
+
+        self._audio_input = audio_input
+        self._audio_output = audio_output
+
+        self._video_input = video_input.device if video_input else None
+        self._video_width = video_width
+        self._video_height = video_height
+        self._video_fps = video_fps
+
+        self._participant = Participant(
+            original=None,
+            user_id="local",
+            id="local",
+        )
+
+        self._mic_task: asyncio.Task[None] | None = None
+        self._video_forward_task: asyncio.Task[None] | None = None
+        self._audio_track: LocalOutputAudioTrack | None = None
+        self._input_video_track: LocalVideoTrack | None = None
+        self._output_video_track = QueuedVideoTrack(
+            width=video_width,
+            height=video_height,
+            fps=video_fps,
+        )
+        self._video_display: VideoDisplay | None = None
+        self._connection: LocalConnection | None = None
+
+    async def publish_tracks(
+        self,
+        audio_track: aiortc.MediaStreamTrack | None,
+        video_track: aiortc.MediaStreamTrack | None,
+    ) -> None:
+        """Publish the agent's media tracks locally."""
+        if audio_track is not None and isinstance(audio_track, LocalOutputAudioTrack):
+            audio_track.start()
+            logger.info("Audio track published and started")
+
+        if video_track is not None:
+            self._video_forward_task = asyncio.create_task(
+                self._forward_video(video_track)
+            )
+            logger.info("Video output track published")
+
+    def create_audio_track(self) -> "LocalOutputAudioTrack":
+        """Create an audio track that plays through the audio output backend."""
+        self._audio_track = LocalOutputAudioTrack(
+            audio_output=self._audio_output,
+        )
+        return self._audio_track
+
+    def create_video_track(self) -> LocalVideoTrack | None:
+        """Create a video track for the agent's camera input."""
+        if self._video_input is None:
+            logger.debug("No video device configured, skipping video track creation")
+            return None
+
+        self._input_video_track = LocalVideoTrack(
+            device=self._video_input,
+            width=self._video_width,
+            height=self._video_height,
+            fps=self._video_fps,
+        )
+        return self._input_video_track
+
+    def add_track_subscriber(self, track_id: str) -> LocalVideoTrack | None:
+        """Return the local camera video track if available."""
+        if track_id == LOCAL_VIDEO_TRACK_ID and self._input_video_track is not None:
+            return self._input_video_track
+        return None
+
+    async def join(
+        self, agent: "Agent", call: Any = None, **kwargs: Any
+    ) -> "LocalConnection":
+        """Start microphone capture and optionally camera."""
+        await self._start_audio()
+
+        if self._video_input is not None:
+            video_track = self.create_video_track()
+            if video_track is not None:
+                self.events.send(
+                    TrackAddedEvent(
+                        plugin_name=PLUGIN_NAME,
+                        track_id=LOCAL_VIDEO_TRACK_ID,
+                        track_type=TrackType.VIDEO,
+                        participant=self._participant,
+                    )
+                )
+                logger.info("Camera video track added")
+
+        self._connection = LocalConnection(self)
+        return self._connection
+
+    async def close(self) -> None:
+        """Stop audio/video and release all resources."""
+        if self._video_forward_task is not None:
+            await cancel_and_wait(self._video_forward_task)
+            self._video_forward_task = None
+
+        self._output_video_track.stop()
+
+        if self._video_display is not None:
+            await self._video_display.stop()
+            self._video_display = None
+
+        await self._stop_audio()
+        self._connection = None
+
+    async def authenticate(self, user: User) -> None:
+        # Local transport does not require any auth
+        return
+
+    def open_demo(self, *args: Any, **kwargs: Any) -> None: ...
+
+    async def open_demo_for_agent(
+        self, agent: "Agent", call_type: str, call_id: str
+    ) -> None:
+        """Open a tkinter window showing the agent's video output."""
+        if not agent.publish_video:
+            logger.info("Agent has no video output, skipping video display")
+            return
+
+        try:
+            self._video_display = VideoDisplay(
+                title="Agent Video Output",
+                width=self._video_width,
+                height=self._video_height,
+                fps=self._video_fps,
+            )
+            await self._video_display.start(self._output_video_track)
+            logger.info("Opened video display")
+        except RuntimeError:
+            logger.warning(
+                "Cannot open video display: tkinter is not available. "
+                "Install python3-tk or equivalent for your platform."
+            )
+
+    async def create_call(self, call_id: str, **kwargs: Any) -> LocalCall:
+        return LocalCall(id=call_id)
+
+    async def send_custom_event(self, data: dict[str, Any]) -> None:
+        raise NotImplementedError("LocalEdge does not support send_custom_event")
+
+    async def create_conversation(
+        self, call: Any, user: User, instructions: str
+    ) -> InMemoryConversation:
+        return InMemoryConversation(instructions=instructions, messages=[])
+
+    def _emit_audio_event(self, data: np.ndarray) -> None:
+        """Convert raw numpy audio to PcmData and emit AudioReceivedEvent."""
+        samples = data.flatten().astype(np.int16)
+        pcm = PcmData(
+            samples=samples,
+            sample_rate=self._audio_input.sample_rate,
+            format=AudioFormat.S16,
+            channels=self._audio_input.channels,
+        )
+        pcm.participant = self._participant
+
+        self.events.send(
+            AudioReceivedEvent(
+                plugin_name=PLUGIN_NAME,
+                pcm_data=pcm,
+                participant=self._participant,
+            )
+        )
+
+    async def _forward_video(self, source: aiortc.MediaStreamTrack) -> None:
+        """Read frames from source track and push them to the output track."""
+        try:
+            while True:
+                frame = cast(av.VideoFrame, await source.recv())
+                await self._output_video_track.add_frame(frame)
+        except asyncio.CancelledError:
+            raise
+        except aiortc.MediaStreamError:
+            logger.debug("Source video track ended")
+
+    async def _mic_loop(self) -> None:
+        """Read mic data via asyncio.to_thread and emit audio events."""
+        try:
+            while True:
+                data = await asyncio.to_thread(self._audio_input.read)
+                if data is not None:
+                    self._emit_audio_event(data)
+        except asyncio.CancelledError:
+            logger.debug("Mic loop cancelled")
+            raise
+
+    async def _start_audio(self) -> None:
+        """Start microphone capture via the audio input backend."""
+        if self._mic_task is not None:
+            return
+
+        self._audio_input.start()
+        logger.info(
+            "Started microphone: %dHz, %d channels",
+            self._audio_input.sample_rate,
+            self._audio_input.channels,
+        )
+        self._mic_task = asyncio.create_task(self._mic_loop())
+
+    async def _stop_audio(self) -> None:
+        """Stop all audio and video streams."""
+        if self._mic_task is not None:
+            self._mic_task.cancel()
+            try:
+                await self._mic_task
+            except asyncio.CancelledError:
+                pass
+            self._mic_task = None
+
+        self._audio_input.stop()
+        logger.info("Stopped microphone")
+
+        if self._audio_track is not None:
+            self._audio_track.stop()
+
+        if self._input_video_track is not None:
+            self._input_video_track.stop()
+            self._input_video_track = None
+
+
+class LocalConnection(Connection):
+    """Connection wrapper for local transport."""
+
+    def __init__(self, transport: "LocalEdge"):
+        super().__init__()
+        self._transport = transport
+
+    def idle_since(self) -> float:
+        """Local transport is never idle."""
+        return 0.0
+
+    async def wait_for_participant(self, timeout: float | None = None) -> None:
+        """Local user is always present, return immediately."""
+        return
+
+    async def close(self, timeout: float = 2.0) -> None:
+        """Close the local connection."""
+        await self._transport.close()

vision_agents_plugins_local-0.4.4/vision_agents/plugins/local/tracks.py

@@ -0,0 +1,263 @@
+"""
+LocalTransport: audio/video track implementations.
+
+Provides LocalOutputAudioTrack for speaker playback and LocalVideoTrack
+for camera capture, enabling vision agents to run locally without cloud
+edge infrastructure.
+"""
+
+import asyncio
+import logging
+import platform
+import threading
+import time
+from fractions import Fraction
+from typing import Any
+
+import av
+import numpy as np
+import sounddevice as sd
+from aiortc import AudioStreamTrack, VideoStreamTrack
+from getstream.video.rtc.track_util import PcmData
+
+from .devices import AudioOutputDevice
+
+logger = logging.getLogger(__name__)
+
+
+def _get_camera_input_format() -> str:
+    """Get the FFmpeg input format for the current platform."""
+    system = platform.system()
+    if system == "Darwin":
+        return "avfoundation"
+    elif system == "Linux":
+        return "v4l2"
+    elif system == "Windows":
+        return "dshow"
+    else:
+        raise RuntimeError(f"Unsupported platform for camera capture: {system}")
+
+
+class LocalOutputAudioTrack(AudioStreamTrack):
+    """Audio track that plays PcmData through an AudioOutputDevice.
+
+    Uses an asyncio.Queue for backpressure: when the queue is full,
+    ``write`` awaits until the playback task drains an item. The playback
+    task offloads blocking device writes via ``asyncio.to_thread``.
+
+    Extends AudioStreamTrack so it satisfies the MediaStreamTrack interface
+    required by EdgeTransport.publish_tracks. Since this is a write-only
+    (playback) track, recv() is not supported.
+    """
+
+    def __init__(self, audio_output: AudioOutputDevice, buffer_limit: int = 20):
+        super().__init__()
+        self._audio_output = audio_output
+        self._queue: asyncio.Queue[np.ndarray] = asyncio.Queue(maxsize=buffer_limit)
+        self._running = False
+        self._playback_task: asyncio.Task[None] | None = None
+        self._write_lock = asyncio.Lock()
+
+    async def recv(self) -> av.AudioFrame:
+        """Not supported — this is a write-only playback track."""
+        raise NotImplementedError(
+            "LocalOutputAudioTrack is a playback-only track; recv() is not supported"
+        )
+
+    def start(self) -> None:
+        """Start the audio output stream."""
+        if self._running:
+            return
+
+        self._audio_output.start()
+        self._running = True
+        self._playback_task = asyncio.create_task(self._playback_loop())
+
+    async def write(self, data: PcmData) -> None:
+        """Write PCM data to be played on the speaker."""
+        if not self._running:
+            return
+
+        async with self._write_lock:
+            samples = self._process_audio(data)
+            await self._queue.put(samples)
+
+    async def flush(self) -> None:
+        """Clear any pending audio data and abort OS-level playback."""
+        async with self._write_lock:
+            while not self._queue.empty():
+                try:
+                    self._queue.get_nowait()
+                except asyncio.QueueEmpty:
+                    break
+            self._audio_output.flush()
+
+    def stop(self) -> None:
+        """Stop the audio output stream."""
+        super().stop()
+        self._running = False
+
+        if self._playback_task is not None:
+            self._playback_task.cancel()
+            self._playback_task = None
+
+        while not self._queue.empty():
+            try:
+                self._queue.get_nowait()
+            except asyncio.QueueEmpty:
+                break
+
+        self._audio_output.stop()
+
+    async def _playback_loop(self) -> None:
+        """Async task that drains the queue into the AudioOutput backend."""
+        try:
+            while True:
+                data = await self._queue.get()
+                try:
+                    await asyncio.to_thread(self._audio_output.write, data)
+                except sd.PortAudioError as err:
+                    logger.debug("PortAudio playback error: %s", err)
+        except asyncio.CancelledError:
+            logger.debug("Playback loop cancelled")
+            raise
+        except ValueError:
+            logger.exception("Audio data processing error")
+        except OSError:
+            logger.exception("Audio playback device error")
+
+    def _process_audio(self, data: PcmData) -> np.ndarray:
+        """Resample and convert PcmData to flat int16 numpy for the backend."""
+        target_rate = self._audio_output.sample_rate
+        target_channels = self._audio_output.channels
+
+        if data.sample_rate != target_rate or data.channels != target_channels:
+            data = data.resample(target_rate, target_channels)
+
+        samples = data.to_int16().samples
+
+        if samples.ndim == 2:
+            samples = samples.T.flatten()
+
+        return samples
+
+
+class LocalVideoTrack(VideoStreamTrack):
+    """Video track that captures from local camera using PyAV."""
+
+    kind = "video"
+
+    def __init__(
+        self,
+        device: str,
+        width: int = 640,
+        height: int = 480,
+        fps: int = 30,
+    ):
+        super().__init__()
+
+        self._device = device
+        self._width = width
+        self._height = height
+        self._fps = fps
+        self._container: Any = None
+        self._stream: Any = None
+        self._started = False
+        self._stopped = False
+        self._frame_count = 0
+        self._start_time: float | None = None
+        self._lock = threading.Lock()
+
+    def _open_camera(self) -> None:
+        """Open the camera device with PyAV."""
+        input_format = _get_camera_input_format()
+        system = platform.system()
+
+        options: dict[str, str] = {
+            "framerate": str(self._fps),
+        }
+
+        if system == "Darwin":
+            device_path = self._device
+            options["video_size"] = f"{self._width}x{self._height}"
+            options["pixel_format"] = "uyvy422"
+        elif system == "Linux":
+            device_path = self._device
+            options["video_size"] = f"{self._width}x{self._height}"
+        elif system == "Windows":
+            device_path = self._device
+            options["video_size"] = f"{self._width}x{self._height}"
+        else:
+            raise RuntimeError(f"Unsupported platform: {system}")
+
+        self._container = av.open(
+            device_path,
+            format=input_format,
+            options=options,
+        )
+        self._stream = self._container.streams.video[0]
+        logger.info(
+            "Opened camera: %s (%dx%d @ %dfps)",
+            self._device,
+            self._width,
+            self._height,
+            self._fps,
+        )
+
+    def _read_frame(self, max_retries: int = 20, retry_timeout: float = 0.02) -> Any:
+        """Read a single frame from the camera (blocking)."""
+        if self._container is None:
+            return None
+
+        for attempt in range(max_retries):
+            try:
+                for packet in self._container.demux(self._stream):
+                    for frame in packet.decode():
+                        return frame
+            except BlockingIOError:
+                if attempt < max_retries - 1:
+                    time.sleep(retry_timeout)
+                    continue
+                logger.debug("Camera not ready after %d retries", max_retries)
+                return None
+            except OSError:
+                logger.warning("Error reading camera frame", exc_info=True)
+                return None
+        return None
+
+    async def recv(self) -> av.VideoFrame:
+        """Receive the next video frame."""
+        if self._stopped:
+            raise RuntimeError("Track has been stopped")
+
+        if not self._started:
+            self._started = True
+            self._start_time = time.time()
+
+            await asyncio.to_thread(self._open_camera)
+
+        frame = await asyncio.to_thread(self._read_frame)
+
+        if frame is None:
+            frame = av.VideoFrame(
+                width=self._width, height=self._height, format="rgb24"
+            )
+            frame.planes[0].update(bytes(self._width * self._height * 3))
+
+        self._frame_count += 1
+        frame.pts = self._frame_count
+        frame.time_base = Fraction(1, self._fps)
+        return frame
+
+    def stop(self) -> None:
+        """Stop camera capture and release resources."""
+        with self._lock:
+            self._stopped = True
+            if self._container is not None:
+                try:
+                    self._container.close()
+                except OSError:
+                    logger.warning("Error closing camera")
+                self._container = None
+                self._stream = None
+            logger.info("Stopped camera capture")

vision_agents_plugins_local-0.4.4/vision_agents/plugins/local/utils.py

@@ -0,0 +1,86 @@
+import sys
+from typing import Callable, TypeVar
+
+if sys.platform != "win32":
+    import termios
+
+    def safe_input(prompt: str) -> str:
+        """Call input() after ensuring the terminal translates CR to NL.
+
+        PortAudio (via sounddevice) can disable the ICRNL terminal flag,
+        which causes Enter (CR) to show as ^M instead of submitting input.
+        """
+        if sys.stdin.isatty():
+            fd = sys.stdin.fileno()
+            attrs = termios.tcgetattr(fd)
+            if not (attrs[0] & termios.ICRNL):
+                attrs[0] |= termios.ICRNL
+                termios.tcsetattr(fd, termios.TCSANOW, attrs)
+        return input(prompt)
+
+else:
+
+    def safe_input(prompt: str) -> str:
+        return input(prompt)
+
+
+T = TypeVar("T")
+
+
+def prompt_selection(
+    items: list[T],
+    formatter: Callable[[T], str],
+    header: str,
+    default: T | None = None,
+    allow_skip: bool = False,
+    empty_message: str | None = None,
+) -> T | None:
+    """Interactive terminal prompt to pick one item from a list."""
+    print("\n" + "=" * 50)
+    print(header)
+    print("=" * 50)
+
+    if not items:
+        if empty_message:
+            print(f"  {empty_message}")
+        print("-" * 50 + "\n")
+        return None
+
+    for i, item in enumerate(items):
+        print(f"  {i}: {formatter(item)}")
+
+    if allow_skip:
+        print("  n: Skip (none)")
+
+    print("-" * 50)
+
+    while True:
+        try:
+            if allow_skip:
+                text = f"Select [0-{len(items) - 1}] or 'n' to skip: "
+            elif default is not None:
+                text = f"Select [0-{len(items) - 1}] (Enter for default): "
+            else:
+                text = f"Select [0-{len(items) - 1}]: "
+
+            choice = safe_input(text).strip().lower()
+
+            if choice == "" and default is not None:
+                print(f"  -> Using default: {formatter(default)}")
+                return default
+
+            if choice in ("n", "") and allow_skip:
+                print("  -> No selection")
+                print("-" * 50 + "\n")
+                return None
+
+            idx = int(choice)
+            if 0 <= idx < len(items):
+                selected = items[idx]
+                print(f"  -> Selected: {formatter(selected)}")
+                print("-" * 50 + "\n")
+                return selected
+
+            print(f"  Invalid choice, enter 0-{len(items) - 1}")
+        except ValueError:
+            print("  Please enter a number" + (" or 'n'" if allow_skip else ""))