reachy-mini 1.0.0rc1__py3-none-any.whl

reachy_mini/media/audio_base.py

@@ -0,0 +1,75 @@
+"""Base classes for audio implementations.
+
+The audio implementations support various backends and provide a unified
+interface for audio input/output.
+"""
+
+import logging
+from abc import ABC, abstractmethod
+from enum import Enum
+from typing import Optional
+
+import numpy as np
+import numpy.typing as npt
+
+
+class AudioBackend(Enum):
+    """Audio backends."""
+
+    SOUNDDEVICE = "sounddevice"
+    GSTREAMER = "gstreamer"
+
+
+class AudioBase(ABC):
+    """Abstract class for opening and managing audio devices."""
+
+    def __init__(self, backend: AudioBackend, log_level: str = "INFO") -> None:
+        """Initialize the audio device."""
+        self.logger = logging.getLogger(__name__)
+        self.logger.setLevel(log_level)
+        self.backend = backend
+
+    @abstractmethod
+    def start_recording(self) -> None:
+        """Start recording audio."""
+        pass
+
+    @abstractmethod
+    def get_audio_sample(self) -> Optional[bytes | npt.NDArray[np.float32]]:
+        """Read audio data from the device. Returns the data or None if error."""
+        pass
+
+    @abstractmethod
+    def get_audio_samplerate(self) -> int:
+        """Return the samplerate of the audio device."""
+        pass
+
+    @abstractmethod
+    def stop_recording(self) -> None:
+        """Close the audio device and release resources."""
+        pass
+
+    @abstractmethod
+    def start_playing(self) -> None:
+        """Start playing audio."""
+        pass
+
+    @abstractmethod
+    def push_audio_sample(self, data: bytes) -> None:
+        """Push audio data to the output device."""
+        pass
+
+    @abstractmethod
+    def stop_playing(self) -> None:
+        """Stop playing audio and release resources."""
+        pass
+
+    @abstractmethod
+    def play_sound(self, sound_file: str) -> None:
+        """Play a sound file.
+
+        Args:
+            sound_file (str): Path to the sound file to play.
+
+        """
+        pass

reachy_mini/media/audio_gstreamer.py

@@ -0,0 +1,194 @@
+"""GStreamer camera backend.
+
+This module provides an implementation of the CameraBase class using GStreamer.
+By default the module directly returns JPEG images as output by the camera.
+"""
+
+from threading import Thread
+from typing import Optional
+
+try:
+    import gi
+except ImportError as e:
+    raise ImportError(
+        "The 'gi' module is required for GStreamerCamera but could not be imported. \
+                      Please install the GStreamer backend: pip install .[gstreamer]."
+    ) from e
+
+gi.require_version("Gst", "1.0")
+gi.require_version("GstApp", "1.0")
+
+
+from gi.repository import GLib, Gst, GstApp  # noqa: E402
+
+from .audio_base import AudioBackend, AudioBase  # noqa: E402
+
+
+class GStreamerAudio(AudioBase):
+    """Audio implementation using GStreamer."""
+
+    def __init__(self, log_level: str = "INFO") -> None:
+        """Initialize the GStreamer audio."""
+        super().__init__(backend=AudioBackend.GSTREAMER, log_level=log_level)
+        Gst.init(None)
+        self._loop = GLib.MainLoop()
+        self._thread_bus_calls = Thread(target=lambda: self._loop.run(), daemon=True)
+        self._thread_bus_calls.start()
+
+        self._samplerate = 24000
+
+        self._pipeline_record = Gst.Pipeline.new("audio_recorder")
+        self._appsink_audio: Optional[GstApp] = None
+        self._init_pipeline_record(self._pipeline_record)
+        self._bus_record = self._pipeline_record.get_bus()
+        self._bus_record.add_watch(
+            GLib.PRIORITY_DEFAULT, self._on_bus_message, self._loop
+        )
+
+        self._pipeline_playback = Gst.Pipeline.new("audio_player")
+        self._appsrc: Optional[GstApp] = None
+        self._init_pipeline_playback(self._pipeline_playback)
+        self._bus_playback = self._pipeline_playback.get_bus()
+        self._bus_playback.add_watch(
+            GLib.PRIORITY_DEFAULT, self._on_bus_message, self._loop
+        )
+
+    def _init_pipeline_record(self, pipeline: Gst.Pipeline) -> None:
+        self._appsink_audio = Gst.ElementFactory.make("appsink")
+        caps = Gst.Caps.from_string(
+            f"audio/x-raw,channels=1,rate={self._samplerate},format=S16LE"
+        )
+        self._appsink_audio.set_property("caps", caps)
+        self._appsink_audio.set_property("drop", True)  # avoid overflow
+        self._appsink_audio.set_property("max-buffers", 200)
+
+        autoaudiosrc = Gst.ElementFactory.make("autoaudiosrc")  # use default mic
+        # caps_respeaker = Gst.Caps.from_string(
+        #    "audio/x-raw, layout=interleaved, format=S16LE, rate=16000, channels=2"
+        # )
+        # autoaudiosrc.set_property("filter-caps", caps_respeaker)
+        queue = Gst.ElementFactory.make("queue")
+        audioconvert = Gst.ElementFactory.make("audioconvert")
+        audioresample = Gst.ElementFactory.make("audioresample")
+
+        if not all(
+            [autoaudiosrc, queue, audioconvert, audioresample, self._appsink_audio]
+        ):
+            raise RuntimeError("Failed to create GStreamer elements")
+
+        pipeline.add(autoaudiosrc)
+        pipeline.add(queue)
+        pipeline.add(audioconvert)
+        pipeline.add(audioresample)
+        pipeline.add(self._appsink_audio)
+
+        autoaudiosrc.link(queue)
+        queue.link(audioconvert)
+        audioconvert.link(audioresample)
+        audioresample.link(self._appsink_audio)
+
+    def __del__(self) -> None:
+        """Destructor to ensure gstreamer resources are released."""
+        self._loop.quit()
+        self._bus_record.remove_watch()
+        self._bus_playback.remove_watch()
+
+    def _init_pipeline_playback(self, pipeline: Gst.Pipeline) -> None:
+        self._appsrc = Gst.ElementFactory.make("appsrc")
+        self._appsrc.set_property("format", Gst.Format.TIME)
+        self._appsrc.set_property("is-live", True)
+        caps = Gst.Caps.from_string(
+            f"audio/x-raw,format=F32LE,channels=1,rate={self._samplerate},layout=interleaved"
+        )
+        self._appsrc.set_property("caps", caps)
+
+        audioconvert = Gst.ElementFactory.make("audioconvert")
+        audioresample = Gst.ElementFactory.make("audioresample")
+
+        queue = Gst.ElementFactory.make("queue")
+        audiosink = Gst.ElementFactory.make("autoaudiosink")  # use default speaker
+
+        pipeline.add(queue)
+        pipeline.add(audiosink)
+        pipeline.add(self._appsrc)
+        pipeline.add(audioconvert)
+        pipeline.add(audioresample)
+
+        self._appsrc.link(queue)
+        queue.link(audioconvert)
+        audioconvert.link(audioresample)
+        audioresample.link(audiosink)
+
+    def _on_bus_message(self, bus: Gst.Bus, msg: Gst.Message, loop) -> bool:  # type: ignore[no-untyped-def]
+        t = msg.type
+        if t == Gst.MessageType.EOS:
+            self.logger.warning("End-of-stream")
+            return False
+
+        elif t == Gst.MessageType.ERROR:
+            err, debug = msg.parse_error()
+            self.logger.error(f"Error: {err} {debug}")
+            return False
+
+        return True
+
+    def start_recording(self) -> None:
+        """Open the audio card using GStreamer."""
+        self._pipeline_record.set_state(Gst.State.PLAYING)
+
+    def _get_sample(self, appsink: Gst.AppSink) -> Optional[bytes]:
+        sample = appsink.try_pull_sample(20_000_000)
+        if sample is None:
+            return None
+        data = None
+        if isinstance(sample, Gst.Sample):
+            buf = sample.get_buffer()
+            if buf is None:
+                self.logger.warning("Buffer is None")
+
+            data = buf.extract_dup(0, buf.get_size())
+        return data
+
+    def get_audio_sample(self) -> Optional[bytes]:
+        """Read a sample from the audio card. Returns the sample or None if error.
+
+        Returns:
+            Optional[bytes]: The captured sample in raw format, or None if error.
+
+        """
+        return self._get_sample(self._appsink_audio)
+
+    def get_audio_samplerate(self) -> int:
+        """Return the samplerate of the audio device."""
+        return self._samplerate
+
+    def stop_recording(self) -> None:
+        """Release the camera resource."""
+        self._pipeline_record.set_state(Gst.State.NULL)
+
+    def start_playing(self) -> None:
+        """Open the audio output using GStreamer."""
+        self._pipeline_playback.set_state(Gst.State.PLAYING)
+
+    def stop_playing(self) -> None:
+        """Stop playing audio and release resources."""
+        self._pipeline_playback.set_state(Gst.State.NULL)
+
+    def push_audio_sample(self, data: bytes) -> None:
+        """Push audio data to the output device."""
+        if self._appsrc is not None:
+            buf = Gst.Buffer.new_wrapped(data)
+            self._appsrc.push_buffer(buf)
+        else:
+            self.logger.warning(
+                "AppSrc is not initialized. Call start_playing() first."
+            )
+
+    def play_sound(self, sound_file: str) -> None:
+        """Play a sound file.
+
+        Args:
+            sound_file (str): Path to the sound file to play.
+
+        """
+        self.logger.warning("play_sound is not implemented for GStreamerAudio.")

reachy_mini/media/audio_sounddevice.py

@@ -0,0 +1,207 @@
+"""Audio implementation using sounddevice backend."""
+
+import os
+import threading
+import time
+from typing import Any, List, Optional
+
+import numpy as np
+import numpy.typing as npt
+import scipy
+import sounddevice as sd
+import soundfile as sf
+
+from reachy_mini.utils.constants import ASSETS_ROOT_PATH
+
+from .audio_base import AudioBackend, AudioBase
+
+
+class SoundDeviceAudio(AudioBase):
+    """Audio device implementation using sounddevice."""
+
+    def __init__(
+        self,
+        frames_per_buffer: int = 1024,
+        log_level: str = "INFO",
+    ) -> None:
+        """Initialize the SoundDevice audio device."""
+        super().__init__(backend=AudioBackend.SOUNDDEVICE, log_level=log_level)
+        self.frames_per_buffer = frames_per_buffer
+        self.stream = None
+        self._output_stream = None
+        self._buffer: List[npt.NDArray[np.float32]] = []
+        self._device_id = self.get_output_device_id("respeaker")
+        self._samplerate = (
+            -1
+        )  # will be set on first use to avoid issues if device is not present (CI)
+
+    def start_recording(self) -> None:
+        """Open the audio input stream, using ReSpeaker card if available."""
+        self.stream = sd.InputStream(
+            blocksize=self.frames_per_buffer,
+            device=self._device_id,
+            callback=self._callback,
+        )
+        if self.stream is None:
+            raise RuntimeError("Failed to open SoundDevice audio stream.")
+        self._buffer.clear()
+        self.stream.start()
+        self.logger.info("SoundDevice audio stream opened.")
+
+    def _callback(
+        self,
+        indata: npt.NDArray[np.float32],
+        frames: int,
+        time: int,
+        status: sd.CallbackFlags,
+    ) -> None:
+        if status:
+            self.logger.warning(f"SoundDevice status: {status}")
+
+        self._buffer.append(indata.copy())
+
+    def get_audio_sample(self) -> Optional[bytes | npt.NDArray[np.float32]]:
+        """Read audio data from the buffer. Returns numpy array or None if empty."""
+        if self._buffer and len(self._buffer) > 0:
+            data: npt.NDArray[np.float32] = np.concatenate(self._buffer, axis=0)
+            self._buffer.clear()
+            return data
+        self.logger.debug("No audio data available in buffer.")
+        return None
+
+    def get_audio_samplerate(self) -> int:
+        """Return the samplerate of the audio device."""
+        if self._samplerate == -1:
+            self._samplerate = int(
+                sd.query_devices(self._device_id)["default_samplerate"]
+            )
+        return self._samplerate
+
+    def stop_recording(self) -> None:
+        """Close the audio stream and release resources."""
+        if self.stream is not None:
+            self.stream.stop()
+            self.stream.close()
+            self.stream = None
+            self.logger.info("SoundDevice audio stream closed.")
+
+    def push_audio_sample(self, data: bytes) -> None:
+        """Push audio data to the output device."""
+        if self._output_stream is not None:
+            self._output_stream.write(data)
+        else:
+            self.logger.warning(
+                "Output stream is not open. Call start_playing() first."
+            )
+
+    def start_playing(self) -> None:
+        """Open the audio output stream."""
+        self._output_stream = sd.OutputStream(
+            samplerate=self.get_audio_samplerate(),
+            device=self._device_id,
+            channels=1,
+        )
+        if self._output_stream is None:
+            raise RuntimeError("Failed to open SoundDevice audio output stream.")
+        self._output_stream.start()
+
+    def stop_playing(self) -> None:
+        """Close the audio output stream."""
+        if self._output_stream is not None:
+            self._output_stream.stop()
+            self._output_stream.close()
+            self._output_stream = None
+            self.logger.info("SoundDevice audio output stream closed.")
+
+    def play_sound(self, sound_file: str, autoclean: bool = False) -> None:
+        """Play a sound file from the assets directory or a given path using sounddevice and soundfile."""
+        file_path = f"{ASSETS_ROOT_PATH}/{sound_file}"
+        if not os.path.exists(file_path):
+            raise FileNotFoundError(f"Sound file {file_path} not found.")
+
+        data, samplerate_in = sf.read(file_path, dtype="float32")
+
+        if samplerate_in != self.get_audio_samplerate():
+            data = scipy.signal.resample(
+                data, int(len(data) * (self.get_audio_samplerate() / samplerate_in))
+            )
+        if data.ndim > 1:  # convert to mono
+            data = np.mean(data, axis=1)
+
+        self.logger.debug(f"Playing sound '{file_path}' at {samplerate_in} Hz")
+
+        self.stop_playing()
+        start = [0]  # using list to modify in callback
+        length = len(data)
+
+        def callback(
+            outdata: npt.NDArray[np.float32],
+            frames: int,
+            time: Any,  # cdata 'struct PaStreamCallbackTimeInfo *
+            status: sd.CallbackFlags,
+        ) -> None:
+            """Actual playback."""
+            if status:
+                self.logger.warning(f"SoundDevice output status: {status}")
+
+            end = start[0] + frames
+            if end > length:
+                # Fill the output buffer with the audio data, or zeros if finished
+                outdata[: length - start[0], 0] = data[start[0] :]
+                outdata[length - start[0] :, 0] = 0
+                raise sd.CallbackStop()
+            else:
+                outdata[:, 0] = data[start[0] : end]
+            start[0] = end
+
+        event = threading.Event()
+
+        self._output_stream = sd.OutputStream(
+            samplerate=self.get_audio_samplerate(),
+            device=self._device_id,
+            channels=1,
+            callback=callback,
+            finished_callback=event.set,  # release the device when done
+        )
+        if self._output_stream is None:
+            raise RuntimeError("Failed to open SoundDevice audio output stream.")
+        self._output_stream.start()
+
+        def _clean_up_thread() -> None:
+            """Thread to clean up the output stream after playback.
+
+            The daemon may play sound but should release the audio device.
+            """
+            event.wait()
+            timeout = 5  # seconds
+            waited = 0
+            while (
+                self._output_stream is not None
+                and self._output_stream.active
+                and waited < timeout
+            ):
+                time.sleep(0.1)
+                waited += 0.1
+            self.stop_playing()
+
+        if autoclean:
+            threading.Thread(
+                target=_clean_up_thread,
+                daemon=True,
+            ).start()
+
+    def get_output_device_id(self, name_contains: str) -> int:
+        """Return the output device id whose name contains the given string (case-insensitive).
+
+        If not found, return the default output device id.
+        """
+        devices = sd.query_devices()
+
+        for idx, dev in enumerate(devices):
+            if name_contains.lower() in dev["name"].lower():
+                return idx
+        # Return default output device if not found
+        self.logger.warning(
+            f"No output device found containing '{name_contains}', using default."
+        )
+        return int(sd.default.device[1])

reachy_mini/media/audio_utils.py

@@ -0,0 +1,27 @@
+"""Utility functions for audio handling, specifically for detecting the ReSpeaker sound card."""
+
+import logging
+import subprocess
+
+
+def get_respeaker_card_number() -> int:
+    """Return the card number of the ReSpeaker sound card, or 0 if not found."""
+    try:
+        result = subprocess.run(
+            ["arecord", "-l"], capture_output=True, text=True, check=True
+        )
+        output = result.stdout
+
+        lines = output.split("\n")
+        for line in lines:
+            if "ReSpeaker" in line and "card" in line:
+                card_number = line.split(":")[0].split("card ")[1].strip()
+                logging.debug(f"Found ReSpeaker sound card: {card_number}")
+                return int(card_number)
+
+        logging.warning("ReSpeaker sound card not found. Returning default card")
+        return 0  # default sound card
+
+    except subprocess.CalledProcessError as e:
+        logging.error(f"Cannot find sound card: {e}")
+        return 0

reachy_mini/media/camera_base.py

@@ -0,0 +1,58 @@
+"""Base classes for camera implementations.
+
+The camera implementations support various backends and provide a unified
+interface for capturing images.
+"""
+
+import logging
+from abc import ABC, abstractmethod
+from enum import Enum
+from typing import Optional
+
+import numpy as np
+import numpy.typing as npt
+
+from reachy_mini.media.camera_constants import CameraResolution
+
+
+class CameraBackend(Enum):
+    """Camera backends."""
+
+    OPENCV = "opencv"
+    GSTREAMER = "gstreamer"
+
+
+class CameraBase(ABC):
+    """Abstract class for opening and managing a camera."""
+
+    def __init__(
+        self,
+        backend: CameraBackend,
+        log_level: str = "INFO",
+        resolution: CameraResolution = CameraResolution.R1280x720,
+    ) -> None:
+        """Initialize the camera."""
+        self.logger = logging.getLogger(__name__)
+        self.logger.setLevel(log_level)
+        self.backend = backend
+        self._resolution = resolution
+
+    @property
+    def resolution(self) -> tuple[int, int]:
+        """Get the current camera resolution as a tuple (width, height)."""
+        return (self._resolution.value[0], self._resolution.value[1])
+
+    @abstractmethod
+    def open(self) -> None:
+        """Open the camera."""
+        pass
+
+    @abstractmethod
+    def read(self) -> Optional[bytes | npt.NDArray[np.uint8]]:
+        """Read an image from the camera. Returns the image or None if error."""
+        pass
+
+    @abstractmethod
+    def close(self) -> None:
+        """Close the camera and release resources."""
+        pass

reachy_mini/media/camera_constants.py

@@ -0,0 +1,13 @@
+"""Camera constants for Reachy Mini."""
+
+from enum import Enum
+
+
+class CameraResolution(Enum):
+    """Camera resolutions. Arducam_12MP."""
+
+    R2304x1296 = (2304, 1296)
+    R4608x2592 = (4608, 2592)
+    R1920x1080 = (1920, 1080)
+    R1600x1200 = (1600, 1200)
+    R1280x720 = (1280, 720)