reachy-mini 1.0.0__py3-none-any.whl

reachy_mini/media/audio_base.py

@@ -0,0 +1,163 @@
+"""Base classes for audio implementations.
+
+The audio implementations support various backends and provide a unified
+interface for audio input/output.
+"""
+
+import logging
+import struct
+from abc import ABC, abstractmethod
+from enum import Enum
+from typing import List, Optional
+
+import numpy as np
+import numpy.typing as npt
+import usb
+from libusb_package import get_libusb1_backend
+
+
+class AudioBackend(Enum):
+    """Audio backends."""
+
+    SOUNDDEVICE = "sounddevice"
+    GSTREAMER = "gstreamer"
+
+
+class AudioBase(ABC):
+    """Abstract class for opening and managing audio devices."""
+
+    SAMPLE_RATE = 16000  # respeaker samplerate
+    TIMEOUT = 100000
+    PARAMETERS = {
+        "VERSION": (48, 0, 4, "ro", "uint8"),
+        "AEC_AZIMUTH_VALUES": (33, 75, 16 + 1, "ro", "radians"),
+        "DOA_VALUE": (20, 18, 4 + 1, "ro", "uint16"),
+        "DOA_VALUE_RADIANS": (20, 19, 8 + 1, "ro", "radians"),
+    }
+
+    def __init__(self, backend: AudioBackend, log_level: str = "INFO") -> None:
+        """Initialize the audio device."""
+        self.logger = logging.getLogger(__name__)
+        self.logger.setLevel(log_level)
+        self.backend = backend
+        self._respeaker = self._init_respeaker_usb()
+        # name, resid, cmdid, length, type
+
+    def __del__(self) -> None:
+        """Destructor to ensure resources are released."""
+        if self._respeaker:
+            usb.util.dispose_resources(self._respeaker)
+
+    @abstractmethod
+    def start_recording(self) -> None:
+        """Start recording audio."""
+        pass
+
+    @abstractmethod
+    def get_audio_sample(self) -> Optional[npt.NDArray[np.float32]]:
+        """Read audio data from the device. Returns the data or None if error."""
+        pass
+
+    @abstractmethod
+    def stop_recording(self) -> None:
+        """Close the audio device and release resources."""
+        pass
+
+    @abstractmethod
+    def start_playing(self) -> None:
+        """Start playing audio."""
+        pass
+
+    @abstractmethod
+    def push_audio_sample(self, data: npt.NDArray[np.float32]) -> None:
+        """Push audio data to the output device."""
+        pass
+
+    @abstractmethod
+    def stop_playing(self) -> None:
+        """Stop playing audio and release resources."""
+        pass
+
+    @abstractmethod
+    def play_sound(self, sound_file: str) -> None:
+        """Play a sound file.
+
+        Args:
+            sound_file (str): Path to the sound file to play.
+
+        """
+        pass
+
+    def _init_respeaker_usb(self) -> Optional[usb.core.Device]:
+        try:
+            dev = usb.core.find(
+                idVendor=0x2886, idProduct=0x001A, backend=get_libusb1_backend()
+            )
+            return dev
+        except usb.core.NoBackendError:
+            self.logger.error(
+                "No USB backend was found ! Make sure libusb_package is correctly installed with `pip install libusb_package`."
+            )
+            return None
+
+    def _read_usb(self, name: str) -> Optional[List[int] | List[float]]:
+        try:
+            data = self.PARAMETERS[name]
+        except KeyError:
+            self.logger.error(f"Unknown parameter: {name}")
+            return None
+
+        if not self._respeaker:
+            self.logger.warning("ReSpeaker device not found.")
+            return None
+
+        resid = data[0]
+        cmdid = 0x80 | data[1]
+        length = data[2]
+
+        response = self._respeaker.ctrl_transfer(
+            usb.util.CTRL_IN
+            | usb.util.CTRL_TYPE_VENDOR
+            | usb.util.CTRL_RECIPIENT_DEVICE,
+            0,
+            cmdid,
+            resid,
+            length,
+            self.TIMEOUT,
+        )
+
+        self.logger.debug(f"Response for {name}: {response}")
+
+        result: Optional[List[float] | List[int]] = None
+        if data[4] == "uint8":
+            result = response.tolist()
+        elif data[4] == "radians":
+            byte_data = response.tobytes()
+            num_values = (data[2] - 1) / 4
+            match_str = "<"
+            for i in range(int(num_values)):
+                match_str += "f"
+            result = [
+                float(x) for x in struct.unpack(match_str, byte_data[1 : data[2]])
+            ]
+        elif data[4] == "uint16":
+            result = response.tolist()
+
+        return result
+
+    def get_DoA(self) -> tuple[float, bool] | None:
+        """Get the Direction of Arrival (DoA) value from the ReSpeaker device.
+
+        0° is left, 90° is front/back, 180° is right
+
+        Returns:
+            tuple: A tuple containing the DoA value as an integer and the speech detection, or None if the device is not found.
+
+        """
+        if not self._respeaker:
+            self.logger.warning("ReSpeaker device not found.")
+            return None
+        result = self._read_usb("DOA_VALUE_RADIANS")
+        if result is None:
+            return None
+        return float(result[0]), bool(result[1])

reachy_mini/media/audio_gstreamer.py

@@ -0,0 +1,195 @@
+"""GStreamer camera backend.
+
+This module provides an implementation of the CameraBase class using GStreamer.
+By default the module directly returns JPEG images as output by the camera.
+"""
+
+from threading import Thread
+from typing import Optional
+
+import numpy as np
+import numpy.typing as npt
+
+try:
+    import gi
+except ImportError as e:
+    raise ImportError(
+        "The 'gi' module is required for GStreamerCamera but could not be imported. \
+                      Please install the GStreamer backend: pip install .[gstreamer]."
+    ) from e
+
+gi.require_version("Gst", "1.0")
+gi.require_version("GstApp", "1.0")
+
+
+from gi.repository import GLib, Gst, GstApp  # noqa: E402
+
+from .audio_base import AudioBackend, AudioBase  # noqa: E402
+
+
+class GStreamerAudio(AudioBase):
+    """Audio implementation using GStreamer."""
+
+    def __init__(self, log_level: str = "INFO") -> None:
+        """Initialize the GStreamer audio."""
+        super().__init__(backend=AudioBackend.GSTREAMER, log_level=log_level)
+        Gst.init(None)
+        self._loop = GLib.MainLoop()
+        self._thread_bus_calls = Thread(target=lambda: self._loop.run(), daemon=True)
+        self._thread_bus_calls.start()
+
+        self._pipeline_record = Gst.Pipeline.new("audio_recorder")
+        self._appsink_audio: Optional[GstApp] = None
+        self._init_pipeline_record(self._pipeline_record)
+        self._bus_record = self._pipeline_record.get_bus()
+        self._bus_record.add_watch(
+            GLib.PRIORITY_DEFAULT, self._on_bus_message, self._loop
+        )
+
+        self._pipeline_playback = Gst.Pipeline.new("audio_player")
+        self._appsrc: Optional[GstApp] = None
+        self._init_pipeline_playback(self._pipeline_playback)
+        self._bus_playback = self._pipeline_playback.get_bus()
+        self._bus_playback.add_watch(
+            GLib.PRIORITY_DEFAULT, self._on_bus_message, self._loop
+        )
+
+    def _init_pipeline_record(self, pipeline: Gst.Pipeline) -> None:
+        self._appsink_audio = Gst.ElementFactory.make("appsink")
+        caps = Gst.Caps.from_string(
+            f"audio/x-raw,channels=2,rate={self.SAMPLE_RATE},format=F32LE"
+        )
+        self._appsink_audio.set_property("caps", caps)
+        self._appsink_audio.set_property("drop", True)  # avoid overflow
+        self._appsink_audio.set_property("max-buffers", 500)
+
+        autoaudiosrc = Gst.ElementFactory.make("autoaudiosrc")  # use default mic
+        # caps_respeaker = Gst.Caps.from_string(
+        #    "audio/x-raw, layout=interleaved, format=S16LE, rate=16000, channels=2"
+        # )
+        # autoaudiosrc.set_property("filter-caps", caps_respeaker)
+        queue = Gst.ElementFactory.make("queue")
+        audioconvert = Gst.ElementFactory.make("audioconvert")
+        audioresample = Gst.ElementFactory.make("audioresample")
+
+        if not all(
+            [autoaudiosrc, queue, audioconvert, audioresample, self._appsink_audio]
+        ):
+            raise RuntimeError("Failed to create GStreamer elements")
+
+        pipeline.add(autoaudiosrc)
+        pipeline.add(queue)
+        pipeline.add(audioconvert)
+        pipeline.add(audioresample)
+        pipeline.add(self._appsink_audio)
+
+        autoaudiosrc.link(queue)
+        queue.link(audioconvert)
+        audioconvert.link(audioresample)
+        audioresample.link(self._appsink_audio)
+
+    def __del__(self) -> None:
+        """Destructor to ensure gstreamer resources are released."""
+        super().__del__()
+        self._loop.quit()
+        self._bus_record.remove_watch()
+        self._bus_playback.remove_watch()
+
+    def _init_pipeline_playback(self, pipeline: Gst.Pipeline) -> None:
+        self._appsrc = Gst.ElementFactory.make("appsrc")
+        self._appsrc.set_property("format", Gst.Format.TIME)
+        self._appsrc.set_property("is-live", True)
+        caps = Gst.Caps.from_string(
+            f"audio/x-raw,format=F32LE,channels=1,rate={self.SAMPLE_RATE},layout=interleaved"
+        )
+        self._appsrc.set_property("caps", caps)
+
+        audioconvert = Gst.ElementFactory.make("audioconvert")
+        audioresample = Gst.ElementFactory.make("audioresample")
+
+        queue = Gst.ElementFactory.make("queue")
+        audiosink = Gst.ElementFactory.make("autoaudiosink")  # use default speaker
+
+        pipeline.add(queue)
+        pipeline.add(audiosink)
+        pipeline.add(self._appsrc)
+        pipeline.add(audioconvert)
+        pipeline.add(audioresample)
+
+        self._appsrc.link(queue)
+        queue.link(audioconvert)
+        audioconvert.link(audioresample)
+        audioresample.link(audiosink)
+
+    def _on_bus_message(self, bus: Gst.Bus, msg: Gst.Message, loop) -> bool:  # type: ignore[no-untyped-def]
+        t = msg.type
+        if t == Gst.MessageType.EOS:
+            self.logger.warning("End-of-stream")
+            return False
+
+        elif t == Gst.MessageType.ERROR:
+            err, debug = msg.parse_error()
+            self.logger.error(f"Error: {err} {debug}")
+            return False
+
+        return True
+
+    def start_recording(self) -> None:
+        """Open the audio card using GStreamer."""
+        self._pipeline_record.set_state(Gst.State.PLAYING)
+
+    def _get_sample(self, appsink: GstApp.AppSink) -> Optional[bytes]:
+        sample = appsink.try_pull_sample(20_000_000)
+        if sample is None:
+            return None
+        data = None
+        if isinstance(sample, Gst.Sample):
+            buf = sample.get_buffer()
+            if buf is None:
+                self.logger.warning("Buffer is None")
+
+            data = buf.extract_dup(0, buf.get_size())
+        return data
+
+    def get_audio_sample(self) -> Optional[npt.NDArray[np.float32]]:
+        """Read a sample from the audio card. Returns the sample or None if error.
+
+        Returns:
+            Optional[npt.NDArray[np.float32]]: The captured sample in raw format, or None if error.
+
+        """
+        sample = self._get_sample(self._appsink_audio)
+        if sample is None:
+            return None
+        return np.frombuffer(sample, dtype=np.float32).reshape(-1, 2)
+
+    def stop_recording(self) -> None:
+        """Release the camera resource."""
+        self._pipeline_record.set_state(Gst.State.NULL)
+
+    def start_playing(self) -> None:
+        """Open the audio output using GStreamer."""
+        self._pipeline_playback.set_state(Gst.State.PLAYING)
+
+    def stop_playing(self) -> None:
+        """Stop playing audio and release resources."""
+        self._pipeline_playback.set_state(Gst.State.NULL)
+
+    def push_audio_sample(self, data: npt.NDArray[np.float32]) -> None:
+        """Push audio data to the output device."""
+        if self._appsrc is not None:
+            buf = Gst.Buffer.new_wrapped(data.tobytes())
+            self._appsrc.push_buffer(buf)
+        else:
+            self.logger.warning(
+                "AppSrc is not initialized. Call start_playing() first."
+            )
+
+    def play_sound(self, sound_file: str) -> None:
+        """Play a sound file.
+
+        Args:
+            sound_file (str): Path to the sound file to play.
+
+        """
+        self.logger.warning("play_sound is not implemented for GStreamerAudio.")

reachy_mini/media/audio_sounddevice.py

@@ -0,0 +1,226 @@
+"""Audio implementation using sounddevice backend."""
+
+import os
+import threading
+import time
+from typing import Any, List, Optional
+
+import numpy as np
+import numpy.typing as npt
+import scipy
+import sounddevice as sd
+import soundfile as sf
+
+from reachy_mini.utils.constants import ASSETS_ROOT_PATH
+
+from .audio_base import AudioBackend, AudioBase
+
+
+class SoundDeviceAudio(AudioBase):
+    """Audio device implementation using sounddevice."""
+
+    def __init__(
+        self,
+        frames_per_buffer: int = 1024,
+        log_level: str = "INFO",
+    ) -> None:
+        """Initialize the SoundDevice audio device."""
+        super().__init__(backend=AudioBackend.SOUNDDEVICE, log_level=log_level)
+        self.frames_per_buffer = frames_per_buffer
+        self.stream = None
+        self._output_stream = None
+        self._buffer: List[npt.NDArray[np.float32]] = []
+        self._output_device_id = self.get_output_device_id("respeaker")
+        self._input_device_id = self.get_input_device_id("respeaker")
+
+    def start_recording(self) -> None:
+        """Open the audio input stream, using ReSpeaker card if available."""
+        self.stream = sd.InputStream(
+            blocksize=self.frames_per_buffer,
+            device=self._input_device_id,
+            callback=self._callback,
+            samplerate=self.SAMPLE_RATE,
+        )
+        if self.stream is None:
+            raise RuntimeError("Failed to open SoundDevice audio stream.")
+        self._buffer.clear()
+        self.stream.start()
+        self.logger.info("SoundDevice audio stream opened.")
+
+    def _callback(
+        self,
+        indata: npt.NDArray[np.float32],
+        frames: int,
+        time: int,
+        status: sd.CallbackFlags,
+    ) -> None:
+        if status:
+            self.logger.warning(f"SoundDevice status: {status}")
+
+        self._buffer.append(indata.copy())
+
+    def get_audio_sample(self) -> Optional[npt.NDArray[np.float32]]:
+        """Read audio data from the buffer. Returns numpy array or None if empty."""
+        if self._buffer and len(self._buffer) > 0:
+            data: npt.NDArray[np.float32] = np.concatenate(self._buffer, axis=0)
+            self._buffer.clear()
+            return data
+        self.logger.debug("No audio data available in buffer.")
+        return None
+
+    def stop_recording(self) -> None:
+        """Close the audio stream and release resources."""
+        if self.stream is not None:
+            self.stream.stop()
+            self.stream.close()
+            self.stream = None
+            self.logger.info("SoundDevice audio stream closed.")
+
+    def push_audio_sample(self, data: npt.NDArray[np.float32]) -> None:
+        """Push audio data to the output device."""
+        if self._output_stream is not None:
+            self._output_stream.write(data)
+        else:
+            self.logger.warning(
+                "Output stream is not open. Call start_playing() first."
+            )
+
+    def start_playing(self) -> None:
+        """Open the audio output stream."""
+        self._output_stream = sd.OutputStream(
+            samplerate=self.SAMPLE_RATE,
+            device=self._output_device_id,
+            channels=1,
+        )
+        if self._output_stream is None:
+            raise RuntimeError("Failed to open SoundDevice audio output stream.")
+        self._output_stream.start()
+
+    def stop_playing(self) -> None:
+        """Close the audio output stream."""
+        if self._output_stream is not None:
+            self._output_stream.stop()
+            self._output_stream.close()
+            self._output_stream = None
+            self.logger.info("SoundDevice audio output stream closed.")
+
+    def play_sound(self, sound_file: str, autoclean: bool = False) -> None:
+        """Play a sound file from the assets directory or a given path using sounddevice and soundfile."""
+        file_path = f"{ASSETS_ROOT_PATH}/{sound_file}"
+        if not os.path.exists(file_path):
+            raise FileNotFoundError(f"Sound file {file_path} not found.")
+
+        data, samplerate_in = sf.read(file_path, dtype="float32")
+
+        if samplerate_in != self.SAMPLE_RATE:
+            data = scipy.signal.resample(
+                data, int(len(data) * (self.SAMPLE_RATE / samplerate_in))
+            )
+        if data.ndim > 1:  # convert to mono
+            data = np.mean(data, axis=1)
+
+        self.logger.debug(f"Playing sound '{file_path}' at {samplerate_in} Hz")
+
+        self.stop_playing()
+        start = [0]  # using list to modify in callback
+        length = len(data)
+
+        def callback(
+            outdata: npt.NDArray[np.float32],
+            frames: int,
+            time: Any,  # cdata 'struct PaStreamCallbackTimeInfo *
+            status: sd.CallbackFlags,
+        ) -> None:
+            """Actual playback."""
+            if status:
+                self.logger.warning(f"SoundDevice output status: {status}")
+
+            end = start[0] + frames
+            if end > length:
+                # Fill the output buffer with the audio data, or zeros if finished
+                outdata[: length - start[0], 0] = data[start[0] :]
+                outdata[length - start[0] :, 0] = 0
+                raise sd.CallbackStop()
+            else:
+                outdata[:, 0] = data[start[0] : end]
+            start[0] = end
+
+        event = threading.Event()
+
+        self._output_stream = sd.OutputStream(
+            samplerate=self.SAMPLE_RATE,
+            device=self._output_device_id,
+            channels=1,
+            callback=callback,
+            finished_callback=event.set,  # release the device when done
+        )
+        if self._output_stream is None:
+            raise RuntimeError("Failed to open SoundDevice audio output stream.")
+        self._output_stream.start()
+
+        def _clean_up_thread() -> None:
+            """Thread to clean up the output stream after playback.
+
+            The daemon may play sound but should release the audio device.
+            """
+            event.wait()
+            timeout = 5  # seconds
+            waited = 0
+            while (
+                self._output_stream is not None
+                and self._output_stream.active
+                and waited < timeout
+            ):
+                time.sleep(0.1)
+                waited += 0.1
+            self.stop_playing()
+
+        if autoclean:
+            threading.Thread(
+                target=_clean_up_thread,
+                daemon=True,
+            ).start()
+
+    def get_output_device_id(self, name_contains: str) -> int:
+        """Return the output device id whose name contains the given string (case-insensitive).
+
+        If not found, return the default output device id.
+        """
+        devices = sd.query_devices()
+
+        for idx, dev in enumerate(devices):
+            if (
+                name_contains.lower() in dev["name"].lower()
+                and dev["max_output_channels"] > 0
+            ):
+                return idx
+        # Return default output device if not found
+        self.logger.warning(
+            f"No output device found containing '{name_contains}', using default."
+        )
+        return self._safe_query_device('output')
+
+    def get_input_device_id(self, name_contains: str) -> int:
+        """Return the input device id whose name contains the given string (case-insensitive).
+
+        If not found, return the default input device id.
+        """
+        devices = sd.query_devices()
+
+        for idx, dev in enumerate(devices):
+            if (
+                name_contains.lower() in dev["name"].lower()
+                and dev["max_input_channels"] > 0
+            ):
+                return idx
+        # Return default input device if not found
+        self.logger.warning(
+            f"No input device found containing '{name_contains}', using default."
+        )
+        return self._safe_query_device('input')
+
+    def _safe_query_device(self, kind: str) -> int:
+        try:
+            return int(sd.query_devices(None, kind)['index'])
+        except sd.PortAudioError:
+            return int(sd.default.device[1])

reachy_mini/media/audio_utils.py

@@ -0,0 +1,27 @@
+"""Utility functions for audio handling, specifically for detecting the ReSpeaker sound card."""
+
+import logging
+import subprocess
+
+
+def get_respeaker_card_number() -> int:
+    """Return the card number of the ReSpeaker sound card, or 0 if not found."""
+    try:
+        result = subprocess.run(
+            ["arecord", "-l"], capture_output=True, text=True, check=True
+        )
+        output = result.stdout
+
+        lines = output.split("\n")
+        for line in lines:
+            if "ReSpeaker" in line and "card" in line:
+                card_number = line.split(":")[0].split("card ")[1].strip()
+                logging.debug(f"Found ReSpeaker sound card: {card_number}")
+                return int(card_number)
+
+        logging.warning("ReSpeaker sound card not found. Returning default card")
+        return 0  # default sound card
+
+    except subprocess.CalledProcessError as e:
+        logging.error(f"Cannot find sound card: {e}")
+        return 0

reachy_mini/media/camera_base.py

@@ -0,0 +1,63 @@
+"""Base classes for camera implementations.
+
+The camera implementations support various backends and provide a unified
+interface for capturing images.
+"""
+
+import logging
+from abc import ABC, abstractmethod
+from enum import Enum
+from typing import Optional
+
+import numpy as np
+import numpy.typing as npt
+
+from reachy_mini.media.camera_constants import CameraResolution
+
+
+class CameraBackend(Enum):
+    """Camera backends."""
+
+    OPENCV = "opencv"
+    GSTREAMER = "gstreamer"
+
+
+class CameraBase(ABC):
+    """Abstract class for opening and managing a camera."""
+
+    def __init__(
+        self,
+        backend: CameraBackend,
+        log_level: str = "INFO",
+        resolution: CameraResolution = CameraResolution.R1280x720,
+    ) -> None:
+        """Initialize the camera."""
+        self.logger = logging.getLogger(__name__)
+        self.logger.setLevel(log_level)
+        self.backend = backend
+        self._resolution = resolution
+
+    @property
+    def resolution(self) -> tuple[int, int]:
+        """Get the current camera resolution as a tuple (width, height)."""
+        return (self._resolution.value[0], self._resolution.value[1])
+
+    @property
+    def framerate(self) -> int:
+        """Get the current camera frames per second."""
+        return self._resolution.value[2]
+
+    @abstractmethod
+    def open(self) -> None:
+        """Open the camera."""
+        pass
+
+    @abstractmethod
+    def read(self) -> Optional[npt.NDArray[np.uint8]]:
+        """Read an image from the camera. Returns the image or None if error."""
+        pass
+
+    @abstractmethod
+    def close(self) -> None:
+        """Close the camera and release resources."""
+        pass

reachy_mini/media/camera_constants.py

@@ -0,0 +1,13 @@
+"""Camera constants for Reachy Mini."""
+
+from enum import Enum
+
+
+class CameraResolution(Enum):
+    """Camera resolutions. Arducam_12MP."""
+
+    R2304x1296 = (2304, 1296, 30)
+    R4608x2592 = (4608, 2592, 10)
+    R1920x1080 = (1920, 1080, 30)
+    R1600x1200 = (1600, 1200, 30)
+    R1280x720 = (1280, 720, 30)