reachy-mini 1.2.5rc1__py3-none-any.whl

reachy_mini/media/audio_gstreamer.py

@@ -0,0 +1,272 @@
+"""GStreamer camera backend.
+
+This module provides an implementation of the CameraBase class using GStreamer.
+By default the module directly returns JPEG images as output by the camera.
+"""
+
+import os
+from threading import Thread
+from typing import Optional
+
+import numpy as np
+import numpy.typing as npt
+
+from reachy_mini.media.audio_utils import (
+    get_respeaker_card_number,
+    has_reachymini_asoundrc,
+)
+from reachy_mini.utils.constants import ASSETS_ROOT_PATH
+
+try:
+    import gi
+except ImportError as e:
+    raise ImportError(
+        "The 'gi' module is required for GStreamerCamera but could not be imported. \
+                      Please install the GStreamer backend: pip install .[gstreamer]."
+    ) from e
+
+gi.require_version("Gst", "1.0")
+gi.require_version("GstApp", "1.0")
+
+
+from gi.repository import GLib, Gst, GstApp  # noqa: E402
+
+from .audio_base import AudioBase  # noqa: E402
+
+
+class GStreamerAudio(AudioBase):
+    """Audio implementation using GStreamer."""
+
+    def __init__(self, log_level: str = "INFO") -> None:
+        """Initialize the GStreamer audio."""
+        super().__init__(log_level=log_level)
+        Gst.init(None)
+        self._loop = GLib.MainLoop()
+        self._thread_bus_calls = Thread(target=lambda: self._loop.run(), daemon=True)
+        self._thread_bus_calls.start()
+
+        self._id_audio_card = get_respeaker_card_number()
+
+        self._pipeline_record = Gst.Pipeline.new("audio_recorder")
+        self._appsink_audio: Optional[GstApp] = None
+        self._init_pipeline_record(self._pipeline_record)
+        self._bus_record = self._pipeline_record.get_bus()
+        self._bus_record.add_watch(
+            GLib.PRIORITY_DEFAULT, self._on_bus_message, self._loop
+        )
+
+        self._pipeline_playback = Gst.Pipeline.new("audio_player")
+        self._appsrc: Optional[GstApp] = None
+        self._init_pipeline_playback(self._pipeline_playback)
+        self._bus_playback = self._pipeline_playback.get_bus()
+        self._bus_playback.add_watch(
+            GLib.PRIORITY_DEFAULT, self._on_bus_message, self._loop
+        )
+
+    def _init_pipeline_record(self, pipeline: Gst.Pipeline) -> None:
+        self._appsink_audio = Gst.ElementFactory.make("appsink")
+        caps = Gst.Caps.from_string(
+            f"audio/x-raw,rate={self.SAMPLE_RATE},channels={self.CHANNELS},format=F32LE,layout=interleaved"
+        )
+        self._appsink_audio.set_property("caps", caps)
+        self._appsink_audio.set_property("drop", True)  # avoid overflow
+        self._appsink_audio.set_property("max-buffers", 200)
+
+        audiosrc: Optional[Gst.Element] = None
+        if self._id_audio_card == -1:
+            audiosrc = Gst.ElementFactory.make("autoaudiosrc")  # use default mic
+        elif has_reachymini_asoundrc():
+            # reachy mini wireless has a preconfigured asoundrc
+            audiosrc = Gst.ElementFactory.make("alsasrc")
+            audiosrc.set_property("device", "reachymini_audio_src")
+        else:
+            audiosrc = Gst.ElementFactory.make("alsasrc")
+            audiosrc.set_property("device", f"hw:{self._id_audio_card},0")
+
+        queue = Gst.ElementFactory.make("queue")
+        audioconvert = Gst.ElementFactory.make("audioconvert")
+        audioresample = Gst.ElementFactory.make("audioresample")
+
+        if not all([audiosrc, queue, audioconvert, audioresample, self._appsink_audio]):
+            raise RuntimeError("Failed to create GStreamer elements")
+
+        pipeline.add(audiosrc)
+        pipeline.add(queue)
+        pipeline.add(audioconvert)
+        pipeline.add(audioresample)
+        pipeline.add(self._appsink_audio)
+
+        audiosrc.link(queue)
+        queue.link(audioconvert)
+        audioconvert.link(audioresample)
+        audioresample.link(self._appsink_audio)
+
+    def __del__(self) -> None:
+        """Destructor to ensure gstreamer resources are released."""
+        super().__del__()
+        self._loop.quit()
+        self._bus_record.remove_watch()
+        self._bus_playback.remove_watch()
+
+    def _init_pipeline_playback(self, pipeline: Gst.Pipeline) -> None:
+        self._appsrc = Gst.ElementFactory.make("appsrc")
+        self._appsrc.set_property("format", Gst.Format.TIME)
+        self._appsrc.set_property("is-live", True)
+        caps = Gst.Caps.from_string(
+            f"audio/x-raw,format=F32LE,channels={self.CHANNELS},rate={self.SAMPLE_RATE},layout=interleaved"
+        )
+        self._appsrc.set_property("caps", caps)
+
+        audioconvert = Gst.ElementFactory.make("audioconvert")
+        audioresample = Gst.ElementFactory.make("audioresample")
+
+        queue = Gst.ElementFactory.make("queue")
+        audiosink: Optional[Gst.Element] = None
+        if self._id_audio_card == -1:
+            audiosink = Gst.ElementFactory.make("autoaudiosink")  # use default speaker
+        elif has_reachymini_asoundrc():
+            # reachy mini wireless has a preconfigured asoundrc
+            audiosink = Gst.ElementFactory.make("alsasink")
+            audiosink.set_property("device", "reachymini_audio_sink")
+        else:
+            audiosink = Gst.ElementFactory.make("alsasink")
+            audiosink.set_property("device", f"hw:{self._id_audio_card},0")
+
+        pipeline.add(queue)
+        pipeline.add(audiosink)
+        pipeline.add(self._appsrc)
+        pipeline.add(audioconvert)
+        pipeline.add(audioresample)
+
+        self._appsrc.link(queue)
+        queue.link(audioconvert)
+        audioconvert.link(audioresample)
+        audioresample.link(audiosink)
+
+    def _on_bus_message(self, bus: Gst.Bus, msg: Gst.Message, loop) -> bool:  # type: ignore[no-untyped-def]
+        t = msg.type
+        if t == Gst.MessageType.EOS:
+            self.logger.warning("End-of-stream")
+            return False
+
+        elif t == Gst.MessageType.ERROR:
+            err, debug = msg.parse_error()
+            self.logger.error(f"Error: {err} {debug}")
+            return False
+
+        return True
+
+    def start_recording(self) -> None:
+        """Open the audio card using GStreamer."""
+        self._pipeline_record.set_state(Gst.State.PLAYING)
+
+    def _get_sample(self, appsink: GstApp.AppSink) -> Optional[bytes]:
+        sample = appsink.try_pull_sample(20_000_000)
+        if sample is None:
+            return None
+        data = None
+        if isinstance(sample, Gst.Sample):
+            buf = sample.get_buffer()
+            if buf is None:
+                self.logger.warning("Buffer is None")
+
+            data = buf.extract_dup(0, buf.get_size())
+        return data
+
+    def get_audio_sample(self) -> Optional[npt.NDArray[np.float32]]:
+        """Read a sample from the audio card. Returns the sample or None if error.
+
+        Returns:
+            Optional[npt.NDArray[np.float32]]: The captured sample in raw format, or None if error.
+
+        """
+        sample = self._get_sample(self._appsink_audio)
+        if sample is None:
+            return None
+        return np.frombuffer(sample, dtype=np.float32).reshape(-1, 2)
+
+    def get_input_audio_samplerate(self) -> int:
+        """Get the input samplerate of the audio device."""
+        return self.SAMPLE_RATE
+
+    def get_output_audio_samplerate(self) -> int:
+        """Get the output samplerate of the audio device."""
+        return self.SAMPLE_RATE
+
+    def get_input_channels(self) -> int:
+        """Get the number of input channels of the audio device."""
+        return self.CHANNELS
+
+    def get_output_channels(self) -> int:
+        """Get the number of output channels of the audio device."""
+        return self.CHANNELS
+
+    def stop_recording(self) -> None:
+        """Release the camera resource."""
+        self._pipeline_record.set_state(Gst.State.NULL)
+
+    def start_playing(self) -> None:
+        """Open the audio output using GStreamer."""
+        self._pipeline_playback.set_state(Gst.State.PLAYING)
+
+    def stop_playing(self) -> None:
+        """Stop playing audio and release resources."""
+        self._pipeline_playback.set_state(Gst.State.NULL)
+
+    def push_audio_sample(self, data: npt.NDArray[np.float32]) -> None:
+        """Push audio data to the output device."""
+        if self._appsrc is not None:
+            buf = Gst.Buffer.new_wrapped(data.tobytes())
+            self._appsrc.push_buffer(buf)
+        else:
+            self.logger.warning(
+                "AppSrc is not initialized. Call start_playing() first."
+            )
+
+    def play_sound(self, sound_file: str) -> None:
+        """Play a sound file.
+
+        Todo: for now this function is mean to be used on the wireless version.
+
+        Args:
+            sound_file (str): Path to the sound file to play.
+
+        """
+        if not os.path.exists(sound_file):
+            file_path = f"{ASSETS_ROOT_PATH}/{sound_file}"
+            if not os.path.exists(file_path):
+                raise FileNotFoundError(
+                    f"Sound file {sound_file} not found in assets directory or given path."
+                )
+        else:
+            file_path = sound_file
+
+        audiosink: Optional[Gst.Element] = None
+
+        if has_reachymini_asoundrc():
+            # reachy mini wireless has a preconfigured asoundrc
+            audiosink = Gst.ElementFactory.make("alsasink")
+            audiosink.set_property("device", "reachymini_audio_sink")
+
+        playbin = Gst.ElementFactory.make("playbin", "player")
+        if not playbin:
+            self.logger.error("Failed to create playbin element")
+            return
+        playbin.set_property("uri", f"file://{file_path}")
+        if audiosink is not None:
+            playbin.set_property("audio-sink", audiosink)
+
+        playbin.set_state(Gst.State.PLAYING)
+
+    def clear_player(self) -> None:
+        """Flush the player's appsrc to drop any queued audio immediately."""
+        if self._appsrc is not None:
+            self._pipeline_playback.set_state(Gst.State.PAUSED)
+            self._appsrc.send_event(Gst.Event.new_flush_start())
+            self._appsrc.send_event(Gst.Event.new_flush_stop(reset_time=True))
+            self._pipeline_playback.set_state(Gst.State.PLAYING)
+            self.logger.info("Cleared player queue")
+        else:
+            self.logger.warning(
+                "AppSrc is not initialized. Call start_playing() first."
+            )

reachy_mini/media/audio_sounddevice.py

@@ -0,0 +1,321 @@
+"""Audio implementation using sounddevice backend."""
+
+import os
+import threading
+from collections import deque
+from typing import Deque, List, Optional
+
+import numpy as np
+import numpy.typing as npt
+import scipy
+import sounddevice as sd
+import soundfile as sf
+
+from reachy_mini.utils.constants import ASSETS_ROOT_PATH
+
+from .audio_base import AudioBase
+
+MAX_INPUT_CHANNELS = 4
+MAX_INPUT_QUEUE_SECONDS = 60.0
+
+
+class SoundDeviceAudio(AudioBase):
+    """Audio device implementation using sounddevice."""
+
+    def __init__(
+        self,
+        log_level: str = "INFO",
+    ) -> None:
+        """Initialize the SoundDevice audio device."""
+        super().__init__(log_level=log_level)
+        self._input_stream = None
+        self._output_stream = None
+        self._input_lock = threading.Lock()
+        self._output_lock = threading.Lock()
+        self._input_buffer: Deque[npt.NDArray[np.float32]] = deque()
+        self._output_buffer: List[npt.NDArray[np.float32]] = []
+        self._input_max_queue_seconds: float = MAX_INPUT_QUEUE_SECONDS
+        self._input_queued_samples: int = 0
+
+        self._output_device_id = self._get_device_id(
+            ["Reachy Mini Audio", "respeaker"], device_io_type="output"
+        )
+        self._input_device_id = self._get_device_id(
+            ["Reachy Mini Audio", "respeaker"], device_io_type="input"
+        )
+
+        self._logs = {
+            "input_underflows": 0,
+            "input_overflows": 0,
+        }
+
+    @property
+    def _input_max_queue_samples(self) -> int:
+        return int(self._input_max_queue_seconds * self.get_input_audio_samplerate())
+
+    @property
+    def _is_recording(self) -> bool:
+        return self._input_stream is not None and self._input_stream.active
+
+    def start_recording(self) -> None:
+        """Open the audio input stream, using ReSpeaker card if available."""
+        if self._is_recording:
+            self.stop_recording()
+
+        self._input_stream = sd.InputStream(
+            device=self._input_device_id,
+            samplerate=self.get_input_audio_samplerate(),
+            callback=self._input_callback,
+        )
+        if self._input_stream is None:
+            raise RuntimeError("Failed to open SoundDevice audio input stream.")
+
+        self._input_buffer.clear()
+        self._input_queued_samples = 0
+        self._input_stream.start()
+        self.logger.info("SoundDevice audio input stream opened.")
+
+    def _input_callback(
+        self,
+        indata: npt.NDArray[np.float32],
+        frames: int,
+        time: int,
+        status: sd.CallbackFlags,
+    ) -> None:
+        if status and status.input_underflow:
+            self._logs["input_underflows"] += 1
+            if self._logs["input_underflows"] % 10 == 1:
+                self.logger.debug(
+                    f"Audio input underflow count: {self._logs['input_underflows']}"
+                )
+
+        with self._input_lock:
+            if (
+                self._input_queued_samples + indata.shape[0]
+                > self._input_max_queue_samples
+            ):
+                while (
+                    self._input_queued_samples + indata.shape[0]
+                    > self._input_max_queue_samples
+                    and len(self._input_buffer) > 0
+                ):
+                    dropped = self._input_buffer.popleft()
+                    self._input_queued_samples -= dropped.shape[0]
+                self._logs["input_overflows"] += 1
+                self.logger.warning(
+                    "Audio input buffer overflowed, dropped old chunks !"
+                )
+            self._input_buffer.append(indata[:, :MAX_INPUT_CHANNELS].copy())
+            self._input_queued_samples += indata.shape[0]
+
+    def get_audio_sample(self) -> Optional[npt.NDArray[np.float32]]:
+        """Read audio data from the buffer. Returns numpy array or None if empty."""
+        with self._input_lock:
+            if self._input_buffer and len(self._input_buffer) > 0:
+                data: npt.NDArray[np.float32] = np.concatenate(
+                    self._input_buffer, axis=0
+                )
+                self._input_buffer.clear()
+                self._input_queued_samples = 0
+                return data
+        self.logger.debug("No audio data available in buffer.")
+        return None
+
+    def get_input_audio_samplerate(self) -> int:
+        """Get the input samplerate of the audio device."""
+        return int(
+            sd.query_devices(self._input_device_id, "input")["default_samplerate"]
+        )
+
+    def get_output_audio_samplerate(self) -> int:
+        """Get the output samplerate of the audio device."""
+        return int(
+            sd.query_devices(self._output_device_id, "output")["default_samplerate"]
+        )
+
+    def get_input_channels(self) -> int:
+        """Get the number of input channels of the audio device."""
+        return min(
+            int(sd.query_devices(self._input_device_id, "input")["max_input_channels"]),
+            MAX_INPUT_CHANNELS,
+        )
+
+    def get_output_channels(self) -> int:
+        """Get the number of output channels of the audio device."""
+        return int(
+            sd.query_devices(self._output_device_id, "output")["max_output_channels"]
+        )
+
+    def stop_recording(self) -> None:
+        """Close the audio stream and release resources."""
+        if self._is_recording:
+            self._input_stream.stop()  # type: ignore[attr-defined]
+            self._input_stream.close()  # type: ignore[attr-defined]
+            self._input_stream = None
+            self.logger.info("SoundDevice audio stream closed.")
+
+    def push_audio_sample(self, data: npt.NDArray[np.float32]) -> None:
+        """Push audio data to the output device."""
+        if self._output_stream is not None:
+            with self._output_lock:
+                self._output_buffer.append(data.copy())
+        else:
+            self.logger.warning(
+                "Output stream is not open. Call start_playing() first."
+            )
+
+    def clear_output_buffer(self) -> None:
+        """Clear the output buffer."""
+        with self._output_lock:
+            self._output_buffer.clear()
+
+    def start_playing(self) -> None:
+        """Open the audio output stream."""
+        self.clear_output_buffer()
+
+        if self._output_stream is not None:
+            self.stop_playing()
+        self._output_stream = sd.OutputStream(
+            samplerate=self.get_output_audio_samplerate(),
+            device=self._output_device_id,
+            callback=self._output_callback,
+        )
+        if self._output_stream is None:
+            raise RuntimeError("Failed to open SoundDevice audio output stream.")
+        self._output_stream.start()
+        self.logger.info("SoundDevice audio output stream opened.")
+
+    def _output_callback(
+        self,
+        outdata: npt.NDArray[np.float32],
+        frames: int,
+        time: int,
+        status: sd.CallbackFlags,
+    ) -> None:
+        """Handle audio output stream callback."""
+        if status:
+            self.logger.warning(f"SoundDevice output status: {status}")
+        
+        with self._output_lock:
+            filled = 0
+            while filled < frames and self._output_buffer:
+                chunk = self._output_buffer[0]
+                
+                needed = frames - filled
+                available = len(chunk)
+                take = min(needed, available)
+                
+                outdata[filled:filled + take] = chunk[:take]
+                filled += take
+                
+                if take < available:
+                    # Partial consumption, keep remainder
+                    self._output_buffer[0] = chunk[take:]
+                else:
+                    # Fully consumed this chunk
+                    self._output_buffer.pop(0)
+            
+            # Only pad with zeros if buffer is truly empty
+            if filled < frames:
+                outdata[filled:] = 0
+
+    def ensure_chunk_shape(
+        self, chunk: npt.NDArray[np.float32], target_shape: tuple[int, ...]
+    ) -> npt.NDArray[np.float32]:
+        """Ensure chunk has the shape (frames, num_channels) as required by outdata.
+
+        - If chunk is 1D, tile to required num_channels.
+        - If chunk is 2D with mismatched channels, use column 0.
+        - If chunk is already correct, return as-is.
+        """
+        num_channels = target_shape[1] if len(target_shape) > 1 else 1
+        if chunk.ndim == 1:
+            return np.tile(chunk[:, None], (1, num_channels))
+        elif chunk.shape[1] != num_channels:
+            # Broadcast first channel only
+            return np.tile(chunk[:, [0]], (1, num_channels))
+        return chunk
+
+    def stop_playing(self) -> None:
+        """Close the audio output stream."""
+        if self._output_stream is not None:
+            self._output_stream.stop()
+            self._output_stream.close()
+            self._output_stream = None
+            self.clear_output_buffer()
+            self.logger.info("SoundDevice audio output stream closed.")
+
+    def play_sound(self, sound_file: str) -> None:
+        """Play a sound file.
+
+        Args:
+            sound_file (str): Path to the sound file to play. May be given relative to the assets directory or as an absolute path.
+
+        """
+        if not os.path.exists(sound_file):
+            file_path = f"{ASSETS_ROOT_PATH}/{sound_file}"
+            if not os.path.exists(file_path):
+                raise FileNotFoundError(
+                    f"Sound file {sound_file} not found in assets directory or given path."
+                )
+        else:
+            file_path = sound_file
+
+        data, samplerate_in = sf.read(file_path, dtype="float32")
+        samplerate_out = self.get_output_audio_samplerate()
+
+        if samplerate_in != samplerate_out:
+            data = scipy.signal.resample(
+                data, int(len(data) * (samplerate_out / samplerate_in))
+            )
+        data = self.ensure_chunk_shape(data, (-1, self.get_output_channels()))
+
+        self.logger.debug(f"Playing sound '{file_path}' at {samplerate_in} Hz")
+
+        if self._output_stream is not None:
+            self.push_audio_sample(data)
+        else:
+            self.logger.warning(
+                "Output stream wasn't open. We are opening it and leaving it open."
+            )
+            self.start_playing()
+            self.push_audio_sample(data)
+
+    def _get_device_id(
+        self, names_contains: List[str], device_io_type: str = "output"
+    ) -> int:
+        """Return the output device id whose name contains the given strings (case-insensitive).
+
+        Args:
+            names_contains (List[str]): List of strings that should be contained in the device name.
+            device_io_type (str): 'input' or 'output' to specify device type.
+
+        If not found, return the default output device id.
+
+        """
+        devices = sd.query_devices()
+
+        for idx, dev in enumerate(devices):
+            for name_contains in names_contains:
+                if (
+                    name_contains.lower() in dev["name"].lower()
+                    and dev[f"max_{device_io_type}_channels"] > 0
+                ):
+                    return idx
+        # Return default output device if not found
+        self.logger.warning(
+            f"No {device_io_type} device found containing '{names_contains}', using default."
+        )
+        return self._safe_query_device(device_io_type)
+
+    def _safe_query_device(self, kind: str) -> int:
+        try:
+            return int(sd.query_devices(None, kind)["index"])
+        except sd.PortAudioError:
+            return (
+                int(sd.default.device[1])
+                if kind == "input"
+                else int(sd.default.device[0])
+            )
+        except IndexError:
+            return 0