reachy-mini 1.2.5rc1__py3-none-any.whl → 1.2.11__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- reachy_mini/apps/app.py +24 -21
- reachy_mini/apps/manager.py +17 -3
- reachy_mini/apps/sources/hf_auth.py +92 -0
- reachy_mini/apps/sources/hf_space.py +1 -1
- reachy_mini/apps/sources/local_common_venv.py +199 -24
- reachy_mini/apps/templates/main.py.j2 +4 -3
- reachy_mini/daemon/app/dashboard/static/js/apps.js +9 -1
- reachy_mini/daemon/app/dashboard/static/js/appstore.js +228 -0
- reachy_mini/daemon/app/dashboard/static/js/logs.js +148 -0
- reachy_mini/daemon/app/dashboard/templates/logs.html +37 -0
- reachy_mini/daemon/app/dashboard/templates/sections/appstore.html +92 -0
- reachy_mini/daemon/app/dashboard/templates/sections/cache.html +82 -0
- reachy_mini/daemon/app/dashboard/templates/sections/daemon.html +5 -0
- reachy_mini/daemon/app/dashboard/templates/settings.html +1 -0
- reachy_mini/daemon/app/main.py +172 -7
- reachy_mini/daemon/app/models.py +8 -0
- reachy_mini/daemon/app/routers/apps.py +56 -0
- reachy_mini/daemon/app/routers/cache.py +58 -0
- reachy_mini/daemon/app/routers/hf_auth.py +57 -0
- reachy_mini/daemon/app/routers/logs.py +124 -0
- reachy_mini/daemon/app/routers/state.py +25 -1
- reachy_mini/daemon/app/routers/wifi_config.py +75 -0
- reachy_mini/daemon/app/services/bluetooth/bluetooth_service.py +1 -1
- reachy_mini/daemon/app/services/bluetooth/commands/WIFI_RESET.sh +8 -0
- reachy_mini/daemon/app/services/wireless/launcher.sh +8 -2
- reachy_mini/daemon/app/services/wireless/reachy-mini-daemon.service +13 -0
- reachy_mini/daemon/backend/abstract.py +29 -9
- reachy_mini/daemon/backend/mockup_sim/__init__.py +12 -0
- reachy_mini/daemon/backend/mockup_sim/backend.py +176 -0
- reachy_mini/daemon/backend/mujoco/backend.py +0 -5
- reachy_mini/daemon/backend/robot/backend.py +78 -5
- reachy_mini/daemon/daemon.py +46 -7
- reachy_mini/daemon/utils.py +71 -15
- reachy_mini/io/zenoh_client.py +26 -0
- reachy_mini/io/zenoh_server.py +10 -6
- reachy_mini/kinematics/nn_kinematics.py +2 -2
- reachy_mini/kinematics/placo_kinematics.py +15 -15
- reachy_mini/media/__init__.py +55 -1
- reachy_mini/media/audio_base.py +185 -13
- reachy_mini/media/audio_control_utils.py +60 -5
- reachy_mini/media/audio_gstreamer.py +97 -16
- reachy_mini/media/audio_sounddevice.py +120 -19
- reachy_mini/media/audio_utils.py +110 -5
- reachy_mini/media/camera_base.py +182 -11
- reachy_mini/media/camera_constants.py +132 -4
- reachy_mini/media/camera_gstreamer.py +42 -2
- reachy_mini/media/camera_opencv.py +83 -5
- reachy_mini/media/camera_utils.py +95 -7
- reachy_mini/media/media_manager.py +139 -6
- reachy_mini/media/webrtc_client_gstreamer.py +142 -13
- reachy_mini/media/webrtc_daemon.py +72 -7
- reachy_mini/motion/recorded_move.py +76 -2
- reachy_mini/reachy_mini.py +196 -40
- reachy_mini/tools/reflash_motors.py +1 -1
- reachy_mini/tools/scan_motors.py +86 -0
- reachy_mini/tools/setup_motor.py +49 -31
- reachy_mini/utils/interpolation.py +1 -1
- reachy_mini/utils/wireless_version/startup_check.py +278 -21
- reachy_mini/utils/wireless_version/update.py +44 -1
- {reachy_mini-1.2.5rc1.dist-info → reachy_mini-1.2.11.dist-info}/METADATA +7 -6
- {reachy_mini-1.2.5rc1.dist-info → reachy_mini-1.2.11.dist-info}/RECORD +65 -53
- {reachy_mini-1.2.5rc1.dist-info → reachy_mini-1.2.11.dist-info}/WHEEL +0 -0
- {reachy_mini-1.2.5rc1.dist-info → reachy_mini-1.2.11.dist-info}/entry_points.txt +0 -0
- {reachy_mini-1.2.5rc1.dist-info → reachy_mini-1.2.11.dist-info}/licenses/LICENSE +0 -0
- {reachy_mini-1.2.5rc1.dist-info → reachy_mini-1.2.11.dist-info}/top_level.txt +0 -0
|
@@ -1,6 +1,37 @@
|
|
|
1
1
|
"""Media Manager.
|
|
2
2
|
|
|
3
|
-
Provides camera and audio access based on the selected
|
|
3
|
+
Provides camera and audio access based on the selected backend.
|
|
4
|
+
|
|
5
|
+
This module offers a unified interface for managing both camera and audio
|
|
6
|
+
devices with support for multiple backends. It simplifies the process of
|
|
7
|
+
initializing, configuring, and using media devices across different
|
|
8
|
+
platforms and use cases.
|
|
9
|
+
|
|
10
|
+
Available backends:
|
|
11
|
+
- NO_MEDIA: No media devices (useful for headless operation)
|
|
12
|
+
- DEFAULT: OpenCV + SoundDevice (cross-platform default)
|
|
13
|
+
- DEFAULT_NO_VIDEO: SoundDevice only (audio without video)
|
|
14
|
+
- GSTREAMER: GStreamer-based media (advanced features)
|
|
15
|
+
- GSTREAMER_NO_VIDEO: GStreamer audio only
|
|
16
|
+
- WEBRTC: WebRTC-based media for real-time communication
|
|
17
|
+
|
|
18
|
+
Example usage:
|
|
19
|
+
>>> from reachy_mini.media.media_manager import MediaManager, MediaBackend
|
|
20
|
+
>>>
|
|
21
|
+
>>> # Initialize with default backend
|
|
22
|
+
>>> media = MediaManager(backend=MediaBackend.DEFAULT)
|
|
23
|
+
>>>
|
|
24
|
+
>>> # Capture a frame
|
|
25
|
+
>>> frame = media.get_frame()
|
|
26
|
+
>>> if frame is not None:
|
|
27
|
+
... cv2.imshow("Frame", frame)
|
|
28
|
+
... cv2.waitKey(1)
|
|
29
|
+
>>>
|
|
30
|
+
>>> # Play a sound
|
|
31
|
+
>>> media.play_sound("/path/to/sound.wav")
|
|
32
|
+
>>>
|
|
33
|
+
>>> # Clean up
|
|
34
|
+
>>> media.close()
|
|
4
35
|
"""
|
|
5
36
|
|
|
6
37
|
import logging
|
|
@@ -17,7 +48,25 @@ from reachy_mini.media.camera_base import CameraBase
|
|
|
17
48
|
|
|
18
49
|
|
|
19
50
|
class MediaBackend(Enum):
|
|
20
|
-
"""Media backends.
|
|
51
|
+
"""Media backends.
|
|
52
|
+
|
|
53
|
+
Enumeration of available media backends that can be used with MediaManager.
|
|
54
|
+
Each backend provides different capabilities and performance characteristics.
|
|
55
|
+
|
|
56
|
+
Attributes:
|
|
57
|
+
NO_MEDIA: No media devices - useful for headless operation or when
|
|
58
|
+
media devices are not needed.
|
|
59
|
+
DEFAULT: Default backend using OpenCV for video and SoundDevice for audio.
|
|
60
|
+
Cross-platform and widely compatible.
|
|
61
|
+
DEFAULT_NO_VIDEO: SoundDevice audio only - for audio processing without video.
|
|
62
|
+
GSTREAMER: GStreamer-based media backend with advanced video and audio
|
|
63
|
+
processing capabilities.
|
|
64
|
+
GSTREAMER_NO_VIDEO: GStreamer audio only - for advanced audio processing
|
|
65
|
+
without video.
|
|
66
|
+
WEBRTC: WebRTC-based media backend for real-time communication and
|
|
67
|
+
streaming applications.
|
|
68
|
+
|
|
69
|
+
"""
|
|
21
70
|
|
|
22
71
|
NO_MEDIA = "no_media"
|
|
23
72
|
DEFAULT = "default"
|
|
@@ -28,7 +77,19 @@ class MediaBackend(Enum):
|
|
|
28
77
|
|
|
29
78
|
|
|
30
79
|
class MediaManager:
|
|
31
|
-
"""
|
|
80
|
+
"""Media Manager for handling camera and audio devices.
|
|
81
|
+
|
|
82
|
+
This class provides a unified interface for managing both camera and audio
|
|
83
|
+
devices across different backends. It handles initialization, configuration,
|
|
84
|
+
and cleanup of media resources.
|
|
85
|
+
|
|
86
|
+
Attributes:
|
|
87
|
+
logger (logging.Logger): Logger instance for media-related messages.
|
|
88
|
+
backend (MediaBackend): The selected media backend.
|
|
89
|
+
camera (Optional[CameraBase]): Camera device instance.
|
|
90
|
+
audio (Optional[AudioBase]): Audio device instance.
|
|
91
|
+
|
|
92
|
+
"""
|
|
32
93
|
|
|
33
94
|
def __init__(
|
|
34
95
|
self,
|
|
@@ -37,7 +98,28 @@ class MediaManager:
|
|
|
37
98
|
use_sim: bool = False,
|
|
38
99
|
signalling_host: str = "localhost",
|
|
39
100
|
) -> None:
|
|
40
|
-
"""Initialize the
|
|
101
|
+
"""Initialize the media manager.
|
|
102
|
+
|
|
103
|
+
Args:
|
|
104
|
+
backend (MediaBackend): The media backend to use. Default is DEFAULT.
|
|
105
|
+
log_level (str): Logging level for media operations.
|
|
106
|
+
Options: 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'.
|
|
107
|
+
Default: 'INFO'.
|
|
108
|
+
use_sim (bool): Whether to use simulation mode (for testing).
|
|
109
|
+
Default: False.
|
|
110
|
+
signalling_host (str): Host address for WebRTC signalling server.
|
|
111
|
+
Only used with WEBRTC backend.
|
|
112
|
+
Default: 'localhost'.
|
|
113
|
+
|
|
114
|
+
Note:
|
|
115
|
+
The constructor initializes the selected media backend and sets up
|
|
116
|
+
the appropriate camera and audio devices based on the backend choice.
|
|
117
|
+
|
|
118
|
+
Example:
|
|
119
|
+
>>> media = MediaManager(backend=MediaBackend.DEFAULT)
|
|
120
|
+
>>> media = MediaManager(backend=MediaBackend.GSTREAMER, log_level="DEBUG")
|
|
121
|
+
|
|
122
|
+
"""
|
|
41
123
|
self.logger = logging.getLogger(__name__)
|
|
42
124
|
self.logger.setLevel(log_level)
|
|
43
125
|
self.backend = backend
|
|
@@ -69,7 +151,26 @@ class MediaManager:
|
|
|
69
151
|
raise NotImplementedError(f"Media backend {backend} not implemented.")
|
|
70
152
|
|
|
71
153
|
def close(self) -> None:
|
|
72
|
-
"""Close the media manager and release resources.
|
|
154
|
+
"""Close the media manager and release resources.
|
|
155
|
+
|
|
156
|
+
This method should be called when the media manager is no longer needed
|
|
157
|
+
to properly clean up and release all media resources. It stops any ongoing
|
|
158
|
+
audio recording/playback and closes the camera device.
|
|
159
|
+
|
|
160
|
+
Note:
|
|
161
|
+
After calling this method, the media manager can be reused by calling
|
|
162
|
+
the appropriate initialization methods again, but it's generally
|
|
163
|
+
recommended to create a new MediaManager instance if needed.
|
|
164
|
+
|
|
165
|
+
Example:
|
|
166
|
+
>>> media = MediaManager()
|
|
167
|
+
>>> try:
|
|
168
|
+
... # Use media devices
|
|
169
|
+
... frame = media.get_frame()
|
|
170
|
+
... finally:
|
|
171
|
+
... media.close()
|
|
172
|
+
|
|
173
|
+
"""
|
|
73
174
|
if self.camera is not None:
|
|
74
175
|
self.camera.close()
|
|
75
176
|
if self.audio is not None:
|
|
@@ -111,7 +212,27 @@ class MediaManager:
|
|
|
111
212
|
"""Get a frame from the camera.
|
|
112
213
|
|
|
113
214
|
Returns:
|
|
114
|
-
Optional[npt.NDArray[np.uint8]]: The captured BGR frame
|
|
215
|
+
Optional[npt.NDArray[np.uint8]]: The captured BGR frame as a numpy array
|
|
216
|
+
with shape (height, width, 3), or None if the camera is not available
|
|
217
|
+
or an error occurred.
|
|
218
|
+
|
|
219
|
+
The image is in BGR format (OpenCV convention) and can be directly
|
|
220
|
+
used with OpenCV functions or converted to RGB if needed.
|
|
221
|
+
|
|
222
|
+
Note:
|
|
223
|
+
This method returns None if the camera is not initialized or if
|
|
224
|
+
there's an error capturing the frame. Always check the return value
|
|
225
|
+
before using the frame.
|
|
226
|
+
|
|
227
|
+
Example:
|
|
228
|
+
>>> frame = media.get_frame()
|
|
229
|
+
>>> if frame is not None:
|
|
230
|
+
... # Process the frame
|
|
231
|
+
... cv2.imshow("Camera", frame)
|
|
232
|
+
... cv2.waitKey(1)
|
|
233
|
+
...
|
|
234
|
+
... # Convert to RGB if needed
|
|
235
|
+
... rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
|
|
115
236
|
|
|
116
237
|
"""
|
|
117
238
|
if self.camera is None:
|
|
@@ -279,3 +400,15 @@ class MediaManager:
|
|
|
279
400
|
self.logger.warning("Audio system is not initialized.")
|
|
280
401
|
return
|
|
281
402
|
self.audio.stop_playing()
|
|
403
|
+
|
|
404
|
+
def get_DoA(self) -> tuple[float, bool] | None:
|
|
405
|
+
"""Get the Direction of Arrival (DoA) from the microphone array.
|
|
406
|
+
|
|
407
|
+
Returns:
|
|
408
|
+
tuple[float, bool] | None: A tuple (angle_radians, speech_detected),
|
|
409
|
+
or None if the audio system is not available.
|
|
410
|
+
|
|
411
|
+
"""
|
|
412
|
+
if self.audio is None:
|
|
413
|
+
return None
|
|
414
|
+
return self.audio.get_DoA()
|
|
@@ -1,10 +1,45 @@
|
|
|
1
1
|
"""GStreamer WebRTC client implementation.
|
|
2
2
|
|
|
3
3
|
The class is a client for the webrtc server hosted on the Reachy Mini Wireless robot.
|
|
4
|
+
|
|
5
|
+
This module provides a GStreamer-based WebRTC client that implements both CameraBase
|
|
6
|
+
and AudioBase interfaces, allowing it to be used as a drop-in replacement for
|
|
7
|
+
traditional camera and audio devices in the media system.
|
|
8
|
+
|
|
9
|
+
The WebRTC client supports real-time audio and video streaming over WebRTC protocol,
|
|
10
|
+
making it suitable for remote operation and telepresence applications.
|
|
11
|
+
|
|
12
|
+
Note:
|
|
13
|
+
This class is typically used internally by the MediaManager when the WEBRTC
|
|
14
|
+
backend is selected. Direct usage is possible but usually not necessary.
|
|
15
|
+
|
|
16
|
+
Example usage via MediaManager:
|
|
17
|
+
>>> from reachy_mini.media.media_manager import MediaManager, MediaBackend
|
|
18
|
+
>>>
|
|
19
|
+
>>> # Create media manager with WebRTC backend
|
|
20
|
+
>>> media = MediaManager(
|
|
21
|
+
... backend=MediaBackend.WEBRTC,
|
|
22
|
+
... signalling_host="192.168.1.100",
|
|
23
|
+
... log_level="INFO"
|
|
24
|
+
... )
|
|
25
|
+
>>>
|
|
26
|
+
>>> # Use camera functionality
|
|
27
|
+
>>> frame = media.get_frame()
|
|
28
|
+
>>> if frame is not None:
|
|
29
|
+
... cv2.imshow("WebRTC Stream", frame)
|
|
30
|
+
... cv2.waitKey(1)
|
|
31
|
+
>>>
|
|
32
|
+
>>> # Use audio functionality
|
|
33
|
+
>>> media.start_recording()
|
|
34
|
+
>>> audio_samples = media.get_audio_sample()
|
|
35
|
+
>>>
|
|
36
|
+
>>> # Clean up
|
|
37
|
+
>>> media.close()
|
|
38
|
+
|
|
4
39
|
"""
|
|
5
40
|
|
|
6
41
|
from threading import Thread
|
|
7
|
-
from typing import Optional, cast
|
|
42
|
+
from typing import Iterator, Optional, cast
|
|
8
43
|
|
|
9
44
|
import gi
|
|
10
45
|
import numpy as np
|
|
@@ -24,7 +59,18 @@ from gi.repository import GLib, Gst, GstApp # noqa: E402, F401
|
|
|
24
59
|
|
|
25
60
|
|
|
26
61
|
class GstWebRTCClient(CameraBase, AudioBase):
|
|
27
|
-
"""GStreamer WebRTC client implementation.
|
|
62
|
+
"""GStreamer WebRTC client implementation.
|
|
63
|
+
|
|
64
|
+
This class implements a WebRTC client using GStreamer that can connect to
|
|
65
|
+
a WebRTC server (such as the one hosted on Reachy Mini Wireless) to stream
|
|
66
|
+
audio and video in real-time. It implements both CameraBase and AudioBase
|
|
67
|
+
interfaces, allowing seamless integration with the media system.
|
|
68
|
+
|
|
69
|
+
Attributes:
|
|
70
|
+
Inherits all attributes from CameraBase and AudioBase.
|
|
71
|
+
Additionally manages GStreamer pipelines for WebRTC communication.
|
|
72
|
+
|
|
73
|
+
"""
|
|
28
74
|
|
|
29
75
|
def __init__(
|
|
30
76
|
self,
|
|
@@ -33,7 +79,30 @@ class GstWebRTCClient(CameraBase, AudioBase):
|
|
|
33
79
|
signaling_host: str = "",
|
|
34
80
|
signaling_port: int = 8443,
|
|
35
81
|
):
|
|
36
|
-
"""Initialize the GStreamer WebRTC client.
|
|
82
|
+
"""Initialize the GStreamer WebRTC client.
|
|
83
|
+
|
|
84
|
+
Args:
|
|
85
|
+
log_level (str): Logging level for WebRTC operations.
|
|
86
|
+
Default: 'INFO'.
|
|
87
|
+
peer_id (str): WebRTC peer ID to connect to. Default: ''.
|
|
88
|
+
signaling_host (str): Host address of the WebRTC signaling server.
|
|
89
|
+
Default: ''.
|
|
90
|
+
signaling_port (int): Port of the WebRTC signaling server.
|
|
91
|
+
Default: 8443.
|
|
92
|
+
|
|
93
|
+
Note:
|
|
94
|
+
This constructor initializes the GStreamer environment and sets up
|
|
95
|
+
the necessary pipelines for WebRTC communication. The WebRTC connection
|
|
96
|
+
is established when open() is called.
|
|
97
|
+
|
|
98
|
+
Example:
|
|
99
|
+
>>> client = GstWebRTCClient(
|
|
100
|
+
... peer_id="reachymini",
|
|
101
|
+
... signaling_host="192.168.1.100",
|
|
102
|
+
... signaling_port=8443
|
|
103
|
+
... )
|
|
104
|
+
|
|
105
|
+
"""
|
|
37
106
|
super().__init__(log_level=log_level)
|
|
38
107
|
Gst.init(None)
|
|
39
108
|
self._loop = GLib.MainLoop()
|
|
@@ -56,7 +125,7 @@ class GstWebRTCClient(CameraBase, AudioBase):
|
|
|
56
125
|
self._pipeline_record.add(self._appsink_audio)
|
|
57
126
|
|
|
58
127
|
self.camera_specs = cast(CameraSpecs, ReachyMiniWirelessCamSpecs)
|
|
59
|
-
self.set_resolution(
|
|
128
|
+
self.set_resolution(self.camera_specs.default_resolution)
|
|
60
129
|
|
|
61
130
|
self._appsink_video = Gst.ElementFactory.make("appsink")
|
|
62
131
|
caps_video = Gst.Caps.from_string("video/x-raw,format=BGR")
|
|
@@ -104,11 +173,32 @@ class GstWebRTCClient(CameraBase, AudioBase):
|
|
|
104
173
|
signaller.set_property("uri", f"ws://{signaling_host}:{signaling_port}")
|
|
105
174
|
return source
|
|
106
175
|
|
|
176
|
+
def _iterate_gst(self, iterator: Gst.Iterator) -> Iterator[Gst.Element]:
|
|
177
|
+
"""Iterate over GStreamer iterators."""
|
|
178
|
+
while True:
|
|
179
|
+
result, elem = iterator.next()
|
|
180
|
+
if result == Gst.IteratorResult.DONE:
|
|
181
|
+
break
|
|
182
|
+
if result == Gst.IteratorResult.OK:
|
|
183
|
+
yield elem
|
|
184
|
+
elif result == Gst.IteratorResult.RESYNC:
|
|
185
|
+
iterator.resync()
|
|
186
|
+
|
|
107
187
|
def _configure_webrtcbin(self, webrtcsrc: Gst.Element) -> None:
|
|
108
188
|
if isinstance(webrtcsrc, Gst.Bin):
|
|
109
|
-
|
|
110
|
-
webrtcbin = webrtcsrc.get_by_name(
|
|
111
|
-
|
|
189
|
+
# Try to find by standard name first (fast path)
|
|
190
|
+
webrtcbin = webrtcsrc.get_by_name("webrtcbin0")
|
|
191
|
+
|
|
192
|
+
if webrtcbin is None:
|
|
193
|
+
# If not found by name (e.g. re-init scenarios), fallback to recursive search by factory type
|
|
194
|
+
self.logger.debug(f"webrtcbin0 not found, scanning elements in {webrtcsrc.get_name()} recursively...")
|
|
195
|
+
for elem in self._iterate_gst(webrtcsrc.iterate_recurse()):
|
|
196
|
+
if elem.get_factory().get_name() == "webrtcbin":
|
|
197
|
+
webrtcbin = elem
|
|
198
|
+
self.logger.debug(f"Found webrtcbin by factory search: {elem.get_name()}")
|
|
199
|
+
break
|
|
200
|
+
|
|
201
|
+
assert webrtcbin is not None, "Could not find webrtcbin element in webrtcsrc"
|
|
112
202
|
# jitterbuffer has a default 200 ms buffer. Should be ok to lower this in localnetwork config
|
|
113
203
|
webrtcbin.set_property("latency", 100)
|
|
114
204
|
|
|
@@ -166,7 +256,10 @@ class GstWebRTCClient(CameraBase, AudioBase):
|
|
|
166
256
|
return True
|
|
167
257
|
|
|
168
258
|
def open(self) -> None:
|
|
169
|
-
"""Open the video stream.
|
|
259
|
+
"""Open the video stream.
|
|
260
|
+
|
|
261
|
+
See CameraBase.open() for complete documentation.
|
|
262
|
+
"""
|
|
170
263
|
self._pipeline_record.set_state(Gst.State.PLAYING)
|
|
171
264
|
|
|
172
265
|
def _get_sample(self, appsink: GstApp.AppSink) -> Optional[bytes]:
|
|
@@ -185,6 +278,8 @@ class GstWebRTCClient(CameraBase, AudioBase):
|
|
|
185
278
|
def get_audio_sample(self) -> Optional[npt.NDArray[np.float32]]:
|
|
186
279
|
"""Read a sample from the audio card. Returns the sample or None if error.
|
|
187
280
|
|
|
281
|
+
See AudioBase.get_audio_sample() for complete documentation.
|
|
282
|
+
|
|
188
283
|
Returns:
|
|
189
284
|
Optional[npt.NDArray[np.float32]]: The captured sample in raw format, or None if error.
|
|
190
285
|
|
|
@@ -197,6 +292,8 @@ class GstWebRTCClient(CameraBase, AudioBase):
|
|
|
197
292
|
def read(self) -> Optional[npt.NDArray[np.uint8]]:
|
|
198
293
|
"""Read a frame from the camera. Returns the frame or None if error.
|
|
199
294
|
|
|
295
|
+
See CameraBase.read() for complete documentation.
|
|
296
|
+
|
|
200
297
|
Returns:
|
|
201
298
|
Optional[npt.NDArray[np.uint8]]: The captured frame in BGR format, or None if error.
|
|
202
299
|
|
|
@@ -211,16 +308,25 @@ class GstWebRTCClient(CameraBase, AudioBase):
|
|
|
211
308
|
return arr
|
|
212
309
|
|
|
213
310
|
def close(self) -> None:
|
|
214
|
-
"""Stop the pipeline.
|
|
311
|
+
"""Stop the pipeline.
|
|
312
|
+
|
|
313
|
+
See CameraBase.close() for complete documentation.
|
|
314
|
+
"""
|
|
215
315
|
# self._loop.quit()
|
|
216
316
|
self._pipeline_record.set_state(Gst.State.NULL)
|
|
217
317
|
|
|
218
318
|
def start_recording(self) -> None:
|
|
219
|
-
"""Open the audio card using GStreamer.
|
|
319
|
+
"""Open the audio card using GStreamer.
|
|
320
|
+
|
|
321
|
+
See AudioBase.start_recording() for complete documentation.
|
|
322
|
+
"""
|
|
220
323
|
pass # already started in open()
|
|
221
324
|
|
|
222
325
|
def stop_recording(self) -> None:
|
|
223
|
-
"""Release the camera resource.
|
|
326
|
+
"""Release the camera resource.
|
|
327
|
+
|
|
328
|
+
See AudioBase.stop_recording() for complete documentation.
|
|
329
|
+
"""
|
|
224
330
|
pass # managed in close()
|
|
225
331
|
|
|
226
332
|
def _init_pipeline_playback(
|
|
@@ -259,12 +365,33 @@ class GstWebRTCClient(CameraBase, AudioBase):
|
|
|
259
365
|
queue.link(rtpopuspay)
|
|
260
366
|
rtpopuspay.link(udpsink)
|
|
261
367
|
|
|
368
|
+
def set_max_output_buffers(self, max_buffers: int) -> None:
|
|
369
|
+
"""Set the maximum number of output buffers to queue in the player.
|
|
370
|
+
|
|
371
|
+
Args:
|
|
372
|
+
max_buffers (int): Maximum number of buffers to queue.
|
|
373
|
+
|
|
374
|
+
"""
|
|
375
|
+
if self._appsrc is not None:
|
|
376
|
+
self._appsrc.set_property("max-buffers", max_buffers)
|
|
377
|
+
self._appsrc.set_property("leaky-type", 2) # drop old buffers
|
|
378
|
+
else:
|
|
379
|
+
self.logger.warning(
|
|
380
|
+
"AppSrc is not initialized. Call start_playing() first."
|
|
381
|
+
)
|
|
382
|
+
|
|
262
383
|
def start_playing(self) -> None:
|
|
263
|
-
"""Open the audio output using GStreamer.
|
|
384
|
+
"""Open the audio output using GStreamer.
|
|
385
|
+
|
|
386
|
+
See AudioBase.start_playing() for complete documentation.
|
|
387
|
+
"""
|
|
264
388
|
self._pipeline_playback.set_state(Gst.State.PLAYING)
|
|
265
389
|
|
|
266
390
|
def stop_playing(self) -> None:
|
|
267
|
-
"""Stop playing audio and release resources.
|
|
391
|
+
"""Stop playing audio and release resources.
|
|
392
|
+
|
|
393
|
+
See AudioBase.stop_playing() for complete documentation.
|
|
394
|
+
"""
|
|
268
395
|
self._pipeline_playback.set_state(Gst.State.NULL)
|
|
269
396
|
|
|
270
397
|
def push_audio_sample(self, data: npt.NDArray[np.float32]) -> None:
|
|
@@ -281,6 +408,8 @@ class GstWebRTCClient(CameraBase, AudioBase):
|
|
|
281
408
|
def play_sound(self, sound_file: str) -> None:
|
|
282
409
|
"""Play a sound file.
|
|
283
410
|
|
|
411
|
+
See AudioBase.play_sound() for complete documentation.
|
|
412
|
+
|
|
284
413
|
Args:
|
|
285
414
|
sound_file (str): Path to the sound file to play.
|
|
286
415
|
|
|
@@ -1,14 +1,41 @@
|
|
|
1
1
|
"""WebRTC daemon.
|
|
2
2
|
|
|
3
3
|
Starts a gstreamer webrtc pipeline to stream video and audio.
|
|
4
|
+
|
|
5
|
+
This module provides a WebRTC server implementation using GStreamer that can
|
|
6
|
+
stream video and audio from the Reachy Mini robot to WebRTC clients. It's
|
|
7
|
+
designed to run as a daemon process on the robot and handle multiple client
|
|
8
|
+
connections for telepresence and remote monitoring applications.
|
|
9
|
+
|
|
10
|
+
The WebRTC daemon supports:
|
|
11
|
+
- Real-time video streaming from the robot's camera
|
|
12
|
+
- Real-time audio streaming from the robot's microphone
|
|
13
|
+
- Multiple client connections
|
|
14
|
+
- Automatic camera detection and configuration
|
|
15
|
+
|
|
16
|
+
Example usage:
|
|
17
|
+
>>> from reachy_mini.media.webrtc_daemon import GstWebRTC
|
|
18
|
+
>>>
|
|
19
|
+
>>> # Create and start WebRTC daemon
|
|
20
|
+
>>> webrtc_daemon = GstWebRTC(log_level="INFO")
|
|
21
|
+
>>> # The daemon will automatically start streaming when initialized
|
|
22
|
+
>>>
|
|
23
|
+
>>> # Run until interrupted
|
|
24
|
+
>>> try:
|
|
25
|
+
... while True:
|
|
26
|
+
... pass # Keep the daemon running
|
|
27
|
+
... except KeyboardInterrupt:
|
|
28
|
+
... pass # Cleanup would be handled automatically
|
|
4
29
|
"""
|
|
5
30
|
|
|
6
31
|
import logging
|
|
32
|
+
import os
|
|
7
33
|
from threading import Thread
|
|
8
34
|
from typing import Optional, Tuple, cast
|
|
9
35
|
|
|
10
36
|
import gi
|
|
11
37
|
|
|
38
|
+
from reachy_mini.daemon.utils import CAMERA_SOCKET_PATH, is_local_camera_available
|
|
12
39
|
from reachy_mini.media.camera_constants import (
|
|
13
40
|
ArducamSpecs,
|
|
14
41
|
CameraSpecs,
|
|
@@ -23,18 +50,50 @@ from gi.repository import GLib, Gst # noqa: E402
|
|
|
23
50
|
|
|
24
51
|
|
|
25
52
|
class GstWebRTC:
|
|
26
|
-
"""WebRTC pipeline using GStreamer.
|
|
53
|
+
"""WebRTC pipeline using GStreamer.
|
|
54
|
+
|
|
55
|
+
This class implements a WebRTC server using GStreamer that streams video
|
|
56
|
+
and audio from the Reachy Mini robot to connected WebRTC clients. It's
|
|
57
|
+
designed to run as a daemon process and handle the complete WebRTC
|
|
58
|
+
signaling and media streaming pipeline.
|
|
59
|
+
|
|
60
|
+
Attributes:
|
|
61
|
+
_logger (logging.Logger): Logger instance for WebRTC daemon operations.
|
|
62
|
+
_loop (GLib.MainLoop): GLib main loop for handling GStreamer events.
|
|
63
|
+
camera_specs (CameraSpecs): Specifications of the detected camera.
|
|
64
|
+
_resolution (CameraResolution): Current streaming resolution.
|
|
65
|
+
resized_K (npt.NDArray[np.float64]): Camera intrinsic matrix for current resolution.
|
|
66
|
+
|
|
67
|
+
"""
|
|
27
68
|
|
|
28
69
|
def __init__(
|
|
29
70
|
self,
|
|
30
71
|
log_level: str = "INFO",
|
|
31
72
|
) -> None:
|
|
32
|
-
"""Initialize the GStreamer WebRTC pipeline.
|
|
73
|
+
"""Initialize the GStreamer WebRTC pipeline.
|
|
74
|
+
|
|
75
|
+
Args:
|
|
76
|
+
log_level (str): Logging level for WebRTC daemon operations.
|
|
77
|
+
Default: 'INFO'.
|
|
78
|
+
|
|
79
|
+
Note:
|
|
80
|
+
This constructor initializes the GStreamer environment, detects the
|
|
81
|
+
available camera, and sets up the WebRTC streaming pipeline. The
|
|
82
|
+
pipeline automatically starts streaming when initialized.
|
|
83
|
+
|
|
84
|
+
Raises:
|
|
85
|
+
RuntimeError: If no camera is detected or camera specifications cannot
|
|
86
|
+
be determined.
|
|
87
|
+
|
|
88
|
+
Example:
|
|
89
|
+
>>> # Initialize WebRTC daemon with debug logging
|
|
90
|
+
>>> webrtc_daemon = GstWebRTC(log_level="DEBUG")
|
|
91
|
+
>>> # The daemon is now streaming and ready to accept client connections
|
|
92
|
+
|
|
93
|
+
"""
|
|
33
94
|
self._logger = logging.getLogger(__name__)
|
|
34
95
|
self._logger.setLevel(log_level)
|
|
35
96
|
|
|
36
|
-
# self._id_audio_card = get_respeaker_card_number()
|
|
37
|
-
|
|
38
97
|
Gst.init(None)
|
|
39
98
|
self._loop = GLib.MainLoop()
|
|
40
99
|
self._thread_bus_calls = Thread(target=lambda: self._loop.run(), daemon=True)
|
|
@@ -70,6 +129,7 @@ class GstWebRTC:
|
|
|
70
129
|
|
|
71
130
|
def __del__(self) -> None:
|
|
72
131
|
"""Destructor to ensure gstreamer resources are released."""
|
|
132
|
+
self._logger.debug("Cleaning up GstWebRTC")
|
|
73
133
|
self._loop.quit()
|
|
74
134
|
self._bus_sender.remove_watch()
|
|
75
135
|
self._bus_receiver.remove_watch()
|
|
@@ -156,7 +216,10 @@ class GstWebRTC:
|
|
|
156
216
|
tee = Gst.ElementFactory.make("tee")
|
|
157
217
|
# make camera accessible to other applications via unixfdsrc/sink
|
|
158
218
|
unixfdsink = Gst.ElementFactory.make("unixfdsink")
|
|
159
|
-
|
|
219
|
+
if is_local_camera_available():
|
|
220
|
+
# prevent crash if socket already exists
|
|
221
|
+
os.remove(CAMERA_SOCKET_PATH)
|
|
222
|
+
unixfdsink.set_property("socket-path", CAMERA_SOCKET_PATH)
|
|
160
223
|
queue_unixfd = Gst.ElementFactory.make("queue", "queue_unixfd")
|
|
161
224
|
queue_encoder = Gst.ElementFactory.make("queue", "queue_encoder")
|
|
162
225
|
v4l2h264enc = Gst.ElementFactory.make("v4l2h264enc")
|
|
@@ -164,8 +227,10 @@ class GstWebRTC:
|
|
|
164
227
|
extra_controls_structure.set_value("repeat_sequence_header", 1)
|
|
165
228
|
extra_controls_structure.set_value("video_bitrate", 5_000_000)
|
|
166
229
|
v4l2h264enc.set_property("extra-controls", extra_controls_structure)
|
|
230
|
+
# Use H264 Level 3.1 + Constrained Baseline for Safari/WebKit compatibility
|
|
167
231
|
caps_h264 = Gst.Caps.from_string(
|
|
168
|
-
"video/x-h264,stream-format=byte-stream,alignment=au,
|
|
232
|
+
"video/x-h264,stream-format=byte-stream,alignment=au,"
|
|
233
|
+
"level=(string)3.1,profile=(string)constrained-baseline"
|
|
169
234
|
)
|
|
170
235
|
capsfilter_h264 = Gst.ElementFactory.make("capsfilter")
|
|
171
236
|
capsfilter_h264.set_property("caps", caps_h264)
|
|
@@ -215,7 +280,7 @@ class GstWebRTC:
|
|
|
215
280
|
alsasrc.link(webrtcsink)
|
|
216
281
|
|
|
217
282
|
def _get_audio_input_device(self) -> Optional[str]:
|
|
218
|
-
"""Use Gst.DeviceMonitor to find the
|
|
283
|
+
"""Use Gst.DeviceMonitor to find the pipewire audio card.
|
|
219
284
|
|
|
220
285
|
Returns the device ID of the found audio card, None if not.
|
|
221
286
|
"""
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import bisect # noqa: D100
|
|
2
2
|
import json
|
|
3
|
+
import logging
|
|
3
4
|
import os
|
|
4
5
|
from glob import glob
|
|
5
6
|
from pathlib import Path
|
|
@@ -8,10 +9,58 @@ from typing import Any, Dict, List, Optional
|
|
|
8
9
|
import numpy as np
|
|
9
10
|
import numpy.typing as npt
|
|
10
11
|
from huggingface_hub import snapshot_download
|
|
12
|
+
from huggingface_hub.errors import LocalEntryNotFoundError
|
|
11
13
|
|
|
12
14
|
from reachy_mini.motion.move import Move
|
|
13
15
|
from reachy_mini.utils.interpolation import linear_pose_interpolation
|
|
14
16
|
|
|
17
|
+
logger = logging.getLogger(__name__)
|
|
18
|
+
|
|
19
|
+
# Default datasets to preload at daemon startup
|
|
20
|
+
DEFAULT_DATASETS = [
|
|
21
|
+
"pollen-robotics/reachy-mini-emotions-library",
|
|
22
|
+
"pollen-robotics/reachy-mini-dances-library",
|
|
23
|
+
]
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def preload_dataset(dataset_name: str) -> str | None:
|
|
27
|
+
"""Pre-download a HuggingFace dataset to local cache.
|
|
28
|
+
|
|
29
|
+
This function downloads the dataset with network access, so it should be
|
|
30
|
+
called during daemon startup (not during playback) to avoid blocking.
|
|
31
|
+
|
|
32
|
+
Args:
|
|
33
|
+
dataset_name: The HuggingFace dataset name (e.g., "pollen-robotics/reachy-mini-emotions-library")
|
|
34
|
+
|
|
35
|
+
Returns:
|
|
36
|
+
The local path to the cached dataset, or None if download failed.
|
|
37
|
+
|
|
38
|
+
"""
|
|
39
|
+
try:
|
|
40
|
+
logger.info(f"Pre-downloading dataset: {dataset_name}")
|
|
41
|
+
local_path: str = snapshot_download(dataset_name, repo_type="dataset")
|
|
42
|
+
logger.info(f"Dataset {dataset_name} cached at: {local_path}")
|
|
43
|
+
return local_path
|
|
44
|
+
except Exception as e:
|
|
45
|
+
logger.warning(f"Failed to pre-download dataset {dataset_name}: {e}")
|
|
46
|
+
return None
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def preload_default_datasets() -> dict[str, str | None]:
|
|
50
|
+
"""Pre-download all default recorded move datasets.
|
|
51
|
+
|
|
52
|
+
Should be called during daemon startup to ensure datasets are cached
|
|
53
|
+
before any playback requests.
|
|
54
|
+
|
|
55
|
+
Returns:
|
|
56
|
+
A dict mapping dataset names to their local paths (or None if failed).
|
|
57
|
+
|
|
58
|
+
"""
|
|
59
|
+
results = {}
|
|
60
|
+
for dataset in DEFAULT_DATASETS:
|
|
61
|
+
results[dataset] = preload_dataset(dataset)
|
|
62
|
+
return results
|
|
63
|
+
|
|
15
64
|
|
|
16
65
|
def lerp(v0: float, v1: float, alpha: float) -> float:
|
|
17
66
|
"""Linear interpolation between two values."""
|
|
@@ -105,12 +154,33 @@ class RecordedMove(Move):
|
|
|
105
154
|
|
|
106
155
|
|
|
107
156
|
class RecordedMoves:
|
|
108
|
-
"""Load a library of recorded moves from a HuggingFace dataset.
|
|
157
|
+
"""Load a library of recorded moves from a HuggingFace dataset.
|
|
158
|
+
|
|
159
|
+
Uses local cache only to avoid blocking network calls during playback.
|
|
160
|
+
The dataset should be pre-downloaded at daemon startup via preload_default_datasets().
|
|
161
|
+
If not cached, falls back to network download (which may cause delays).
|
|
162
|
+
"""
|
|
109
163
|
|
|
110
164
|
def __init__(self, hf_dataset_name: str):
|
|
111
165
|
"""Initialize RecordedMoves."""
|
|
112
166
|
self.hf_dataset_name = hf_dataset_name
|
|
113
|
-
|
|
167
|
+
# Try local cache first (instant, no network)
|
|
168
|
+
try:
|
|
169
|
+
self.local_path = snapshot_download(
|
|
170
|
+
self.hf_dataset_name,
|
|
171
|
+
repo_type="dataset",
|
|
172
|
+
local_files_only=True,
|
|
173
|
+
)
|
|
174
|
+
except LocalEntryNotFoundError:
|
|
175
|
+
# Fallback: download from network (slow, but ensures it works)
|
|
176
|
+
logger.warning(
|
|
177
|
+
f"Dataset {hf_dataset_name} not in cache, downloading from HuggingFace. "
|
|
178
|
+
"This may take a moment. Consider pre-loading datasets at daemon startup."
|
|
179
|
+
)
|
|
180
|
+
self.local_path = snapshot_download(
|
|
181
|
+
self.hf_dataset_name,
|
|
182
|
+
repo_type="dataset",
|
|
183
|
+
)
|
|
114
184
|
self.moves: Dict[str, Any] = {}
|
|
115
185
|
self.sounds: Dict[str, Optional[Path]] = {}
|
|
116
186
|
|
|
@@ -119,6 +189,10 @@ class RecordedMoves:
|
|
|
119
189
|
def process(self) -> None:
|
|
120
190
|
"""Populate recorded moves and sounds."""
|
|
121
191
|
move_paths_tmp = glob(f"{self.local_path}/*.json")
|
|
192
|
+
data_dir = os.path.join(self.local_path, "data")
|
|
193
|
+
if os.path.isdir(data_dir):
|
|
194
|
+
# Newer datasets keep their moves inside data/; look there as well.
|
|
195
|
+
move_paths_tmp.extend(glob(f"{data_dir}/*.json"))
|
|
122
196
|
move_paths = [Path(move_path) for move_path in move_paths_tmp]
|
|
123
197
|
for move_path in move_paths:
|
|
124
198
|
move_name = move_path.stem
|