lerobot-teleoperator-livekit 0.1.0__tar.gz

lerobot_teleoperator_livekit-0.1.0/.gitignore

@@ -0,0 +1,31 @@
+# Design docs
+spec.md
+implementation.md
+
+# Rust
+/target
+**/*.rs.bk
+
+# Python
+__pycache__/
+*.py[cod]
+*.egg-info/
+dist/
+build/
+.venv/
+.pytest_cache/
+
+# Env files — keep `.env.example` as a committed template.
+.env
+.env.*
+!.env.example
+
+# Built cdylib + generated UniFFI Python module (both produced by
+# scripts/build_native.sh). Match at any depth for the workspace layout.
+**/livekit/portal/liblivekit_portal_ffi.dylib
+**/livekit/portal/liblivekit_portal_ffi.so
+**/livekit/portal/livekit_portal_ffi.dll
+**/livekit/portal/livekit_portal_ffi.py
+
+# OS
+.DS_Store

lerobot_teleoperator_livekit-0.1.0/PKG-INFO

@@ -0,0 +1,9 @@
+Metadata-Version: 2.4
+Name: lerobot-teleoperator-livekit
+Version: 0.1.0
+Summary: LiveKit Portal teleoperator plugin for lerobot (robot-side)
+License: Apache-2.0
+Requires-Python: >=3.12
+Requires-Dist: lerobot>=0.5.0
+Requires-Dist: livekit-portal
+Requires-Dist: numpy>=1.24

lerobot_teleoperator_livekit-0.1.0/lerobot_teleoperator_livekit/__init__.py

@@ -0,0 +1,12 @@
+"""LiveKit Portal teleoperator plugin for lerobot.
+
+Deployed on the **robot side**. Wraps `livekit.portal.Portal` in `Role.ROBOT`
+so lerobot can drive a remote physical robot by running a teleop loop that
+pushes actions over LiveKit. Importing this module registers
+``LiveKitTeleoperator`` as ``--teleop.type=livekit``.
+"""
+from __future__ import annotations
+
+from .teleoperator import LiveKitTeleoperator, LiveKitTeleoperatorConfig
+
+__all__ = ["LiveKitTeleoperator", "LiveKitTeleoperatorConfig"]

lerobot_teleoperator_livekit-0.1.0/lerobot_teleoperator_livekit/teleoperator.py

@@ -0,0 +1,293 @@
+"""LiveKit Portal teleoperator implementation.
+
+Runs on the physical robot. Opens a Portal as ``Role.ROBOT``, publishing
+camera frames + state via ``send_feedback(...)`` and surfacing received
+actions via ``get_action()``. When constructed with a local lerobot
+``Robot`` instance, it introspects ``observation_features`` /
+``action_features`` to derive motor and camera shapes automatically — the
+physical robot stays in the user's loop; the plugin just brokers the wire.
+"""
+from __future__ import annotations
+
+import asyncio
+import threading
+from dataclasses import dataclass, field
+from typing import Any
+
+import numpy as np
+from lerobot.teleoperators.config import TeleoperatorConfig
+from lerobot.teleoperators.teleoperator import Teleoperator
+
+from livekit.portal import DType, Portal, PortalConfig, Role
+
+
+@TeleoperatorConfig.register_subclass("livekit")
+@dataclass
+class LiveKitTeleoperatorConfig(TeleoperatorConfig):
+    url: str = ""
+    token: str = ""
+    session: str = ""
+    fps: int = 30
+
+    # Explicit-mode fallbacks when no local Robot is passed to the constructor
+    # (e.g. the ``--teleop.type=livekit`` CLI path, which can't pass instances).
+    # Ignored when a robot is provided.
+    motors: tuple[str, ...] = ()
+    camera_names: tuple[str, ...] = ()
+
+    # Portal tuning.
+    slack: int | None = None
+    tolerance: float | None = None
+    state_reliable: bool = True
+    action_reliable: bool = True
+    reuse_stale_frames: bool = False
+
+
+def _split_observation_features(
+    features: dict,
+) -> tuple[list[str], dict[str, tuple[int, ...]]]:
+    """Separate scalar motor keys from camera (tuple-valued) keys."""
+    motor_keys: list[str] = []
+    cameras: dict[str, tuple[int, ...]] = {}
+    for key, val in features.items():
+        if isinstance(val, tuple):
+            cameras[key] = val
+        else:
+            motor_keys.append(key)
+    return sorted(motor_keys), cameras
+
+
+class LiveKitTeleoperator(Teleoperator):
+    """lerobot Teleoperator that forwards actions over a Portal session.
+
+    Construct with an optional local ``Robot`` instance; its features
+    determine the motor keys and camera list used by the network layer.
+    Without a robot, falls back to ``config.motors`` + ``config.camera_names``.
+
+    Typical use on the robot side::
+
+        robot = MyPhysicalRobot(...)
+        teleop = LiveKitTeleoperator(cfg, robot=robot)
+        teleop.connect()
+        while running:
+            obs = robot.get_observation()
+            teleop.send_feedback(obs)
+            action = teleop.get_action()
+            if action:
+                robot.send_action(action)
+    """
+
+    config_class = LiveKitTeleoperatorConfig
+    name = "livekit"
+
+    def __init__(
+        self,
+        config: LiveKitTeleoperatorConfig,
+        robot: Any | None = None,
+    ) -> None:
+        super().__init__(config)
+        self.config = config
+
+        self._state_keys, self._action_keys, self._cameras = self._resolve_schema(
+            config, robot
+        )
+        # Portal's state/action fields are the raw motor names (without the
+        # ".pos" suffix); lerobot observation/action dicts use the suffix.
+        self._state_motors = [_strip_pos(k) for k in self._state_keys]
+        self._action_motors = [_strip_pos(k) for k in self._action_keys]
+        self._camera_names = list(self._cameras.keys())
+
+        self._act_features: dict = {k: float for k in self._action_keys}
+        self._feedback_features: dict = {k: float for k in self._state_keys}
+        for name, shape in self._cameras.items():
+            self._feedback_features[name] = shape
+
+        self._portal: Portal | None = None
+        self._portal_cfg: PortalConfig | None = None
+        self._loop: asyncio.AbstractEventLoop | None = None
+        self._loop_thread: threading.Thread | None = None
+        self._connected = False
+
+    # -- lerobot interface ----------------------------------------------------
+
+    @property
+    def action_features(self) -> dict:
+        return self._act_features
+
+    @property
+    def feedback_features(self) -> dict:
+        return self._feedback_features
+
+    @property
+    def is_connected(self) -> bool:
+        return self._connected
+
+    @property
+    def is_calibrated(self) -> bool:
+        return True
+
+    def calibrate(self) -> None:
+        pass
+
+    def configure(self) -> None:
+        pass
+
+    def connect(self, calibrate: bool = True) -> None:
+        if self._connected:
+            return
+        if not self.config.url or not self.config.token:
+            raise RuntimeError(
+                "LiveKitTeleoperatorConfig.url and .token are required; mint a"
+                " token with Role.ROBOT grants before calling connect()."
+            )
+
+        self._start_loop()
+
+        self._portal_cfg = PortalConfig(self.config.session or "lerobot", Role.ROBOT)
+        for cam in self._camera_names:
+            self._portal_cfg.add_video(cam)
+        if self._state_motors:
+            self._portal_cfg.add_state_typed(
+                [(name, DType.F64) for name in self._state_motors]
+            )
+        if self._action_motors:
+            self._portal_cfg.add_action_typed(
+                [(name, DType.F64) for name in self._action_motors]
+            )
+        self._portal_cfg.set_fps(self.config.fps)
+        if self.config.slack is not None:
+            self._portal_cfg.set_slack(self.config.slack)
+        if self.config.tolerance is not None:
+            self._portal_cfg.set_tolerance(self.config.tolerance)
+        self._portal_cfg.set_state_reliable(self.config.state_reliable)
+        self._portal_cfg.set_action_reliable(self.config.action_reliable)
+        self._portal_cfg.set_reuse_stale_frames(self.config.reuse_stale_frames)
+
+        self._portal = Portal(self._portal_cfg)
+        self._run(self._portal.connect(self.config.url, self.config.token))
+        self._connected = True
+
+    def disconnect(self) -> None:
+        if not self._connected:
+            return
+        try:
+            if self._portal is not None:
+                self._run(self._portal.disconnect())
+        finally:
+            if self._portal is not None:
+                self._portal.close()
+                self._portal = None
+            if self._portal_cfg is not None:
+                self._portal_cfg.close()
+                self._portal_cfg = None
+            self._stop_loop()
+            self._connected = False
+
+    def get_action(self) -> dict[str, Any]:
+        """Latest action received from the operator, keyed like the local
+        robot's ``action_features``. Empty dict if nothing has arrived."""
+        if self._portal is None:
+            return {}
+        action = self._portal.get_action()
+        if action is None:
+            return {}
+        return {
+            k: float(action.values.get(m, 0.0))
+            for k, m in zip(self._action_keys, self._action_motors)
+        }
+
+    def send_feedback(self, feedback: dict[str, Any]) -> None:
+        """Publish the local robot's observation to the operator.
+
+        ``feedback`` is the same dict shape returned by
+        ``robot.get_observation()``: motor keys (e.g. ``"shoulder.pos"``) map
+        to floats, camera keys (matching those in ``observation_features``)
+        map to ``np.ndarray(H, W, 3)`` uint8 RGB. Unknown keys are ignored.
+        """
+        if self._portal is None or not self._connected:
+            return
+
+        if self._state_motors:
+            state: dict[str, float] = {}
+            for key, motor in zip(self._state_keys, self._state_motors):
+                if key in feedback:
+                    state[motor] = float(feedback[key])
+            if state:
+                self._portal.send_state(state)
+
+        for cam in self._camera_names:
+            frame = feedback.get(cam)
+            if frame is None:
+                continue
+            if not isinstance(frame, np.ndarray):
+                raise TypeError(
+                    f"camera feedback '{cam}' must be np.ndarray (H, W, 3)"
+                    f" uint8 RGB; got {type(frame).__name__}"
+                )
+            self._portal.send_video_frame(cam, frame)
+
+    # -- schema resolution ---------------------------------------------------
+
+    @staticmethod
+    def _resolve_schema(
+        config: LiveKitTeleoperatorConfig,
+        robot: Any | None,
+    ) -> tuple[list[str], list[str], dict[str, tuple[int, ...]]]:
+        if robot is not None:
+            obs_features = dict(getattr(robot, "observation_features", {}))
+            act_features = dict(getattr(robot, "action_features", {}))
+            if not obs_features and not act_features:
+                raise ValueError(
+                    "local robot has empty observation_features /"
+                    " action_features; cannot infer schema"
+                )
+            state_keys, cameras = _split_observation_features(obs_features)
+            action_keys = sorted(act_features.keys())
+            return state_keys, action_keys, cameras
+
+        if config.motors or config.camera_names:
+            state_keys = sorted(f"{m}.pos" for m in config.motors)
+            action_keys = list(state_keys)
+            # Shape is metadata only; Portal accepts any resolution at runtime.
+            cameras = {name: () for name in config.camera_names}
+            return state_keys, action_keys, cameras
+
+        raise ValueError(
+            "LiveKitTeleoperator needs either a local Robot instance or"
+            " config.motors / config.camera_names to derive its schema"
+        )
+
+    # -- background loop plumbing --------------------------------------------
+
+    def _start_loop(self) -> None:
+        self._loop = asyncio.new_event_loop()
+        started = threading.Event()
+
+        def _runner() -> None:
+            asyncio.set_event_loop(self._loop)
+            started.set()
+            self._loop.run_forever()
+
+        self._loop_thread = threading.Thread(
+            target=_runner, name="livekit-portal-loop", daemon=True
+        )
+        self._loop_thread.start()
+        started.wait()
+
+    def _stop_loop(self) -> None:
+        if self._loop is None:
+            return
+        self._loop.call_soon_threadsafe(self._loop.stop)
+        if self._loop_thread is not None:
+            self._loop_thread.join(timeout=5.0)
+        self._loop.close()
+        self._loop = None
+        self._loop_thread = None
+
+    def _run(self, coro):
+        assert self._loop is not None, "background loop not started"
+        return asyncio.run_coroutine_threadsafe(coro, self._loop).result()
+
+
+def _strip_pos(key: str) -> str:
+    return key[: -len(".pos")] if key.endswith(".pos") else key

lerobot_teleoperator_livekit-0.1.0/pyproject.toml

@@ -0,0 +1,26 @@
+[build-system]
+requires = ["hatchling>=1.20", "hatch-vcs"]
+build-backend = "hatchling.build"
+
+[project]
+name = "lerobot-teleoperator-livekit"
+dynamic = ["version"]
+description = "LiveKit Portal teleoperator plugin for lerobot (robot-side)"
+requires-python = ">=3.12"
+license = { text = "Apache-2.0" }
+dependencies = [
+    "livekit-portal",
+    "lerobot>=0.5.0",
+    "numpy>=1.24",
+]
+
+[tool.uv.sources]
+livekit-portal = { workspace = true }
+
+[tool.hatch.build.targets.wheel]
+packages = ["lerobot_teleoperator_livekit"]
+
+[tool.hatch.version]
+source = "vcs"
+tag-pattern = "^v(?P<version>.+)$"
+raw-options = { search_parent_directories = true }