robits 0.5.0__tar.gz

robits-0.5.0/PKG-INFO

@@ -0,0 +1,219 @@
+Metadata-Version: 2.3
+Name: robits
+Version: 0.5.0
+Summary: Lightweight, modular, and scalable software stack for AI-driven robotic manipulation.
+Keywords: robotics,AI,manipulation,research,perception,control,simulation
+Author: Markus Grotz
+Author-email: grotz@uw.edu
+Requires-Python: >=3.9,<3.13
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Console
+Classifier: Framework :: Robot Framework
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Provides-Extra: all
+Provides-Extra: audio
+Provides-Extra: dev
+Provides-Extra: real
+Provides-Extra: remote
+Provides-Extra: sim
+Provides-Extra: ur
+Provides-Extra: usd
+Provides-Extra: xarm
+Requires-Dist: black[dev] (>=25.1.0,<26.0.0) ; extra == "dev"
+Requires-Dist: click-prompt (>=0.6.0)
+Requires-Dist: dm-control[all,sim] (>=1.0.30,<2.0.0) ; extra == "sim" or extra == "usd" or extra == "all"
+Requires-Dist: franky-panda[all,real] ; (platform_machine == "x86_64") and (extra == "real" or extra == "all")
+Requires-Dist: ipython (>=8.0)
+Requires-Dist: matplotlib (>=3.7.5,<4.0.0)
+Requires-Dist: mujoco[all,sim] (>=3.3,<4.0) ; extra == "sim" or extra == "usd" or extra == "all"
+Requires-Dist: myst-parser[dev] (==3.0.1) ; extra == "dev"
+Requires-Dist: numpy (>=1.24)
+Requires-Dist: open3d (==0.18.*) ; python_version < "3.10"
+Requires-Dist: open3d (>=0.18) ; python_version >= "3.10"
+Requires-Dist: openai-whisper[audio] (>=20240930,<20240931) ; extra == "audio"
+Requires-Dist: openai[audio] (>=1.64.0,<2.0.0) ; extra == "audio"
+Requires-Dist: psutil[all,remote] (>=7.0.0,<8.0.0) ; extra == "all" or extra == "remote"
+Requires-Dist: pyrealsense2 (>=2.55.1,<3.0.0) ; (python_version < "3.12" and platform_machine == "x86_64") and (extra == "real" or extra == "all")
+Requires-Dist: pyrealsense2-beta (==2.56.0.8430) ; (python_version >= "3.12" and platform_machine == "x86_64") and (extra == "real" or extra == "all")
+Requires-Dist: pyzmq[all,remote] (>=26.4.0,<27.0.0) ; extra == "all" or extra == "remote"
+Requires-Dist: rich-click (>=1,<2)
+Requires-Dist: robot-descriptions[all,sim] (>=1.14.0,<2.0.0) ; extra == "sim" or extra == "usd" or extra == "all"
+Requires-Dist: ruff[dev] (>=0.11.11,<0.12.0) ; extra == "dev"
+Requires-Dist: scipy (>=1.13,<2.0)
+Requires-Dist: sounddevice[audio] (>=0.5.2,<0.6.0) ; extra == "audio"
+Requires-Dist: sphinx-rtd-theme[dev] (>=3.0.2,<4.0.0) ; extra == "dev"
+Requires-Dist: sphinx[dev] (==7.4.7) ; extra == "dev"
+Requires-Dist: sphinxcontrib-video[dev] (>=0.4.1,<0.5.0) ; extra == "dev"
+Requires-Dist: textual (>=1.0.0,<2.0.0)
+Requires-Dist: textual-slider (>=0.1.2,<0.2.0)
+Requires-Dist: toml[dev] (>=0.10.2,<0.11.0) ; extra == "dev"
+Requires-Dist: urx[all,ur] (>=0.11.0,<0.12.0) ; extra == "ur"
+Requires-Dist: usd-core[all,usd] (>=23.5.0,<24.0.0) ; (python_full_version >= "3.8.1" and python_version < "3.11") and (extra == "usd" or extra == "all")
+Requires-Dist: xarm-python-sdk[all,xarm] (>=1.15.3,<2.0.0) ; extra == "xarm"
+Requires-Dist: zmq[all,remote] (>=0.0.0,<0.0.1) ; extra == "all" or extra == "remote"
+Project-URL: documentation, https://robits.readthedocs.io/en/latest/
+Project-URL: homepage, https://github.com/markusgrotz/robits
+Project-URL: repository, https://github.com/markusgrotz/robits
+Project-URL: tracker, https://github.com/markusgrotz/robits/issues
+Description-Content-Type: text/markdown
+
+# RoBits - Bits and Bytes for Robotic Manipulation
+
+## `from robits import ♥`
+
+[![Supported Python Versions](https://img.shields.io/pypi/pyversions/robits)](https://pypi.org/project/robits/) 
+[![PyPI version](https://img.shields.io/pypi/v/robits)](https://pypi.org/project/robits/) 
+[![License](https://img.shields.io/pypi/l/robits)](https://github.com/markusgrotz/robits/LICENSE.md)
+[![Code style](https://img.shields.io/badge/code%20style-black-black)](https://black.readthedocs.io/en/stable/)
+
+
+RoBits is a lightweight, modular and scalable software stack for AI-driven
+robotic manipulation.  It is designed for seamless integration with robotic
+manipulation policies, by providing essential tools for perception and robotic
+control.
+
+RoBits features an intuitive command-line interface to get started quickly.
+It's user-friendly and ensures adaptability, efficiency, and ease of use across
+diverse robotic applications to help with research experiments.
+
+If you are using this work or find it otherwise useful please cite:
+```
+M. Grotz, M. Shridhar, Y. Chao, T. Asfour and D. Fox
+PerAct2: Benchmarking and Learning for Robotic Bimanual Manipulation Tasks.
+https://doi.org/10.48550/arXiv.2407.00278
+```
+
+Also consider citing additional resources if necessary (See [Acknowledgement](#Acknowledgement))
+
+
+![Logo](https://raw.githubusercontent.com/markusgrotz/robits/master/docs/source/_static/logo.png)
+
+
+## Quickstart
+
+RoBits comes with some default configurations, for example for the Franka Panda
+robot. Run `pip install 'robits[all]'`. This will install necessary dependencies and
+provide an entry point command `rb`. You can use `rb` to get a list of
+commands. For example `rb move home` moves the robot to a default joint
+position.
+
+## Command-Line-Interface
+
+Once you have installed `RoBits` you can access the CLI with `rb` in your shell:
+```bash
+(robits-py3.8) markus @ sockeye ➜  ~  rb                  [2025-02-23 19:53:43]
+                                                                                
+ Usage: rb [OPTIONS] COMMAND [ARGS]...                                          
+                                                                                
+ RoBits command line interface for robotic manipulation.                        
+ This is the main entry point for all RoBits CLI commands. It provides global   
+ verbosity controls and sets up logging configuration.                          
+ Use --verbose/-v to increase log detail level (can be used multiple times).    
+ Use --quiet/-q to suppress all but error messages.                             
+ Example: rb -vv move home --robot-name robot_panda_sim                         
+                                                                                
+╭─ Options ────────────────────────────────────────────────────────────────────╮
+│ --verbose  -v  INTEGER RANGE  Increase verbosity of the logging output. Can  │
+│                               base used multiple times                       │
+│ --quiet    -q                 Suppress all logging output except critical    │
+│                               and error messages                             │
+│ --help                        Show this message and exit.                    │
+╰──────────────────────────────────────────────────────────────────────────────╯
+╭─ Commands ───────────────────────────────────────────────────────────────────╮
+│ camera    Commands for working with cameras and visual data.                 │
+│ config    Commands for viewing and managing system configurations.           │
+│ dataset   Dataset related commands                                           │
+│ gripper   Gripper related commands                                           │
+│ info      Read various information about the current robot state             │
+│ move      Commands for moving the robot in different directions.             │
+│ panda     Franka Panda related commands                                      │
+│ shell     Creates an interactive shell. Use robot variable to interact       │
+│ speech    Audio related commands                                             │
+╰──────────────────────────────────────────────────────────────────────────────╯
+```
+To learn more about a command you can write for example `rb dataset --help`,
+which will list the subcommands that are related to dataset.
+
+
+### Configuration
+
+You can display available config files with `rb config list`. To show the
+actual content use `rb config show <config_name>` where `config_name`
+specifies the name of the config file, e.g., `rb config show robot_panda_real`.
+You can specify a user configuration by exporting `ROBITS_CONFIG_DIR`. E.g. by adding
+```bash
+export ROBITS_CONFIG_DIR=/home/markus/robits_config
+```
+to your `.bashrc`.
+This allows you to specify additional configurations for robots, cameras,
+grippers.  For example, you can change the serial number of the camera or the
+ip address of a robot.
+Once you set your user confiuration folder you can copy and modify an existing configuration with:
+```bash
+rb config copy
+```
+Select the configuration you want to modify and open it with your favorite text editor
+
+
+#### Robot setup
+
+Best practice is to install a RT (Real-Time) kernel. See the script for more details.
+With some patches, you can install the NVIDIA driver and access the GPU through
+PyTorch on a real-time kernel system. This avoids unnecessary networking calls
+and removes the burden of deploying the software as everything can be run from a
+single repository.
+Once that is done you can get the robot pose with:
+`rb info pose` 
+To move the robot, use one of the `rb move` commands, e.g.:
+`rb move up`
+
+#### Camera setup
+
+To test your camera setup you can use:
+`rb camera view` and `rb camera point-cloud view`
+For calibration to get the extrinsics, you can use:
+`rb camera calibrate extrinsics`
+This will launch a TUI. Select the camera and press "connect". Then use the
+sliders to adjust the camera pose. Press "save" to store the camera calibration
+to your user config directory, which is set by the environment variable
+`ROBITS_CONFIG_DIR`.
+
+#### Gripper setup
+
+Basic commands for the gripper are `rb gripper open` or `rb gripper close`.
+Please note that there are some limitations with the Franka Panda gripper.
+Basically, the robot stops when the gripper is active. See the implementation
+for details.
+
+### Data Collection and Replay
+
+To replay collected data, use `rb dataset replay`. You can specify the path, the control method, and the robot name:
+```bash
+rb dataset replay --input-path ~/data/demo_0000/ --robot-name robot_panda_sim --control-method position
+```
+
+
+## Libraries Acknowledgement
+
+- Libraries
+  - [NumPy](https://numpy.org), [SciPy](https://scipy.org/), [Open3D](https://www.open3d.org/), and more
+- Real robots and grippers
+  - [libfranka](https://github.com/frankaemika/libfranka)
+  - [Franky](https://github.com/TimSchneider42/franky)
+  - pyRobotiqGripper originally from [castetsb/pyRobotiqGripper](https://github.com/castetsb/pyRobotiqGripper)
+  - [xArm Python SDK](https://github.com/xArm-Developer/xArm-Python-SDK)
+  - [Python URx](https://github.com/SintefManufacturing/python-urx)
+- Simulation
+  - [MuJoCo](https://mujoco.org/)
+  - [MuJoCo Menagerie](https://github.com/google-deepmind/mujoco_menagerie) and [Robot Descriptions](https://github.com/robot-descriptions/robot_descriptions)
+  - [MuJoCo Controllers](https://github.com/kevinzakka/mjctrl)
+

robits-0.5.0/README.md

@@ -0,0 +1,150 @@
+# RoBits - Bits and Bytes for Robotic Manipulation
+
+## `from robits import ♥`
+
+[![Supported Python Versions](https://img.shields.io/pypi/pyversions/robits)](https://pypi.org/project/robits/) 
+[![PyPI version](https://img.shields.io/pypi/v/robits)](https://pypi.org/project/robits/) 
+[![License](https://img.shields.io/pypi/l/robits)](https://github.com/markusgrotz/robits/LICENSE.md)
+[![Code style](https://img.shields.io/badge/code%20style-black-black)](https://black.readthedocs.io/en/stable/)
+
+
+RoBits is a lightweight, modular and scalable software stack for AI-driven
+robotic manipulation.  It is designed for seamless integration with robotic
+manipulation policies, by providing essential tools for perception and robotic
+control.
+
+RoBits features an intuitive command-line interface to get started quickly.
+It's user-friendly and ensures adaptability, efficiency, and ease of use across
+diverse robotic applications to help with research experiments.
+
+If you are using this work or find it otherwise useful please cite:
+```
+M. Grotz, M. Shridhar, Y. Chao, T. Asfour and D. Fox
+PerAct2: Benchmarking and Learning for Robotic Bimanual Manipulation Tasks.
+https://doi.org/10.48550/arXiv.2407.00278
+```
+
+Also consider citing additional resources if necessary (See [Acknowledgement](#Acknowledgement))
+
+
+![Logo](https://raw.githubusercontent.com/markusgrotz/robits/master/docs/source/_static/logo.png)
+
+
+## Quickstart
+
+RoBits comes with some default configurations, for example for the Franka Panda
+robot. Run `pip install 'robits[all]'`. This will install necessary dependencies and
+provide an entry point command `rb`. You can use `rb` to get a list of
+commands. For example `rb move home` moves the robot to a default joint
+position.
+
+## Command-Line-Interface
+
+Once you have installed `RoBits` you can access the CLI with `rb` in your shell:
+```bash
+(robits-py3.8) markus @ sockeye ➜  ~  rb                  [2025-02-23 19:53:43]
+                                                                                
+ Usage: rb [OPTIONS] COMMAND [ARGS]...                                          
+                                                                                
+ RoBits command line interface for robotic manipulation.                        
+ This is the main entry point for all RoBits CLI commands. It provides global   
+ verbosity controls and sets up logging configuration.                          
+ Use --verbose/-v to increase log detail level (can be used multiple times).    
+ Use --quiet/-q to suppress all but error messages.                             
+ Example: rb -vv move home --robot-name robot_panda_sim                         
+                                                                                
+╭─ Options ────────────────────────────────────────────────────────────────────╮
+│ --verbose  -v  INTEGER RANGE  Increase verbosity of the logging output. Can  │
+│                               base used multiple times                       │
+│ --quiet    -q                 Suppress all logging output except critical    │
+│                               and error messages                             │
+│ --help                        Show this message and exit.                    │
+╰──────────────────────────────────────────────────────────────────────────────╯
+╭─ Commands ───────────────────────────────────────────────────────────────────╮
+│ camera    Commands for working with cameras and visual data.                 │
+│ config    Commands for viewing and managing system configurations.           │
+│ dataset   Dataset related commands                                           │
+│ gripper   Gripper related commands                                           │
+│ info      Read various information about the current robot state             │
+│ move      Commands for moving the robot in different directions.             │
+│ panda     Franka Panda related commands                                      │
+│ shell     Creates an interactive shell. Use robot variable to interact       │
+│ speech    Audio related commands                                             │
+╰──────────────────────────────────────────────────────────────────────────────╯
+```
+To learn more about a command you can write for example `rb dataset --help`,
+which will list the subcommands that are related to dataset.
+
+
+### Configuration
+
+You can display available config files with `rb config list`. To show the
+actual content use `rb config show <config_name>` where `config_name`
+specifies the name of the config file, e.g., `rb config show robot_panda_real`.
+You can specify a user configuration by exporting `ROBITS_CONFIG_DIR`. E.g. by adding
+```bash
+export ROBITS_CONFIG_DIR=/home/markus/robits_config
+```
+to your `.bashrc`.
+This allows you to specify additional configurations for robots, cameras,
+grippers.  For example, you can change the serial number of the camera or the
+ip address of a robot.
+Once you set your user confiuration folder you can copy and modify an existing configuration with:
+```bash
+rb config copy
+```
+Select the configuration you want to modify and open it with your favorite text editor
+
+
+#### Robot setup
+
+Best practice is to install a RT (Real-Time) kernel. See the script for more details.
+With some patches, you can install the NVIDIA driver and access the GPU through
+PyTorch on a real-time kernel system. This avoids unnecessary networking calls
+and removes the burden of deploying the software as everything can be run from a
+single repository.
+Once that is done you can get the robot pose with:
+`rb info pose` 
+To move the robot, use one of the `rb move` commands, e.g.:
+`rb move up`
+
+#### Camera setup
+
+To test your camera setup you can use:
+`rb camera view` and `rb camera point-cloud view`
+For calibration to get the extrinsics, you can use:
+`rb camera calibrate extrinsics`
+This will launch a TUI. Select the camera and press "connect". Then use the
+sliders to adjust the camera pose. Press "save" to store the camera calibration
+to your user config directory, which is set by the environment variable
+`ROBITS_CONFIG_DIR`.
+
+#### Gripper setup
+
+Basic commands for the gripper are `rb gripper open` or `rb gripper close`.
+Please note that there are some limitations with the Franka Panda gripper.
+Basically, the robot stops when the gripper is active. See the implementation
+for details.
+
+### Data Collection and Replay
+
+To replay collected data, use `rb dataset replay`. You can specify the path, the control method, and the robot name:
+```bash
+rb dataset replay --input-path ~/data/demo_0000/ --robot-name robot_panda_sim --control-method position
+```
+
+
+## Libraries Acknowledgement
+
+- Libraries
+  - [NumPy](https://numpy.org), [SciPy](https://scipy.org/), [Open3D](https://www.open3d.org/), and more
+- Real robots and grippers
+  - [libfranka](https://github.com/frankaemika/libfranka)
+  - [Franky](https://github.com/TimSchneider42/franky)
+  - pyRobotiqGripper originally from [castetsb/pyRobotiqGripper](https://github.com/castetsb/pyRobotiqGripper)
+  - [xArm Python SDK](https://github.com/xArm-Developer/xArm-Python-SDK)
+  - [Python URx](https://github.com/SintefManufacturing/python-urx)
+- Simulation
+  - [MuJoCo](https://mujoco.org/)
+  - [MuJoCo Menagerie](https://github.com/google-deepmind/mujoco_menagerie) and [Robot Descriptions](https://github.com/robot-descriptions/robot_descriptions)
+  - [MuJoCo Controllers](https://github.com/kevinzakka/mjctrl)

robits-0.5.0/pyproject.toml

@@ -0,0 +1,109 @@
+[tool.poetry]
+name = "robits"
+version = "0.5.0"
+description = "Lightweight, modular, and scalable software stack for AI-driven robotic manipulation."
+authors = ["Markus Grotz <grotz@uw.edu>"]
+packages = [{include = "robits"}, {include = "robits_config"}]
+readme = "README.md"
+keywords = ["robotics", "AI", "manipulation", "research", "perception", "control", "simulation"]
+
+classifiers = [
+    "Development Status :: 4 - Beta", 
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Operating System :: OS Independent",
+    "Framework :: Robot Framework",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    "Topic :: Scientific/Engineering",
+    "Intended Audience :: Science/Research",
+    "Intended Audience :: Developers",
+    "Environment :: Console",
+]
+
+[tool.poetry.urls]
+homepage = "https://github.com/markusgrotz/robits"
+repository = "https://github.com/markusgrotz/robits"
+documentation = "https://robits.readthedocs.io/en/latest/"
+tracker = "https://github.com/markusgrotz/robits/issues"
+
+[tool.poetry.dependencies]
+python = ">=3.9,<3.13" # open3d is not available yet for newer version
+numpy = ">=1.24"
+open3d = [
+  { version = "0.18.*", markers = "python_version < '3.10'" },
+  { version = ">=0.18", markers = "python_version >= '3.10'" }
+]
+matplotlib = "^3.7.5"
+scipy = "^1.13"
+
+ipython = ">=8.0"
+rich-click = "^1"
+click-prompt = ">=0.6.0"
+
+textual = "^1.0.0"
+textual-slider = "^0.1.2"
+
+# sim 
+mujoco = {version = "^3.3", extras = ["sim", "all"], optional = true}
+robot-descriptions = {version = "^1.14.0", extras = ["sim", "all"], optional = true}
+dm-control = {version = "^1.0.30", extras = ["sim", "all"], optional = true}
+
+# real
+
+# For Python versions less than 3.12
+pyrealsense2 = { version = "^2.55.1", markers = "python_version < '3.12' and platform_machine == 'x86_64'", optional = true }
+
+# For Python 3.12 and above, using the beta version
+pyrealsense2-beta = { version = "2.56.0.8430", markers = "python_version >= '3.12' and platform_machine == 'x86_64'", optional = true }
+#pyrobotiqgripper = {git = "https://github.com/markusgrotz/pyRobotiqGripper.git", extras = ["real", "all"], optional = true}
+franky-panda = { version = "*", markers = "platform_machine == 'x86_64'", extras = ["real", "all"], optional = true}
+urx = {version = "^0.11.0", extras = ["ur", "all"], optional = true}
+xarm-python-sdk = {version ="^1.15.3", extras = ["xarm", "all"], optional = true}
+
+# remote 
+zmq = {version = "^0.0.0", extras = ["remote", "all"], optional = true}
+pyzmq = {version = "^26.4.0", extras = ["remote", "all"], optional = true}
+psutil = {version = "^7.0.0", extras = ["remote", "all"], optional = true}
+
+# optional 
+openai = {version = "^1.64.0", extras = ["audio"], optional = true}
+openai-whisper = {version = "^20240930", extras = ["audio"], optional = true}
+sounddevice = {version = "^0.5.2", extras = ["audio"], optional = true}
+
+# usd - only compatible with Python < 3.11
+usd-core = {version = "^23.5.0", python = ">=3.8.1,<3.11", extras = ["usd", "all"], optional = true}
+
+# dev
+toml = {version = "^0.10.2", extras = ["dev"], optional = true}
+sphinx-rtd-theme = {version = "^3.0.2", extras = ["dev"], optional = true}
+sphinxcontrib-video = {version = "^0.4.1", extras = ["dev"], optional = true}
+myst-parser = {version = "3.0.1", extras = ["dev"], optional = true}
+sphinx = {version = "7.4.7", extras = ["dev"], optional = true}
+ruff = {version = "^0.11.11", extras = ["dev"], optional = true}
+black = {version = "^25.1.0",  extras = ["dev"], optional = true}
+
+
+[tool.poetry.extras]
+real = ["franky-panda", "pyrealsense2", "pyrealsense2-beta"] #"pyrobotiqgripper"
+sim = ["mujoco", "robot-descriptions", "dm-control"]
+usd = ["usd-core", "mujoco", "robot-descriptions", "dm-control"]
+all = ["franky-panda", "pyrealsense2", "pyrealsense2-beta", "mujoco", "robot-descriptions", "dm-control", "usd-core",  "zmq", "pyzmq", "psutil"] #"pyrobotiqgripper",
+dev = ["sphinx", "toml", "myst-parser", "sphinx-rtd-theme", "sphinxcontrib-video", "black", "ruff"]
+ur = ["urx"]
+xarm = ["xarm-python-sdk"]
+remote = ["zmq", "pyzmq", "psutil"]
+audio = ["openai", "openai-whisper", "sounddevice"]
+
+[build-system]
+requires = ["poetry-core>=1.0.8"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.poetry.scripts]
+rb = 'robits.cli.main:cli'
+panda = 'robits.cli.devices.panda_cli:panda'
+gripper = 'robits.cli.devices.gripper_cli:gripper'
+camera = 'robits.cli.devices.camera_cli:camera'
+sam2act-agent = 'robits.cli.agents.sam2act_cli:cli'

robits-0.5.0/robits/agents/base_agent.py

@@ -0,0 +1,63 @@
+from abc import ABC
+from abc import abstractmethod
+
+import logging
+
+import numpy as np
+
+
+logger = logging.getLogger(__name__)
+
+
+class BaseAgent(ABC):
+    """ """
+
+    agent_name: str
+
+    lang_goal: str
+
+    def prepare_observation(self, obs, i, episode_length):
+
+        from clip import tokenize
+        import torch
+
+        logger.info("Using language goal %s", self.lang_goal)
+
+        obs["lang_goal_tokens"] = tokenize([self.lang_goal])[0].numpy()
+
+        elapsed_time = (1.0 - (i / float(episode_length - 1))) * 2.0 - 1.0
+        gripper_joint_positions = obs["gripper_joint_positions"][
+            0:1
+        ]  # gripper is normalized here
+        logger.info("timestamp: %.2f", elapsed_time)
+
+        obs["ignore_collisions"] = np.array([0])
+        obs["low_dim_state"] = np.concatenate(
+            [
+                gripper_joint_positions,
+                gripper_joint_positions,
+                gripper_joint_positions,
+                elapsed_time,
+            ],
+            axis=None,
+        )
+
+        for k, v in obs.items():
+            if v is None:
+                logger.info("No values for key %s", k)
+                # obs[k] = torch.tensor([np.zeros(4)], device=self.device).unsqueeze(0)
+                continue
+
+            if isinstance(v, np.ndarray):
+                v = v.astype(np.float32)
+                logger.debug("Item %s has shape %s", k, v.shape)
+            else:
+                logger.debug("Key %s is not a numpy array", k)
+
+            obs[k] = torch.tensor([v], device=self.device).unsqueeze(0)
+
+        return obs
+
+    @abstractmethod
+    def get_action(self, step, observation):
+        pass

robits-0.5.0/robits/agents/mello_agent.py

@@ -0,0 +1,72 @@
+from typing import Optional
+import logging
+import time
+import requests
+
+from dataclasses import dataclass
+
+import numpy as np
+
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class MelloConfig:
+
+    max_delta: float = 0.25
+
+    initial_joint_positions: Optional[np.ndarray] = None
+
+
+class MelloAgent:
+
+    def __init__(self, device_addr, robot):
+        self.device_addr = device_addr
+        self.robot = robot
+        self.max_delta = 0.25
+
+    def get_mello_data(self):
+        data = requests.get(self.device_addr).json()
+        joint_positions = np.array(data["joint_positions"])
+        joint_positions = np.deg2rad(joint_positions)
+        joint_positions *= -1
+
+        joint_velocities = np.array(data["joint_velocities"])
+        joint_velocities = np.deg2rad(joint_velocities)
+        joint_velocities *= -1
+        return joint_positions, joint_velocities
+
+    def get_mello_joint_positions(self):
+        joint_positions = np.array(
+            requests.get(self.device_addr).json()["joint_positions"]
+        )
+        joint_positions = np.deg2rad(joint_positions)
+        joint_positions *= -1
+        return joint_positions
+
+    def get_action(self) -> np.ndarray:
+        mello_joint_positions = self.get_mello_joint_positions()
+        robot_joint_positions = self.robot.get_proprioception_data(False, False)[
+            "joint_positions"
+        ]
+        delta = mello_joint_positions - robot_joint_positions
+        delta = np.clip(delta, -self.max_delta, self.max_delta)
+        new_joint_positions = robot_joint_positions + delta
+        return new_joint_positions
+
+    def wait_for_pose(self):
+        delta = np.ones_like(self.get_mello_joint_positions()) * self.max_delta
+        while np.any(delta >= self.max_delta):
+            mello_joint_positions = self.get_mello_joint_positions()
+            robot_joint_positions = self.robot.get_joint_obs(False, False)[
+                "joint_positions"
+            ]
+            delta = np.abs(mello_joint_positions - robot_joint_positions)
+            logger.info(
+                "Delta between robot and mello is too large %s. Should be with %s",
+                np.rad2deg(delta),
+                np.rad2deg(self.max_delta),
+            )
+            time.sleep(0.25)
+        logger.info("Done.")

robits-0.5.0/robits/agents/sam2act_agent.py

@@ -0,0 +1,29 @@
+import logging
+
+import torch
+
+from sam2act.eval import load_agent
+from robits.agents.base_agent import BaseAgent
+
+logger = logging.getLogger(__name__)
+
+
+class SAM2Act(BaseAgent):
+
+    def __init__(self, model_path):
+        self.device = torch.device("cuda:0")
+
+        self.agent = load_agent(model_path)
+        self.agent.load_clip()
+        self.agent.eval()
+        self.lang_goal = "push the buttons in the following order: red, green, blue"
+
+    def get_action(self, step, observation):
+
+        if not observation:
+            logger.error("Nothing todo.")
+            return
+
+        with torch.jit.optimized_execution(False):
+            act_result = self.agent.act(step, observation, deterministic=True)
+        return act_result

robits-0.5.0/robits/app/__init__.py

@@ -0,0 +1,13 @@
+"""
+Interactive applications for the RoBits framework.
+
+This package provides user interface applications for interacting with
+robots, cameras, and other components of the RoBits system. These applications
+use terminal-based UI libraries to provide graphical interfaces that work
+in both terminal and GUI environments.
+
+Components:
+
+- robot_app: An interactive TUI for controlling robot arms
+- extrinsics_app: An application for calibrating camera extrinsics
+"""