dora-rs 0.3.9rc2__cp37-abi3-musllinux_1_2_armv7l.whl

dora/__init__.py

@@ -0,0 +1,42 @@
+""" 
+# dora-rs
+This is the dora python client for interacting with dora dataflow.
+You can install it via:
+```bash
+pip install dora-rs
+```
+"""
+
+from enum import Enum
+
+from .dora import *
+from .dora import (
+    Node,
+    Ros2Context,
+    Ros2Durability,
+    Ros2Liveliness,
+    Ros2Node,
+    Ros2NodeOptions,
+    Ros2Publisher,
+    Ros2QosPolicies,
+    Ros2Subscription,
+    Ros2Topic,
+    __author__,
+    __version__,
+    start_runtime,
+)
+
+
+class DoraStatus(Enum):
+    """Dora status to indicate if operator `on_input` loop
+     should be stopped.
+
+    Args:
+        Enum (u8): Status signaling to dora operator to
+        stop or continue the operator.
+
+    """
+
+    CONTINUE = 0
+    STOP = 1
+    STOP_ALL = 2

dora/__init__.pyi

@@ -0,0 +1,311 @@
+import dora
+import pyarrow
+import typing
+
+@typing.final
+class Enum:
+    """Generic enumeration.
+
+Derive from this class to define new enumerations."""
+    __members__: mappingproxy = ...
+
+@typing.final
+class Node:
+    """The custom node API lets you integrate `dora` into your application.
+It allows you to retrieve input and send output in any fashion you want.
+
+Use with:
+
+```python
+from dora import Node
+
+node = Node()
+```"""
+
+    def __init__(self, node_id: str=None) -> None:
+        """The custom node API lets you integrate `dora` into your application.
+It allows you to retrieve input and send output in any fashion you want.
+
+Use with:
+
+```python
+from dora import Node
+
+node = Node()
+```"""
+
+    def dataflow_descriptor(self) -> dict:
+        """Returns the full dataflow descriptor that this node is part of.
+
+This method returns the parsed dataflow YAML file."""
+
+    def dataflow_id(self) -> str:
+        """Returns the dataflow id."""
+
+    def merge_external_events(self, subscription: dora.Ros2Subscription) -> None:
+        """Merge an external event stream with dora main loop.
+This currently only work with ROS2."""
+
+    def next(self, timeout: float=None) -> dict:
+        """`.next()` gives you the next input that the node has received.
+It blocks until the next event becomes available.
+You can use timeout in seconds to return if no input is available.
+It will return `None` when all senders has been dropped.
+
+```python
+event = node.next()
+```
+
+You can also iterate over the event stream with a loop
+
+```python
+for event in node:
+match event["type"]:
+case "INPUT":
+match event["id"]:
+case "image":
+```"""
+
+    def send_output(self, output_id: str, data: pyarrow.Array, metadata: dict=None) -> None:
+        """`send_output` send data from the node.
+
+```python
+Args:
+output_id: str,
+data: pyarrow.Array,
+metadata: Option[Dict],
+```
+
+ex:
+
+```python
+node.send_output("string", b"string", {"open_telemetry_context": "7632e76"})
+```"""
+
+    def __iter__(self) -> typing.Any:
+        """Implement iter(self)."""
+
+    def __next__(self) -> typing.Any:
+        """Implement next(self)."""
+
+@typing.final
+class Ros2Context:
+    """ROS2 Context holding all messages definition for receiving and sending messages to ROS2.
+
+By default, Ros2Context will use env `AMENT_PREFIX_PATH` to search for message definition.
+
+AMENT_PREFIX_PATH folder structure should be the following:
+
+- For messages: <namespace>/msg/<name>.msg
+- For services: <namespace>/srv/<name>.srv
+
+You can also use `ros_paths` if you don't want to use env variable.
+
+warning::
+dora Ros2 bridge functionality is considered **unstable**. It may be changed
+at any point without it being considered a breaking change.
+
+```python
+context = Ros2Context()
+```"""
+
+    def __init__(self, ros_paths: typing.List[str]=None) -> None:
+        """ROS2 Context holding all messages definition for receiving and sending messages to ROS2.
+
+By default, Ros2Context will use env `AMENT_PREFIX_PATH` to search for message definition.
+
+AMENT_PREFIX_PATH folder structure should be the following:
+
+- For messages: <namespace>/msg/<name>.msg
+- For services: <namespace>/srv/<name>.srv
+
+You can also use `ros_paths` if you don't want to use env variable.
+
+warning::
+dora Ros2 bridge functionality is considered **unstable**. It may be changed
+at any point without it being considered a breaking change.
+
+```python
+context = Ros2Context()
+```"""
+
+    def new_node(self, name: str, namespace: str, options: dora.Ros2NodeOptions) -> dora.Ros2Node:
+        """Create a new ROS2 node
+
+```python
+ros2_node = ros2_context.new_node(
+"turtle_teleop",
+"/ros2_demo",
+Ros2NodeOptions(rosout=True),
+)
+```
+
+warning::
+dora Ros2 bridge functionality is considered **unstable**. It may be changed
+at any point without it being considered a breaking change."""
+
+@typing.final
+class Ros2Durability:
+    """DDS 2.2.3.4 DURABILITY"""
+
+    def __eq__(self, value: typing.Any) -> bool:
+        """Return self==value."""
+
+    def __ge__(self, value: typing.Any) -> bool:
+        """Return self>=value."""
+
+    def __gt__(self, value: typing.Any) -> bool:
+        """Return self>value."""
+
+    def __int__(self) -> None:
+        """int(self)"""
+
+    def __le__(self, value: typing.Any) -> bool:
+        """Return self<=value."""
+
+    def __lt__(self, value: typing.Any) -> bool:
+        """Return self<value."""
+
+    def __ne__(self, value: typing.Any) -> bool:
+        """Return self!=value."""
+
+    def __repr__(self) -> str:
+        """Return repr(self)."""
+    Persistent: Ros2Durability = ...
+    Transient: Ros2Durability = ...
+    TransientLocal: Ros2Durability = ...
+    Volatile: Ros2Durability = ...
+
+@typing.final
+class Ros2Liveliness:
+    """DDS 2.2.3.11 LIVELINESS"""
+
+    def __eq__(self, value: typing.Any) -> bool:
+        """Return self==value."""
+
+    def __ge__(self, value: typing.Any) -> bool:
+        """Return self>=value."""
+
+    def __gt__(self, value: typing.Any) -> bool:
+        """Return self>value."""
+
+    def __int__(self) -> None:
+        """int(self)"""
+
+    def __le__(self, value: typing.Any) -> bool:
+        """Return self<=value."""
+
+    def __lt__(self, value: typing.Any) -> bool:
+        """Return self<value."""
+
+    def __ne__(self, value: typing.Any) -> bool:
+        """Return self!=value."""
+
+    def __repr__(self) -> str:
+        """Return repr(self)."""
+    Automatic: Ros2Liveliness = ...
+    ManualByParticipant: Ros2Liveliness = ...
+    ManualByTopic: Ros2Liveliness = ...
+
+@typing.final
+class Ros2Node:
+    """ROS2 Node
+
+warnings::
+- dora Ros2 bridge functionality is considered **unstable**. It may be changed
+at any point without it being considered a breaking change.
+- There's a known issue about ROS2 nodes not being discoverable by ROS2
+See: https://github.com/jhelovuo/ros2-client/issues/4"""
+
+    def create_publisher(self, topic: dora.Ros2Topic, qos: dora.Ros2QosPolicies=None) -> dora.Ros2Publisher:
+        """Create a ROS2 publisher
+
+```python
+pose_publisher = ros2_node.create_publisher(turtle_pose_topic)
+```
+warnings:
+- dora Ros2 bridge functionality is considered **unstable**. It may be changed
+at any point without it being considered a breaking change."""
+
+    def create_subscription(self, topic: dora.Ros2Topic, qos: dora.Ros2QosPolicies=None) -> dora.Ros2Subscription:
+        """Create a ROS2 subscription
+
+```python
+pose_reader = ros2_node.create_subscription(turtle_pose_topic)
+```
+
+warnings:
+- dora Ros2 bridge functionality is considered **unstable**. It may be changed
+at any point without it being considered a breaking change."""
+
+    def create_topic(self, name: str, message_type: str, qos: dora.Ros2QosPolicies) -> dora.Ros2Topic:
+        """Create a ROS2 topic to connect to.
+
+```python
+turtle_twist_topic = ros2_node.create_topic(
+"/turtle1/cmd_vel", "geometry_msgs/Twist", topic_qos
+)
+```"""
+
+@typing.final
+class Ros2NodeOptions:
+    """ROS2 Node Options"""
+
+    def __init__(self, rosout: bool=None) -> None:
+        """ROS2 Node Options"""
+
+@typing.final
+class Ros2Publisher:
+    """ROS2 Publisher
+
+warnings:
+- dora Ros2 bridge functionality is considered **unstable**. It may be changed
+at any point without it being considered a breaking change."""
+
+    def publish(self, data: pyarrow.Array) -> None:
+        """Publish a message into ROS2 topic.
+
+Remember that the data format should respect the structure of the ROS2 message using an arrow Structure.
+
+ex:
+```python
+gripper_command.publish(
+pa.array(
+[
+{
+"name": "gripper",
+"cmd": np.float32(5),
+}
+]
+),
+)
+```"""
+
+@typing.final
+class Ros2QosPolicies:
+    """ROS2 QoS Policy"""
+
+    def __init__(self, durability: dora.Ros2Durability=None, liveliness: dora.Ros2Liveliness=None, reliable: bool=None, keep_all: bool=None, lease_duration: float=None, max_blocking_time: float=None, keep_last: int=None) -> dora.Ros2QoSPolicies:
+        """ROS2 QoS Policy"""
+
+@typing.final
+class Ros2Subscription:
+    """ROS2 Subscription
+
+
+warnings:
+- dora Ros2 bridge functionality is considered **unstable**. It may be changed
+at any point without it being considered a breaking change."""
+
+    def next(self):...
+
+@typing.final
+class Ros2Topic:
+    """ROS2 Topic
+
+warnings:
+- dora Ros2 bridge functionality is considered **unstable**. It may be changed
+at any point without it being considered a breaking change."""
+
+def start_runtime() -> None:
+    """Start a runtime for Operators"""

dora/cuda.py

@@ -0,0 +1,100 @@
+import pyarrow as pa
+
+# Make sure to install torch with cuda
+import torch
+from numba.cuda import to_device
+
+# Make sure to install numba with cuda
+from numba.cuda.cudadrv.devicearray import DeviceNDArray
+
+# To install pyarrow.cuda, run `conda install pyarrow "arrow-cpp-proc=*=cuda" -c conda-forge`
+from pyarrow import cuda
+
+
+def torch_to_ipc_buffer(tensor: torch.TensorType) -> tuple[pa.array, dict]:
+    """Converts a Pytorch tensor into a pyarrow buffer containing the IPC handle and its metadata.
+
+    Example Use:
+    ```python
+    torch_tensor = torch.tensor(random_data, dtype=torch.int64, device="cuda")
+    ipc_buffer, metadata = torch_to_ipc_buffer(torch_tensor)
+    node.send_output("latency", ipc_buffer, metadata)
+    ```
+    """
+    device_arr = to_device(tensor)
+    cuda_buf = pa.cuda.CudaBuffer.from_numba(device_arr.gpu_data)
+    handle_buffer = cuda_buf.export_for_ipc().serialize()
+    metadata = {
+        "shape": device_arr.shape,
+        "strides": device_arr.strides,
+        "dtype": device_arr.dtype.str,
+    }
+    return pa.array(handle_buffer, type=pa.uint8()), metadata
+
+
+def ipc_buffer_to_ipc_handle(handle_buffer: pa.array) -> cuda.IpcMemHandle:
+    """Converts a buffer containing a serialized handler into cuda IPC MemHandle.
+
+    example use:
+    ```python
+
+    import pyarrow as pa
+    from dora.cuda import ipc_buffer_to_ipc_handle, cudabuffer_to_torch
+
+    ctx = pa.cuda.context()
+    event = node.next()
+
+    ipc_handle = ipc_buffer_to_ipc_handle(event["value"])
+    cudabuffer = ctx.open_ipc_buffer(ipc_handle)
+    torch_tensor = cudabuffer_to_torch(cudabuffer, event["metadata"])  # on cuda
+    ```
+    """
+    handle_buffer = handle_buffer.buffers()[1]
+    ipc_handle = pa.cuda.IpcMemHandle.from_buffer(handle_buffer)
+    return ipc_handle
+
+
+def cudabuffer_to_numba(buffer: cuda.CudaBuffer, metadata: dict) -> DeviceNDArray:
+    """Converts a pyarrow CUDA buffer to numba.
+
+    example use:
+    ```python
+
+    import pyarrow as pa
+    from dora.cuda import ipc_buffer_to_ipc_handle, cudabuffer_to_torch
+
+    ctx = pa.cuda.context()
+    event = node.next()
+
+    ipc_handle = ipc_buffer_to_ipc_handle(event["value"])
+    cudabuffer = ctx.open_ipc_buffer(ipc_handle)
+    numba_tensor = cudabuffer_to_numbda(cudabuffer, event["metadata"])
+    ```
+    """
+    shape = metadata["shape"]
+    strides = metadata["strides"]
+    dtype = metadata["dtype"]
+    device_arr = DeviceNDArray(shape, strides, dtype, gpu_data=buffer.to_numba())
+    return device_arr
+
+
+def cudabuffer_to_torch(buffer: cuda.CudaBuffer, metadata: dict) -> torch.Tensor:
+    """Converts a pyarrow CUDA buffer to a torch tensor.
+
+    example use:
+    ```python
+
+    import pyarrow as pa
+    from dora.cuda import ipc_buffer_to_ipc_handle, cudabuffer_to_torch
+
+    ctx = pa.cuda.context()
+    event = node.next()
+
+    ipc_handle = ipc_buffer_to_ipc_handle(event["value"])
+    cudabuffer = ctx.open_ipc_buffer(ipc_handle)
+    torch_tensor = cudabuffer_to_torch(cudabuffer, event["metadata"])  # on cuda
+    ```
+    """
+    device_arr = cudabuffer_to_numba(buffer, metadata)
+    torch_tensor = torch.as_tensor(device_arr, device="cuda")
+    return torch_tensor

dora_rs-0.3.9rc2.dist-info/METADATA

@@ -0,0 +1,32 @@
+Metadata-Version: 2.4
+Name: dora-rs
+Version: 0.3.9rc2
+Requires-Dist: pyarrow
+Summary: `dora` goal is to be a low latency, composable, and distributed data flow.
+License: MIT
+Requires-Python: >=3.7
+Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
+Project-URL: Source Code, https://github.com/dora-rs/dora/
+
+This crate corresponds to the Node API for Dora.
+
+## Building
+
+To build the Python module for development:
+
+```bash
+python -m venv .env
+source .env/bin/activate
+pip install maturin
+maturin develop
+```
+
+## Type hinting
+
+Type hinting requires to run a second step
+
+```bash
+python generate_stubs.py dora dora/__init__.pyi
+maturin develop
+```
+

dora_rs-0.3.9rc2.dist-info/RECORD

@@ -0,0 +1,8 @@
+dora_rs-0.3.9rc2.dist-info/METADATA,sha256=4aDA44TmaXVAckYLSoztWKfjJtjue7vcoC9RWTnXLhg,665
+dora_rs-0.3.9rc2.dist-info/WHEEL,sha256=_rxC7VyJxGZeonyKlCWBCllzF0Zu2ip4bWvmnNh7jMg,105
+dora.libs/libgcc_s-5b5488a6.so.1,sha256=HGKUsVmTeNAxEdSy7Ua5Vh_I9FN3RCbPWzvZ7H_TrwE,2749061
+dora/__init__.py,sha256=x3SdgwncpcHNnvLdsSc8Jrj1BBEC2uZVOCMYULyTrvo,711
+dora/__init__.pyi,sha256=MZfafEGo1F_Al2RLrD36_cGRfFgOYY8Qa1_14bFYcf0,8342
+dora/cuda.py,sha256=KuB-J8cykDkcbLjeSYWt4oz5lbkrKOUycUh3o1hfJRA,3161
+dora/dora.abi3.so,sha256=2DpTmhHlheLfdLtk66dCyLMkddQtRYI5dztYL5Y5fqk,21184405
+dora_rs-0.3.9rc2.dist-info/RECORD,,

dora_rs-0.3.9rc2.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: maturin (1.8.1)
+Root-Is-Purelib: false
+Tag: cp37-abi3-musllinux_1_2_armv7l