dora-rs 0.3.14rc0__cp37-abi3-manylinux_2_28_x86_64.whl

dora/__init__.py

@@ -0,0 +1,42 @@
+""" 
+# dora-rs.
+
+This is the dora python client for interacting with dora dataflow.
+You can install it via:
+```bash
+pip install dora-rs
+```.
+"""
+
+from enum import Enum
+
+from .dora import *
+from .dora import (
+    Node,
+    Ros2Context,
+    Ros2Durability,
+    Ros2Liveliness,
+    Ros2Node,
+    Ros2NodeOptions,
+    Ros2Publisher,
+    Ros2QosPolicies,
+    Ros2Subscription,
+    Ros2Topic,
+    __author__,
+    __version__,
+    start_runtime,
+)
+
+
+class DoraStatus(Enum):
+    """Dora status to indicate if operator `on_input` loop should be stopped.
+
+    Args:
+        Enum (u8): Status signaling to dora operator to
+        stop or continue the operator.
+
+    """
+
+    CONTINUE = 0
+    STOP = 1
+    STOP_ALL = 2

dora/__init__.pyi

@@ -0,0 +1,461 @@
+import typing
+
+import pyarrow
+
+import dora
+
+@typing.final
+class Enum:
+    """Create a collection of name/value pairs.
+
+    Example enumeration:
+
+    >>> class Color(Enum):
+    ...     RED = 1
+    ...     BLUE = 2
+    ...     GREEN = 3
+
+    Access them by:
+
+    - attribute access:
+
+    >>> Color.RED
+    <Color.RED: 1>
+
+    - value lookup:
+
+    >>> Color(1)
+    <Color.RED: 1>
+
+    - name lookup:
+
+    >>> Color['RED']
+    <Color.RED: 1>
+
+    Enumerations can be iterated over, and know how many members they have:
+
+    >>> len(Color)
+    3
+
+    >>> list(Color)
+    [<Color.RED: 1>, <Color.BLUE: 2>, <Color.GREEN: 3>]
+
+    Methods can be added to enumerations, and members can have their own
+    attributes -- see the documentation for details."""
+
+    @staticmethod
+    def __contains__(value: typing.Any) -> bool:
+        """Return True if `value` is in `cls`.
+
+        `value` is in `cls` if:
+        1) `value` is a member of `cls`, or
+        2) `value` is the value of one of the `cls`'s members."""
+
+    @staticmethod
+    def __getitem__(name: typing.Any) -> typing.Any:
+        """Return the member matching `name`."""
+
+    @staticmethod
+    def __iter__() -> typing.Any:
+        """Return members in definition order."""
+
+    @staticmethod
+    def __len__() -> int:
+        """Return the number of members (no aliases)"""
+
+@typing.final
+class Node:
+    """The custom node API lets you integrate `dora` into your application.
+    It allows you to retrieve input and send output in any fashion you want.
+
+    Use with:
+
+    ```python
+    from dora import Node
+
+    node = Node()
+    ```"""
+
+    def __init__(self, node_id: str = None) -> None:
+        """The custom node API lets you integrate `dora` into your application.
+        It allows you to retrieve input and send output in any fashion you want.
+
+        Use with:
+
+        ```python
+        from dora import Node
+
+        node = Node()
+        ```"""
+
+    def dataflow_descriptor(self) -> dict:
+        """Returns the full dataflow descriptor that this node is part of.
+
+        This method returns the parsed dataflow YAML file."""
+
+    def dataflow_id(self) -> str:
+        """Returns the dataflow id."""
+
+    def merge_external_events(self, subscription: dora.Ros2Subscription) -> None:
+        """Merge an external event stream with dora main loop.
+        This currently only work with ROS2."""
+
+    def next(self, timeout: float = None) -> dict:
+        """`.next()` gives you the next input that the node has received.
+        It blocks until the next event becomes available.
+        You can use timeout in seconds to return if no input is available.
+        It will return `None` when all senders has been dropped.
+
+        ```python
+        event = node.next()
+        ```
+
+        You can also iterate over the event stream with a loop
+
+        ```python
+        for event in node:
+        match event["type"]:
+        case "INPUT":
+        match event["id"]:
+        case "image":
+        ```"""
+
+    def node_config(self) -> dict:
+        """Returns the node configuration."""
+
+    def recv_async(self, timeout: float = None) -> dict:
+        """`.recv_async()` gives you the next input that the node has received asynchronously.
+        It does not blocks until the next event becomes available.
+        You can use timeout in seconds to return if no input is available.
+        It will return an Error if the timeout is reached.
+        It will return `None` when all senders has been dropped.
+
+        warning::
+        This feature is experimental as pyo3 async (rust-python FFI) is still in development.
+
+        ```python
+        event = await node.recv_async()
+        ```
+
+        You can also iterate over the event stream with a loop"""
+
+    def send_output(
+        self, output_id: str, data: pyarrow.Array, metadata: dict = None
+    ) -> None:
+        """`send_output` send data from the node.
+
+        ```python
+        Args:
+        output_id: str,
+        data: pyarrow.Array,
+        metadata: Option[Dict],
+        ```
+
+        ex:
+
+        ```python
+        node.send_output("string", b"string", {"open_telemetry_context": "7632e76"})
+        ```"""
+
+    def __iter__(self) -> typing.Any:
+        """Implement iter(self)."""
+
+    def __next__(self) -> typing.Any:
+        """Implement next(self)."""
+
+    def __repr__(self) -> str:
+        """Return repr(self)."""
+
+    def __str__(self) -> str:
+        """Return str(self)."""
+
+@typing.final
+class Ros2Context:
+    """ROS2 Context holding all messages definition for receiving and sending messages to ROS2.
+
+    By default, Ros2Context will use env `AMENT_PREFIX_PATH` to search for message definition.
+
+    AMENT_PREFIX_PATH folder structure should be the following:
+
+    - For messages: <namespace>/msg/<name>.msg
+    - For services: <namespace>/srv/<name>.srv
+
+    You can also use `ros_paths` if you don't want to use env variable.
+
+    warning::
+    dora Ros2 bridge functionality is considered **unstable**. It may be changed
+    at any point without it being considered a breaking change.
+
+    ```python
+    context = Ros2Context()
+    ```"""
+
+    def __init__(self, ros_paths: typing.List[str] = None) -> None:
+        """ROS2 Context holding all messages definition for receiving and sending messages to ROS2.
+
+        By default, Ros2Context will use env `AMENT_PREFIX_PATH` to search for message definition.
+
+        AMENT_PREFIX_PATH folder structure should be the following:
+
+        - For messages: <namespace>/msg/<name>.msg
+        - For services: <namespace>/srv/<name>.srv
+
+        You can also use `ros_paths` if you don't want to use env variable.
+
+        warning::
+        dora Ros2 bridge functionality is considered **unstable**. It may be changed
+        at any point without it being considered a breaking change.
+
+        ```python
+        context = Ros2Context()
+        ```"""
+
+    def new_node(
+        self, name: str, namespace: str, options: dora.Ros2NodeOptions
+    ) -> dora.Ros2Node:
+        """Create a new ROS2 node
+
+        ```python
+        ros2_node = ros2_context.new_node(
+        "turtle_teleop",
+        "/ros2_demo",
+        Ros2NodeOptions(rosout=True),
+        )
+        ```
+
+        warning::
+        dora Ros2 bridge functionality is considered **unstable**. It may be changed
+        at any point without it being considered a breaking change."""
+
+    def __repr__(self) -> str:
+        """Return repr(self)."""
+
+    def __str__(self) -> str:
+        """Return str(self)."""
+
+@typing.final
+class Ros2Durability:
+    """DDS 2.2.3.4 DURABILITY"""
+
+    def __eq__(self, value: typing.Any) -> bool:
+        """Return self==value."""
+
+    def __ge__(self, value: typing.Any) -> bool:
+        """Return self>=value."""
+
+    def __gt__(self, value: typing.Any) -> bool:
+        """Return self>value."""
+
+    def __int__(self) -> None:
+        """int(self)"""
+
+    def __le__(self, value: typing.Any) -> bool:
+        """Return self<=value."""
+
+    def __lt__(self, value: typing.Any) -> bool:
+        """Return self<value."""
+
+    def __ne__(self, value: typing.Any) -> bool:
+        """Return self!=value."""
+
+    def __repr__(self) -> str:
+        """Return repr(self)."""
+
+    def __str__(self) -> str:
+        """Return str(self)."""
+
+@typing.final
+class Ros2Liveliness:
+    """DDS 2.2.3.11 LIVELINESS"""
+
+    def __eq__(self, value: typing.Any) -> bool:
+        """Return self==value."""
+
+    def __ge__(self, value: typing.Any) -> bool:
+        """Return self>=value."""
+
+    def __gt__(self, value: typing.Any) -> bool:
+        """Return self>value."""
+
+    def __int__(self) -> None:
+        """int(self)"""
+
+    def __le__(self, value: typing.Any) -> bool:
+        """Return self<=value."""
+
+    def __lt__(self, value: typing.Any) -> bool:
+        """Return self<value."""
+
+    def __ne__(self, value: typing.Any) -> bool:
+        """Return self!=value."""
+
+    def __repr__(self) -> str:
+        """Return repr(self)."""
+
+    def __str__(self) -> str:
+        """Return str(self)."""
+
+@typing.final
+class Ros2Node:
+    """ROS2 Node
+
+    warnings::
+    - dora Ros2 bridge functionality is considered **unstable**. It may be changed
+    at any point without it being considered a breaking change.
+    - There's a known issue about ROS2 nodes not being discoverable by ROS2
+    See: https://github.com/jhelovuo/ros2-client/issues/4"""
+
+    def create_publisher(
+        self, topic: dora.Ros2Topic, qos: dora.Ros2QosPolicies = None
+    ) -> dora.Ros2Publisher:
+        """Create a ROS2 publisher
+
+        ```python
+        pose_publisher = ros2_node.create_publisher(turtle_pose_topic)
+        ```
+        warnings:
+        - dora Ros2 bridge functionality is considered **unstable**. It may be changed
+        at any point without it being considered a breaking change."""
+
+    def create_subscription(
+        self, topic: dora.Ros2Topic, qos: dora.Ros2QosPolicies = None
+    ) -> dora.Ros2Subscription:
+        """Create a ROS2 subscription
+
+        ```python
+        pose_reader = ros2_node.create_subscription(turtle_pose_topic)
+        ```
+
+        warnings:
+        - dora Ros2 bridge functionality is considered **unstable**. It may be changed
+        at any point without it being considered a breaking change."""
+
+    def create_topic(
+        self, name: str, message_type: str, qos: dora.Ros2QosPolicies
+    ) -> dora.Ros2Topic:
+        """Create a ROS2 topic to connect to.
+
+        ```python
+        turtle_twist_topic = ros2_node.create_topic(
+        "/turtle1/cmd_vel", "geometry_msgs/Twist", topic_qos
+        )
+        ```"""
+
+    def __repr__(self) -> str:
+        """Return repr(self)."""
+
+    def __str__(self) -> str:
+        """Return str(self)."""
+
+@typing.final
+class Ros2NodeOptions:
+    """ROS2 Node Options"""
+
+    def __init__(self, rosout: bool = None) -> None:
+        """ROS2 Node Options"""
+
+    def __repr__(self) -> str:
+        """Return repr(self)."""
+
+    def __str__(self) -> str:
+        """Return str(self)."""
+
+@typing.final
+class Ros2Publisher:
+    """ROS2 Publisher
+
+    warnings:
+    - dora Ros2 bridge functionality is considered **unstable**. It may be changed
+    at any point without it being considered a breaking change."""
+
+    def publish(self, data: pyarrow.Array) -> None:
+        """Publish a message into ROS2 topic.
+
+        Remember that the data format should respect the structure of the ROS2 message using an arrow Structure.
+
+        ex:
+        ```python
+        gripper_command.publish(
+        pa.array(
+        [
+        {
+        "name": "gripper",
+        "cmd": np.float32(5),
+        }
+        ]
+        ),
+        )
+        ```"""
+
+    def __repr__(self) -> str:
+        """Return repr(self)."""
+
+    def __str__(self) -> str:
+        """Return str(self)."""
+
+@typing.final
+class Ros2QosPolicies:
+    """ROS2 QoS Policy"""
+
+    def __init__(
+        self,
+        durability: dora.Ros2Durability = None,
+        liveliness: dora.Ros2Liveliness = None,
+        reliable: bool = None,
+        keep_all: bool = None,
+        lease_duration: float = None,
+        max_blocking_time: float = None,
+        keep_last: int = None,
+    ) -> dora.Ros2QosPolicies:
+        """ROS2 QoS Policy"""
+
+    def __repr__(self) -> str:
+        """Return repr(self)."""
+
+    def __str__(self) -> str:
+        """Return str(self)."""
+
+@typing.final
+class Ros2Subscription:
+    """ROS2 Subscription
+
+
+    warnings:
+    - dora Ros2 bridge functionality is considered **unstable**. It may be changed
+    at any point without it being considered a breaking change."""
+
+    def next(self): ...
+    def __repr__(self) -> str:
+        """Return repr(self)."""
+
+    def __str__(self) -> str:
+        """Return str(self)."""
+
+@typing.final
+class Ros2Topic:
+    """ROS2 Topic
+
+    warnings:
+    - dora Ros2 bridge functionality is considered **unstable**. It may be changed
+    at any point without it being considered a breaking change."""
+
+    def __repr__(self) -> str:
+        """Return repr(self)."""
+
+    def __str__(self) -> str:
+        """Return str(self)."""
+
+def build(
+    dataflow_path: str,
+    uv: bool = None,
+    coordinator_addr: str = None,
+    coordinator_port: int = None,
+    force_local: bool = False,
+) -> None:
+    """Build a Dataflow, exactly the same way as `dora build` command line tool."""
+
+def run(dataflow_path: str, uv: bool = None) -> None:
+    """Run a Dataflow, exactly the same way as `dora run` command line tool."""
+
+def start_runtime() -> None:
+    """Start a runtime for Operators"""

dora/builder.py

@@ -0,0 +1,166 @@
+from __future__ import annotations
+
+import yaml
+
+
+class DataflowBuilder:
+    """A dora dataflow."""
+
+    def __init__(self, name: str = "dora-dataflow"):
+        self.name = name
+        self.nodes = []
+
+    def add_node(self, id: str, **kwargs) -> Node:
+        """Adds a new node to the dataflow."""
+        node = Node(id, **kwargs)
+        self.nodes.append(node)
+        return node
+
+    def to_yaml(self, path: str = None) -> str | None:
+        """Generates the YAML representation of the dataflow."""
+        dataflow_spec = {"nodes": [node.to_dict() for node in self.nodes]}
+        if path:
+            with open(path, "w") as f:
+                yaml.dump(dataflow_spec, f)
+            return None
+        else:
+            return yaml.dump(dataflow_spec)
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        pass
+
+
+class Output:
+    """Represents an output from a node."""
+
+    def __init__(self, node: Node, output_id: str):
+        self.node = node
+        self.output_id = output_id
+
+    def __str__(self) -> str:
+        return f"{self.node.id}/{self.output_id}"
+
+
+class Node:
+    """A node in a dora dataflow."""
+
+    def __init__(self, id: str, **kwargs):
+        self.id = id
+        self.config = kwargs
+        self.config["id"] = id
+        self.operators = []
+
+    def path(self, path: str) -> Node:
+        """Sets the path to the executable or script."""
+        self.config["path"] = path
+        return self
+
+    def args(self, args: str) -> Node:
+        """Sets the command-line arguments for the node."""
+        self.config["args"] = args
+        return self
+
+    def env(self, env: dict) -> Node:
+        """Sets the environment variables for the node."""
+        self.config["env"] = env
+        return self
+
+    def build(self, build_command: str) -> Node:
+        """Sets the build command for the node."""
+        self.config["build"] = build_command
+        return self
+
+    def git(
+        self, url: str, branch: str = None, tag: str = None, rev: str = None
+    ) -> Node:
+        """Sets the Git repository for the node."""
+        self.config["git"] = url
+        if branch:
+            self.config["branch"] = branch
+        if tag:
+            self.config["tag"] = tag
+        if rev:
+            self.config["rev"] = rev
+        return self
+
+    def add_operator(self, operator: Operator) -> Node:
+        """Adds an operator to this node."""
+        self.operators.append(operator)
+        return self
+
+    def add_output(self, output_id: str) -> Output:
+        """Adds an output to the node and returns an Output object."""
+        if "outputs" not in self.config:
+            self.config["outputs"] = []
+        if output_id not in self.config["outputs"]:
+            self.config["outputs"].append(output_id)
+        return Output(self, output_id)
+
+    def add_input(
+        self, input_id: str, source: str | Output, queue_size: int = None
+    ) -> Node:
+        """Adds a user-defined input to the node. Source can be a string or an Output object."""
+        if "inputs" not in self.config:
+            self.config["inputs"] = {}
+
+        if isinstance(source, Output):
+            source_str = str(source)
+            if queue_size is not None:
+                self.config["inputs"][input_id] = {
+                    "source": source_str,
+                    "queue_size": queue_size,
+                }
+            else:
+                self.config["inputs"][input_id] = source_str
+        else:
+            if queue_size is not None:
+                self.config["inputs"][input_id] = {
+                    "source": source,
+                    "queue_size": queue_size,
+                }
+            else:
+                self.config["inputs"][input_id] = source
+        return self
+
+    def to_dict(self) -> dict:
+        """Returns the dictionary representation of the node."""
+        config = self.config.copy()
+        if self.operators:
+            config["operators"] = [op.to_dict() for op in self.operators]
+        return config
+
+
+class Operator:
+    """An operator in a dora dataflow."""
+
+    def __init__(
+        self,
+        id: str,
+        name: str = None,
+        description: str = None,
+        build: str = None,
+        python: str = None,
+        shared_library: str = None,
+        send_stdout_as: str = None,
+    ):
+        self.id = id
+        self.config = {"id": id}
+        if name:
+            self.config["name"] = name
+        if description:
+            self.config["description"] = description
+        if build:
+            self.config["build"] = build
+        if python:
+            self.config["python"] = python
+        if shared_library:
+            self.config["shared-library"] = shared_library
+        if send_stdout_as:
+            self.config["send_stdout_as"] = send_stdout_as
+
+    def to_dict(self) -> dict:
+        """Returns the dictionary representation of the operator."""
+        return self.config

dora/cuda.py

@@ -0,0 +1,94 @@
+"""TODO: Add docstring."""
+
+import pyarrow as pa
+
+# Make sure to install torch with cuda
+import torch
+from numba.cuda import to_device
+
+# Make sure to install numba with cuda
+from numba.cuda.cudadrv.devicearray import DeviceNDArray
+from numba.cuda.cudadrv.devices import get_context
+from numba.cuda.cudadrv.driver import IpcHandle
+
+
+import json
+
+from contextlib import contextmanager
+from typing import ContextManager
+
+
+def torch_to_ipc_buffer(tensor: torch.TensorType) -> tuple[pa.array, dict]:
+    """Convert a Pytorch tensor into a pyarrow buffer containing the IPC handle
+    and its metadata.
+
+    Example Use:
+    ```python
+    torch_tensor = torch.tensor(random_data, dtype=torch.int64, device="cuda")
+    ipc_buffer, metadata = torch_to_ipc_buffer(torch_tensor)
+    node.send_output("latency", ipc_buffer, metadata)
+    ```
+    """
+    device_arr = to_device(tensor)
+    ipch = get_context().get_ipc_handle(device_arr.gpu_data)
+    _, handle, size, source_info, offset = ipch.__reduce__()[1]
+    metadata = {
+        "shape": device_arr.shape,
+        "strides": device_arr.strides,
+        "dtype": device_arr.dtype.str,
+        "size": size,
+        "offset": offset,
+        "source_info": json.dumps(source_info),
+    }
+    return pa.array(handle, pa.int8()), metadata
+
+
+def ipc_buffer_to_ipc_handle(handle_buffer: pa.array, metadata: dict) -> IpcHandle:
+    """Convert a buffer containing a serialized handler into cuda IPC Handle.
+
+    example use:
+    ```python
+    from dora.cuda import ipc_buffer_to_ipc_handle, open_ipc_handle
+
+    event = node.next()
+
+    ipc_handle = ipc_buffer_to_ipc_handle(event["value"], event["metadata"])
+    with open_ipc_handle(ipc_handle, event["metadata"]) as tensor:
+        pass
+    ```
+    """
+    handle = handle_buffer.to_pylist()
+    return IpcHandle._rebuild(
+        handle,
+        metadata["size"],
+        json.loads(metadata["source_info"]),
+        metadata["offset"],
+    )
+
+
+@contextmanager
+def open_ipc_handle(
+    ipc_handle: IpcHandle, metadata: dict
+) -> ContextManager[torch.TensorType]:
+    """Open a CUDA IPC handle and return a Pytorch tensor.
+
+    example use:
+    ```python
+    from dora.cuda import ipc_buffer_to_ipc_handle, open_ipc_handle
+
+    event = node.next()
+
+    ipc_handle = ipc_buffer_to_ipc_handle(event["value"], event["metadata"])
+    with open_ipc_handle(ipc_handle, event["metadata"]) as tensor:
+        pass
+    ```
+    """
+    shape = metadata["shape"]
+    strides = metadata["strides"]
+    dtype = metadata["dtype"]
+    try:
+        buffer = ipc_handle.open(get_context())
+        device_arr = DeviceNDArray(shape, strides, dtype, gpu_data=buffer)
+        yield torch.as_tensor(device_arr, device="cuda")
+    finally:
+        ipc_handle.close()

dora_rs-0.3.14rc0.dist-info/METADATA

@@ -0,0 +1,29 @@
+Metadata-Version: 2.4
+Name: dora-rs
+Version: 0.3.14rc0
+Requires-Dist: pyarrow
+Requires-Dist: pyyaml>=6.0
+License: MIT
+Requires-Python: >=3.7
+Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
+
+This crate corresponds to the Node API for Dora.
+
+## Building
+
+To build the Python module for development:
+
+```bash
+uv venv --seed -p 3.11
+uv pip install -e .
+```
+
+## Type hinting
+
+Type hinting requires to run a second step
+
+```bash
+python generate_stubs.py dora dora/__init__.pyi
+maturin develop
+```
+

dora_rs-0.3.14rc0.dist-info/RECORD

@@ -0,0 +1,9 @@
+dora/__init__.py,sha256=mogA09uYFBIwnBnnMfeQtD52ZJbVbWmWo1h70GZj2aU,709
+dora/__init__.pyi,sha256=mWpvxM1zQ5Ewvwmy-PRRGxyPvXHtpWKlo5U958Sxrkw,12469
+dora/builder.py,sha256=WqYuhtm8tCOlxEbGvsRiTfW0Ji84P1MmA4FIRfpbWQo,4956
+dora/cuda.py,sha256=tsp76SLAeR1gOXoQTTMv4oVHFoSzkXeevEQG41U9xFU,2736
+dora/dora.abi3.so,sha256=NtA4MxKVty2Df_qnCuAyhmW5uHF32PgAGbjizVuSsEM,44810632
+dora/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dora_rs-0.3.14rc0.dist-info/METADATA,sha256=sOEcV2em-UqTryZBeGJkSsq_TZtBOTcoYgF89irD7os,512
+dora_rs-0.3.14rc0.dist-info/WHEEL,sha256=4Z4Wh-oqVxwrJfLhZsSx6Sb2NPMEZBoDWAurZQhih_g,107
+dora_rs-0.3.14rc0.dist-info/RECORD,,

dora_rs-0.3.14rc0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: maturin (1.11.2)
+Root-Is-Purelib: false
+Tag: cp37-abi3-manylinux_2_28_x86_64