luckyrobots 0.1.63__py3-none-any.whl → 0.1.72__py3-none-any.whl

luckyrobots/grpc/proto/hazel_rpc.proto

@@ -0,0 +1,32 @@
+syntax = "proto3";
+
+// LuckyEngine / Hazel ScriptCore gRPC API (v1)
+//
+// This .proto is the *single source of truth* for the editor/runtime gRPC surface.
+// Code generators produce language-specific stubs:
+// - Client stubs: classes like `SceneServiceStub` / `AgentServiceStub` used by external tools
+// - Server bindings: base/service descriptors used by ScriptCore to host implementations
+//
+// In this repo:
+// - ScriptCore server host + impls live under `Hazel-ScriptCore/Source/Hazel/Net/Grpc/`
+// - C# generated files are checked in under `Hazel-ScriptCore/Source/Hazel/Net/Grpc/Generated/`
+// - A reference external Python client that generates Python stubs on-the-fly is `scripts/grpc_env.py`
+//
+// Compatibility note:
+// - Avoid renumbering existing field tags (the `= N` numbers). Add new fields with new tags instead.
+
+package hazel.rpc;
+
+option cc_enable_arenas = true;
+
+// This file is an umbrella entrypoint for the full API. The canonical message/service
+// definitions live in smaller domain-specific protos for readability.
+
+import "common.proto";
+import "scene.proto";
+import "mujoco.proto";
+import "telemetry.proto";
+import "media.proto";
+import "agent.proto";
+import "viewport.proto";
+import "camera.proto";

luckyrobots/grpc/proto/media.proto

@@ -0,0 +1,26 @@
+syntax = "proto3";
+
+// Media / image payload types shared across services.
+
+package hazel.rpc;
+
+option cc_enable_arenas = true;
+
+// Generic image frame. For `format="raw"`, `data` is tightly-packed pixels (usually RGBA).
+// For compressed formats, `data` is the encoded blob (e.g. JPEG bytes).
+message ImageFrame {
+    bytes data = 1;              // Raw pixel data (RGBA or compressed)
+    uint32 width = 2;
+    uint32 height = 3;
+    uint32 channels = 4;         // 3 for RGB, 4 for RGBA
+    string format = 5;           // "raw", "jpeg", "png"
+    uint64 timestamp_ms = 6;     // Timestamp in milliseconds
+    uint32 frame_number = 7;
+}
+
+// Named image payload for observation snapshots.
+// `name` can be a camera name or viewport name, depending on source.
+message NamedImageFrame {
+    string name = 1;
+    ImageFrame frame = 2;
+}

luckyrobots/grpc/proto/mujoco.proto

@@ -0,0 +1,64 @@
+syntax = "proto3";
+
+// MuJoCo service definitions for the LuckyEngine / Hazel ScriptCore gRPC API (v1).
+
+package hazel.rpc;
+
+option cc_enable_arenas = true;
+
+// MuJoCo joint state.
+message JointState {
+    repeated float positions = 1;   // qpos
+    repeated float velocities = 2;  // qvel
+    uint32 nq = 3;                  // Number of position coordinates
+    uint32 nv = 4;                  // Number of velocity coordinates
+}
+
+message GetJointStateRequest {
+    // Entity name or tag hint for which robot to target.
+    // Implementation may ignore this and use a "registered default robot" depending on current scene setup.
+    string robot_name = 1;
+}
+
+message GetJointStateResponse {
+    bool success = 1;
+    string message = 2;
+    JointState state = 3;
+}
+
+message SendControlRequest {
+    // Entity name or tag hint for which robot to target.
+    // Current Learn integration treats this as "set external actions for the next step".
+    string robot_name = 1;
+    repeated float controls = 2;    // Control input vector
+}
+
+message SendControlResponse {
+    bool success = 1;
+    string message = 2;
+}
+
+message GetMujocoInfoRequest {
+    // Entity name or tag hint.
+    string robot_name = 1;
+}
+
+message GetMujocoInfoResponse {
+    bool success = 1;
+    string message = 2;
+    uint32 nq = 3;
+    uint32 nv = 4;
+    uint32 nu = 5;  // Number of actuators
+    repeated string joint_names = 6;
+    repeated string actuator_names = 7;
+}
+
+// MuJoCo state + control IO.
+service MujocoService {
+    rpc GetJointState(GetJointStateRequest) returns (GetJointStateResponse);
+    rpc SendControl(SendControlRequest) returns (SendControlResponse);
+    rpc GetMujocoInfo(GetMujocoInfoRequest) returns (GetMujocoInfoResponse);
+
+    // Streaming variant for continuous state observation
+    rpc StreamJointState(GetJointStateRequest) returns (stream GetJointStateResponse);
+}

luckyrobots/grpc/proto/scene.proto

@@ -0,0 +1,104 @@
+syntax = "proto3";
+
+// Scene service definitions for the LuckyEngine / Hazel ScriptCore gRPC API (v1).
+
+package hazel.rpc;
+
+option cc_enable_arenas = true;
+
+import "common.proto";
+
+// Lightweight entity snapshot for editor tooling / inspection.
+// Depending on request flags, `transform` and/or `components` may be left default/empty.
+message EntityInfo {
+    EntityId id = 1;
+    string name = 2;
+    Transform transform = 3;
+    repeated string components = 4;  // List of component type names (best-effort)
+}
+
+// Fetch basic active-scene metadata.
+message GetSceneInfoRequest {}
+
+message GetSceneInfoResponse {
+    string scene_name = 1;
+    string scene_path = 2;
+    uint32 entity_count = 3;
+}
+
+// List all entities, optionally including extra data to avoid unnecessary bandwidth.
+message ListEntitiesRequest {
+    bool include_transforms = 1;
+    bool include_components = 2;
+}
+
+message ListEntitiesResponse {
+    repeated EntityInfo entities = 1;
+}
+
+// Fetch a single entity by ID or by name.
+message GetEntityRequest {
+    oneof identifier {
+        EntityId id = 1;
+        string name = 2;
+    }
+}
+
+message GetEntityResponse {
+    bool found = 1;
+    EntityInfo entity = 2;
+}
+
+// Set an entity's transform.
+// Note: server may clamp/ignore values depending on scene state.
+message SetEntityTransformRequest {
+    EntityId id = 1;
+    Transform transform = 2;
+}
+
+message SetEntityTransformResponse {
+    bool success = 1;
+    string message = 2;
+}
+
+// =============================================================================
+// Simulation Mode
+// =============================================================================
+
+// Simulation timing mode controls how physics stepping relates to wall-clock time.
+enum SimulationMode {
+    // Real-time: physics runs at 1x wall-clock speed (for visualization, games).
+    SIMULATION_MODE_REALTIME = 0;
+    // Deterministic: physics runs at fixed rate regardless of wall-clock (for reproducibility).
+    SIMULATION_MODE_DETERMINISTIC = 1;
+    // Fast: physics runs as fast as possible without real-time limiting (for RL training).
+    SIMULATION_MODE_FAST = 2;
+}
+
+message SetSimulationModeRequest {
+    SimulationMode mode = 1;
+}
+
+message SetSimulationModeResponse {
+    bool success = 1;
+    string message = 2;
+    SimulationMode current_mode = 3;
+}
+
+message GetSimulationModeRequest {}
+
+message GetSimulationModeResponse {
+    SimulationMode mode = 1;
+}
+
+// Scene inspection + basic editing.
+service SceneService {
+    rpc GetSceneInfo(GetSceneInfoRequest) returns (GetSceneInfoResponse);
+    rpc ListEntities(ListEntitiesRequest) returns (ListEntitiesResponse);
+    rpc GetEntity(GetEntityRequest) returns (GetEntityResponse);
+    rpc SetEntityTransform(SetEntityTransformRequest) returns (SetEntityTransformResponse);
+    // Set simulation timing mode (realtime, deterministic, or fast).
+    rpc SetSimulationMode(SetSimulationModeRequest) returns (SetSimulationModeResponse);
+    // Get current simulation timing mode.
+    rpc GetSimulationMode(GetSimulationModeRequest) returns (GetSimulationModeResponse);
+}

luckyrobots/grpc/proto/telemetry.proto

@@ -0,0 +1,43 @@
+syntax = "proto3";
+
+// Telemetry service definitions for the LuckyEngine / Hazel ScriptCore gRPC API (v1).
+
+package hazel.rpc;
+
+option cc_enable_arenas = true;
+
+// Schema describing telemetry vectors (names are informational; sizes are authoritative).
+message TelemetrySchema {
+    repeated string observation_names = 1;
+    repeated string action_names = 2;
+    uint32 nq = 3;
+    uint32 nu = 4;
+}
+
+message GetTelemetrySchemaRequest {}
+
+message GetTelemetrySchemaResponse {
+    TelemetrySchema schema = 1;
+}
+
+// Request a server-stream of telemetry frames.
+// Server may clamp `target_fps` based on configured limits.
+message StreamTelemetryRequest {
+    uint32 target_fps = 1;        // Desired sampling rate (frames per second)
+}
+
+// Telemetry frame: lightweight stream of simulator state (qpos) and last applied control.
+message TelemetryFrame {
+    uint64 timestamp_ms = 1;      // Wall-clock timestamp in milliseconds
+    uint32 frame_number = 2;      // Monotonic frame counter
+    uint32 task_index = 3;        // Reserved for future multi-task setups
+
+    repeated float observation_qpos = 4; // MuJoCo qpos[0..nq)
+    repeated float action_ctrl = 5;      // MuJoCo ctrl[0..nu)
+}
+
+// Stream-only service intended for external tools (plotting, training logs, debugging).
+service TelemetryService {
+    rpc GetTelemetrySchema(GetTelemetrySchemaRequest) returns (GetTelemetrySchemaResponse);
+    rpc StreamTelemetry(StreamTelemetryRequest) returns (stream TelemetryFrame);
+}

luckyrobots/grpc/proto/viewport.proto

@@ -0,0 +1,45 @@
+syntax = "proto3";
+
+// Viewport service definitions for the LuckyEngine / Hazel ScriptCore gRPC API (v1).
+
+package hazel.rpc;
+
+option cc_enable_arenas = true;
+
+import "media.proto";
+
+// Start a viewport stream. Server may clamp FPS and may choose an actual resolution.
+message StartViewportStreamRequest {
+    string viewport_name = 1;    // Which viewport to stream
+    uint32 target_fps = 2;       // Desired frames per second
+    uint32 width = 3;            // Desired width (0 = native)
+    uint32 height = 4;           // Desired height (0 = native)
+    string format = 5;           // "raw", "jpeg" (default: raw)
+}
+
+message ViewportStreamConfig {
+    bool streaming = 1;
+    string viewport_name = 2;
+    uint32 fps = 3;
+    uint32 width = 4;
+    uint32 height = 5;
+}
+
+message StopViewportStreamRequest {}
+
+message StopViewportStreamResponse {
+    bool success = 1;
+}
+
+message GetViewportInfoRequest {}
+
+message GetViewportInfoResponse {
+    repeated string available_viewports = 1;
+    ViewportStreamConfig current_config = 2;
+}
+
+// Streams pixels from a named viewport (useful for external visualization / training input).
+service ViewportService {
+    rpc GetViewportInfo(GetViewportInfoRequest) returns (GetViewportInfoResponse);
+    rpc StreamViewport(StartViewportStreamRequest) returns (stream ImageFrame);
+}

luckyrobots/luckyrobots.py

@@ -0,0 +1,252 @@
+import logging
+import time
+
+from collections.abc import Sequence
+from typing import Optional
+
+from typing import Any
+
+from .engine import launch_luckyengine, stop_luckyengine
+from .models import ObservationResponse
+from .client import LuckyEngineClient, GrpcConnectionError
+from .utils import validate_params, get_robot_config
+
+logger = logging.getLogger("luckyrobots")
+
+
+class LuckyRobots:
+    """
+    gRPC-only control surface for LuckyEngine.
+
+    This is a small convenience wrapper around `LuckyEngineClient` that can:
+    - launch the LuckyEngine executable (optional)
+    - connect to the LuckyEngine gRPC server
+    - send controls and fetch unified observations
+    """
+
+    def __init__(
+        self,
+        host: str = "127.0.0.1",
+        port: int = 50051,
+    ) -> None:
+        self.host = host
+        self.port = port
+
+        self._engine_client: Optional[LuckyEngineClient] = None
+        self._robot_name: Optional[str] = None
+
+        # Cached metadata (filled after connect)
+        self._joint_names: Optional[list[str]] = None
+
+    @staticmethod
+    def get_robot_config(robot: str = None) -> dict:
+        """Return robot config from `luckyrobots/config/robots.yaml`."""
+        return get_robot_config(robot)
+
+    def start(
+        self,
+        scene: str,
+        robot: str,
+        task: str,
+        executable_path: str = None,
+        observation_type: str = "pixels_agent_pos",
+        headless: bool = False,
+        timeout_s: float = 120.0,
+    ) -> None:
+        """
+        Launch LuckyEngine (if needed) and connect to gRPC.
+
+        Args:
+            scene: LuckyEngine scene name.
+            robot: Robot name (must exist in `robots.yaml`).
+            task: Task name (must exist in `robots.yaml`).
+            executable_path: Path to LuckyEngine executable (optional; auto-detected).
+            observation_type: Used for validation and optional camera processing.
+            headless: Launch without rendering.
+            timeout_s: How long to wait for gRPC server to come up.
+        """
+        validate_params(scene, robot, task, observation_type)
+        self._robot_name = robot
+
+        success = launch_luckyengine(
+            scene=scene,
+            robot=robot,
+            task=task,
+            executable_path=executable_path,
+            headless=headless,
+        )
+        if not success:
+            logger.error("Failed to launch LuckyEngine")
+            raise RuntimeError(
+                "Failed to launch LuckyEngine. Look through the log for more information."
+            )
+
+        self.connect(timeout_s=timeout_s, robot=robot)
+
+    def connect(self, timeout_s: float = 120.0, robot: Optional[str] = None) -> None:
+        """Connect to LuckyEngine gRPC server and cache MuJoCo metadata."""
+        if robot is not None:
+            self._robot_name = robot
+        if not self._robot_name:
+            raise ValueError("Robot name is required (pass `robot=` or call start()).")
+
+        self._engine_client = LuckyEngineClient(
+            host=self.host,
+            port=self.port,
+            robot_name=self._robot_name,
+        )
+        logger.info(
+            "Waiting for LuckyEngine gRPC server at %s:%s", self.host, self.port
+        )
+        if not self._engine_client.wait_for_server(timeout=timeout_s):
+            raise GrpcConnectionError(
+                f"LuckyEngine gRPC server connection timeout after {timeout_s} seconds"
+            )
+
+        mujoco_info = self._engine_client.get_mujoco_info(robot_name=self._robot_name)
+        self._joint_names = (
+            list(mujoco_info.joint_names) if mujoco_info.joint_names else []
+        )
+        logger.info(
+            "Connected. MuJoCo: nq=%s nv=%s nu=%s joints=%s",
+            getattr(mujoco_info, "nq", None),
+            getattr(mujoco_info, "nv", None),
+            getattr(mujoco_info, "nu", None),
+            len(self._joint_names),
+        )
+
+    def _require_client(self) -> LuckyEngineClient:
+        if self._engine_client is None or not self._engine_client.is_connected():
+            raise GrpcConnectionError("Not connected. Call start() or connect() first.")
+        return self._engine_client
+
+    def get_observation(self, agent_name: str = "") -> ObservationResponse:
+        """
+        Get observation vector.
+
+        This returns only the flat observation vector defined by the agent's
+        observation spec. For sensor data, use the dedicated methods:
+        - get_joint_state() for joint positions/velocities
+        - engine_client.stream_telemetry() for telemetry
+        - engine_client.stream_camera() for camera frames
+
+        Args:
+            agent_name: Agent name (empty = default agent).
+
+        Returns:
+            ObservationResponse with observation vector, actions, timestamp.
+        """
+        client = self._require_client()
+        return client.get_observation(agent_name=agent_name)
+
+    def get_joint_state(self):
+        """
+        Get joint positions/velocities.
+
+        Returns the raw MuJoCo joint state for the robot.
+        """
+        client = self._require_client()
+        if not self._robot_name:
+            raise ValueError("Robot name is not set.")
+        return client.get_joint_state(robot_name=self._robot_name)
+
+    def send_control(self, controls: Sequence[float]) -> None:
+        """Send control commands to the robot via gRPC."""
+        client = self._require_client()
+        if not self._robot_name:
+            raise ValueError("Robot name is not set.")
+
+        resp = client.send_control(
+            controls=[float(x) for x in controls],
+            robot_name=self._robot_name,
+        )
+        if hasattr(resp, "success") and not resp.success:
+            raise RuntimeError(f"SendControl failed: {getattr(resp, 'message', '')}")
+
+    def step(
+        self,
+        actions: Sequence[float],
+        agent_name: str = "",
+    ) -> ObservationResponse:
+        """
+        Synchronous RL step: apply action, wait for physics, return observation.
+
+        This is the recommended interface for RL training. It uses the gRPC Step RPC
+        which combines SendControl + physics step + GetObservation into a single call,
+        eliminating one network round-trip.
+
+        Args:
+            actions: Action vector to apply for this step.
+            agent_name: Agent name (empty = default agent).
+
+        Returns:
+            ObservationResponse with observation after physics step.
+        """
+        client = self._require_client()
+        return client.step(actions=list(actions), agent_name=agent_name)
+
+    def set_simulation_mode(self, mode: str = "fast"):
+        """
+        Set simulation timing mode.
+
+        Args:
+            mode: "realtime", "deterministic", or "fast"
+                - realtime: Physics runs at 1x wall-clock speed (for visualization)
+                - deterministic: Physics runs at fixed rate (for reproducibility)
+                - fast: Physics runs as fast as possible (for RL training)
+        """
+        client = self._require_client()
+        return client.set_simulation_mode(mode=mode)
+
+    def get_simulation_mode(self):
+        """Get current simulation timing mode."""
+        client = self._require_client()
+        return client.get_simulation_mode()
+
+    def reset(
+        self,
+        agent_name: str = "",
+        randomization_cfg: Optional[Any] = None,
+    ) -> ObservationResponse:
+        """
+        Reset the agent and return a fresh observation.
+
+        Args:
+            agent_name: Agent logical name. Empty string means default agent.
+            randomization_cfg: Optional domain randomization config for this reset.
+                Use this to randomize physics parameters (friction, mass, etc.)
+                at the start of each episode for sim-to-real transfer.
+
+        Returns:
+            ObservationResponse after reset.
+
+        Raises:
+            RuntimeError: If reset fails.
+        """
+        client = self._require_client()
+        resp = client.reset_agent(agent_name=agent_name, randomization_cfg=randomization_cfg)
+        if hasattr(resp, "success") and not resp.success:
+            raise RuntimeError(f"Reset failed: {getattr(resp, 'message', '')}")
+        return self.get_observation(agent_name=agent_name)
+
+    def close(self, stop_engine: bool = True) -> None:
+        """Close gRPC client and optionally stop the engine executable."""
+        if self._engine_client is not None:
+            try:
+                self._engine_client.close()
+            finally:
+                self._engine_client = None
+
+        if stop_engine:
+            stop_luckyengine()
+
+    def __enter__(self) -> "LuckyRobots":
+        return self
+
+    def __exit__(self, exc_type, exc, tb) -> None:
+        self.close(stop_engine=True)
+
+    @property
+    def engine_client(self) -> Optional[LuckyEngineClient]:
+        """Access the underlying LuckyEngine gRPC client for advanced operations."""
+        return self._engine_client

luckyrobots/models/__init__.py

@@ -0,0 +1,15 @@
+"""
+Pydantic models for LuckyRobots.
+"""
+
+from .observation import ObservationResponse, StateSnapshot
+from .camera import CameraData, CameraShape
+from .randomization import DomainRandomizationConfig
+
+__all__ = [
+    "ObservationResponse",
+    "StateSnapshot",
+    "CameraData",
+    "CameraShape",
+    "DomainRandomizationConfig",
+]

luckyrobots/models/camera.py

@@ -0,0 +1,97 @@
+"""
+Camera models for LuckyRobots.
+
+These models handle camera frame data from LuckyEngine.
+"""
+
+from typing import Any, Dict, Optional, Union
+from pydantic import BaseModel, Field, ConfigDict
+import numpy as np
+import cv2
+
+
+class CameraShape(BaseModel):
+    """Shape of camera images."""
+
+    width: float = Field(description="Width of the image")
+    height: float = Field(description="Height of the image")
+    channel: int = Field(description="Number of color channels")
+
+
+class CameraData(BaseModel):
+    """Camera frame data."""
+
+    model_config = ConfigDict(populate_by_name=True, arbitrary_types_allowed=True)
+
+    camera_name: str = Field(alias="CameraName", description="Name of the camera")
+    dtype: str = Field(default="uint8", description="Data type of the image")
+    shape: Optional[Union[CameraShape, Dict[str, Union[float, int]]]] = Field(
+        default=None, description="Shape of the image"
+    )
+    time_stamp: Optional[str] = Field(
+        None, alias="TimeStamp", description="Camera timestamp"
+    )
+    image_data: Optional[Any] = Field(
+        None, alias="ImageData", description="Image data (bytes or numpy array)"
+    )
+    width: Optional[int] = Field(default=None, description="Image width")
+    height: Optional[int] = Field(default=None, description="Image height")
+    channels: Optional[int] = Field(default=None, description="Number of channels")
+    format: Optional[str] = Field(
+        default=None, description="Image format (raw, jpeg, png)"
+    )
+    frame_number: Optional[int] = Field(default=None, description="Frame number")
+
+    def process_image(self) -> None:
+        """Process the image data into a numpy array."""
+        if self.image_data is None:
+            return None
+
+        # If already a numpy array, skip processing
+        if isinstance(self.image_data, np.ndarray):
+            return
+
+        # Handle bytes data
+        if isinstance(self.image_data, bytes):
+            nparr = np.frombuffer(self.image_data, np.uint8)
+
+            # Check if it's raw RGBA/RGB data or encoded
+            if self.format == "raw" and self.width and self.height:
+                channels = self.channels or 4
+                try:
+                    self.image_data = nparr.reshape((self.height, self.width, channels))
+                    # Convert RGBA to BGR for OpenCV compatibility
+                    if channels == 4:
+                        self.image_data = cv2.cvtColor(
+                            self.image_data, cv2.COLOR_RGBA2BGR
+                        )
+                    elif channels == 3:
+                        self.image_data = cv2.cvtColor(
+                            self.image_data, cv2.COLOR_RGB2BGR
+                        )
+                except ValueError:
+                    # Fallback to decode
+                    self.image_data = cv2.imdecode(nparr, cv2.IMREAD_COLOR)
+            else:
+                # Encoded image (JPEG, PNG)
+                self.image_data = cv2.imdecode(nparr, cv2.IMREAD_COLOR)
+
+    @classmethod
+    def from_grpc_frame(cls, frame: Any, camera_name: str = "camera") -> "CameraData":
+        """Create CameraData from a gRPC ImageFrame message."""
+        return cls(
+            camera_name=camera_name,
+            dtype="uint8",
+            width=frame.width,
+            height=frame.height,
+            channels=frame.channels,
+            format=frame.format,
+            time_stamp=str(frame.timestamp_ms),
+            frame_number=frame.frame_number,
+            image_data=frame.data,
+            shape=CameraShape(
+                width=frame.width,
+                height=frame.height,
+                channel=frame.channels,
+            ),
+        )