gs-nyx-plugin 0.1.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- gs_nyx_plugin/__init__.py +13 -0
- gs_nyx_plugin/_version.py +24 -0
- gs_nyx_plugin/nyx_camera_options.py +98 -0
- gs_nyx_plugin/nyx_camera_sensor.py +517 -0
- gs_nyx_plugin/nyx_camera_shared_metadata.py +129 -0
- gs_nyx_plugin/nyx_renderer.py +1115 -0
- gs_nyx_plugin/nyx_scene_exporter.py +809 -0
- gs_nyx_plugin/nyx_scene_utils.py +330 -0
- gs_nyx_plugin-0.1.2.dist-info/METADATA +19 -0
- gs_nyx_plugin-0.1.2.dist-info/RECORD +11 -0
- gs_nyx_plugin-0.1.2.dist-info/WHEEL +4 -0
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
"""Nyx Genesis Plugin - GPU-accelerated rendering for Genesis simulation."""
|
|
2
|
+
|
|
3
|
+
from ._version import __version__
|
|
4
|
+
|
|
5
|
+
try:
|
|
6
|
+
import torch # noqa: F401
|
|
7
|
+
except ImportError as e:
|
|
8
|
+
raise ImportError(
|
|
9
|
+
"'torch' module not available. Please install pytorch manually: "
|
|
10
|
+
"https://pytorch.org/get-started/locally/"
|
|
11
|
+
) from e
|
|
12
|
+
|
|
13
|
+
__all__ = ["__version__"]
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# file generated by vcs-versioning
|
|
2
|
+
# don't change, don't track in version control
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
__all__ = [
|
|
6
|
+
"__version__",
|
|
7
|
+
"__version_tuple__",
|
|
8
|
+
"version",
|
|
9
|
+
"version_tuple",
|
|
10
|
+
"__commit_id__",
|
|
11
|
+
"commit_id",
|
|
12
|
+
]
|
|
13
|
+
|
|
14
|
+
version: str
|
|
15
|
+
__version__: str
|
|
16
|
+
__version_tuple__: tuple[int | str, ...]
|
|
17
|
+
version_tuple: tuple[int | str, ...]
|
|
18
|
+
commit_id: str | None
|
|
19
|
+
__commit_id__: str | None
|
|
20
|
+
|
|
21
|
+
__version__ = version = '0.1.2'
|
|
22
|
+
__version_tuple__ = version_tuple = (0, 1, 2)
|
|
23
|
+
|
|
24
|
+
__commit_id__ = commit_id = None
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Nyx Camera Options for Genesis - GPU-accelerated rendering via Nyx renderer.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
# External imports
|
|
6
|
+
from pydantic import ConfigDict, Field
|
|
7
|
+
from typing import TYPE_CHECKING, Literal, Annotated
|
|
8
|
+
|
|
9
|
+
# Internal imports
|
|
10
|
+
import genesis as gs
|
|
11
|
+
from genesis.options.sensors.camera import BaseCameraOptions
|
|
12
|
+
from genesis.typing import PositiveInt, PositiveFloat, PositiveVec2IType
|
|
13
|
+
from gs_nyx import nyx_py_sdk as nps
|
|
14
|
+
from gs_nyx import nyx_py_renderer as npr
|
|
15
|
+
|
|
16
|
+
if TYPE_CHECKING:
|
|
17
|
+
pass
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
__all__ = ["NyxCameraOptions"]
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
# Register init and destroy functions for the Nyx sdk plugin.
|
|
24
|
+
gs.register_external_module(nps.startup, nps.shutdown)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
# ========================== Nyx Camera Options ==========================
|
|
28
|
+
class NyxCameraOptions(BaseCameraOptions["NyxCameraSensor"]):
|
|
29
|
+
"""
|
|
30
|
+
Options for the Nyx Camera Sensor plugin.
|
|
31
|
+
|
|
32
|
+
Inherits from BaseCameraOptions which provides:
|
|
33
|
+
- res, pos, lookat, up, fov, lights, offset_T (from BaseCameraOptions)
|
|
34
|
+
- entity_idx, link_idx_local, pos_offset, euler_offset (from RigidSensorOptionsMixin)
|
|
35
|
+
|
|
36
|
+
Nyx-specific parameters are defined below.
|
|
37
|
+
|
|
38
|
+
Parameters
|
|
39
|
+
----------
|
|
40
|
+
model : Literal
|
|
41
|
+
Camera model: "pinhole". Default is "pinhole".
|
|
42
|
+
spp : PositiveInt
|
|
43
|
+
Samples per pixel for path tracing. Default is 16.
|
|
44
|
+
denoise : bool
|
|
45
|
+
Whether to apply denoising. Default is True.
|
|
46
|
+
near : PositiveFloat
|
|
47
|
+
Near clipping plane distance. Default is 0.1.
|
|
48
|
+
far : PositiveFloat
|
|
49
|
+
Far clipping plane distance. Default is 100.0.
|
|
50
|
+
render_mode : ~gs_nyx.nyx_py_renderer.ERenderMode
|
|
51
|
+
Rendering mode for Nyx. Default is
|
|
52
|
+
:attr:`~gs_nyx.nyx_py_renderer.ERenderMode.FastPathTracer`.
|
|
53
|
+
debug_view : ~gs_nyx.nyx_py_renderer.EDebugView
|
|
54
|
+
Debug view mode for Nyx. Default is
|
|
55
|
+
:attr:`~gs_nyx.nyx_py_renderer.EDebugView.Meshlet`.
|
|
56
|
+
tone_mapper : ~gs_nyx.nyx_py_sdk.EToneMapper
|
|
57
|
+
Tone mapper to apply post-render. Default is
|
|
58
|
+
:attr:`~gs_nyx.nyx_py_sdk.EToneMapper.Reinhard`.
|
|
59
|
+
anti_aliasing : ~gs_nyx.nyx_py_sdk.EAntiAliasing
|
|
60
|
+
Anti-aliasing technique. Default is
|
|
61
|
+
:attr:`~gs_nyx.nyx_py_sdk.EAntiAliasing.SMAA`.
|
|
62
|
+
open_window : bool
|
|
63
|
+
Whether to open a GUI window. Default is False.
|
|
64
|
+
window_size : PositiveVec2IType
|
|
65
|
+
Window size if open_window is enabled. Default is (1024, 1024).
|
|
66
|
+
"""
|
|
67
|
+
|
|
68
|
+
model_config = ConfigDict(arbitrary_types_allowed=True)
|
|
69
|
+
|
|
70
|
+
# Nyx-specific rendering parameters
|
|
71
|
+
model: Literal["pinhole"] = "pinhole"
|
|
72
|
+
spp: PositiveInt = 16
|
|
73
|
+
denoise: bool = True
|
|
74
|
+
near: PositiveFloat = 0.1
|
|
75
|
+
far: PositiveFloat = 100.0
|
|
76
|
+
render_mode: npr.ERenderMode = npr.ERenderMode.FastPathTracer
|
|
77
|
+
debug_view: npr.EDebugView = npr.EDebugView.Meshlet
|
|
78
|
+
tone_mapper: nps.EToneMapper = nps.EToneMapper.Reinhard
|
|
79
|
+
anti_aliasing: nps.EAntiAliasing = nps.EAntiAliasing.SMAA
|
|
80
|
+
|
|
81
|
+
# Window/export settings
|
|
82
|
+
open_window: bool = False
|
|
83
|
+
window_size: PositiveVec2IType = (1920, 1080)
|
|
84
|
+
|
|
85
|
+
# Sensor manager integration
|
|
86
|
+
update_ground_truth_only: bool = True
|
|
87
|
+
|
|
88
|
+
# Nyx additional params
|
|
89
|
+
env_maps: Annotated[tuple[nps.EnvironmentMapAsset, ...], Field(strict=False)] = ()
|
|
90
|
+
light_fields: Annotated[tuple[nps.LightFieldAsset, ...], Field(strict=False)] = ()
|
|
91
|
+
|
|
92
|
+
# Post-init validation.
|
|
93
|
+
def model_post_init(self, _):
|
|
94
|
+
super().model_post_init(_)
|
|
95
|
+
|
|
96
|
+
# We require far to be greater than near for valid depth range.
|
|
97
|
+
if self.far <= self.near:
|
|
98
|
+
gs.raise_exception("far must be greater than near")
|
|
@@ -0,0 +1,517 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Nyx Camera Sensor for Genesis - GPU-accelerated rendering via Nyx renderer.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
# External imports
|
|
6
|
+
import ctypes
|
|
7
|
+
import os
|
|
8
|
+
from importlib.metadata import PackageNotFoundError, distribution
|
|
9
|
+
from typing import TYPE_CHECKING, Any, Dict, List, NamedTuple, Optional
|
|
10
|
+
import torch
|
|
11
|
+
|
|
12
|
+
# Internal imports
|
|
13
|
+
import genesis as gs
|
|
14
|
+
|
|
15
|
+
# Required for the pick result
|
|
16
|
+
if TYPE_CHECKING:
|
|
17
|
+
from genesis.engine.entities.base_entity import Entity
|
|
18
|
+
from genesis.engine.sensors.camera import BaseCameraSensor
|
|
19
|
+
from genesis.engine.sensors.base_sensor import (
|
|
20
|
+
Sensor,
|
|
21
|
+
)
|
|
22
|
+
from gs_nyx import nyx_py_sdk as nps
|
|
23
|
+
|
|
24
|
+
# Relative imports
|
|
25
|
+
from .nyx_renderer import NyxPyRenderer
|
|
26
|
+
from .nyx_scene_exporter import NyxSceneExporter
|
|
27
|
+
from .nyx_camera_options import NyxCameraOptions
|
|
28
|
+
from .nyx_camera_shared_metadata import NyxCameraSharedMetadata, _nyx_make_complete_path
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
__all__ = ["NyxCameraSensor", "NyxCameraData", "NyxPickPixelResult"]
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
# ========================== Data Class ==========================
|
|
35
|
+
class NyxCameraData(NamedTuple):
|
|
36
|
+
"""Return type of :meth:`NyxCameraSensor.read`.
|
|
37
|
+
|
|
38
|
+
Attributes
|
|
39
|
+
----------
|
|
40
|
+
rgb : torch.Tensor
|
|
41
|
+
Rendered RGB image, shape ``(B, H, W, 3)`` with ``dtype=torch.uint8``,
|
|
42
|
+
resident on the CUDA device. ``B`` is the number of parallel Genesis
|
|
43
|
+
environments (``1`` for non-batched scenes); ``H`` and ``W`` match
|
|
44
|
+
the camera's configured :attr:`~NyxCameraOptions.res`. Channel order
|
|
45
|
+
is RGB, values in ``[0, 255]``.
|
|
46
|
+
"""
|
|
47
|
+
|
|
48
|
+
rgb: torch.Tensor
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
# ========================== Nyx Pick Pixel Result ==========================
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class NyxPickPixelResult(NamedTuple):
|
|
55
|
+
"""Hit information returned by :meth:`NyxCameraSensor.pick_pixel`.
|
|
56
|
+
|
|
57
|
+
Attributes
|
|
58
|
+
----------
|
|
59
|
+
entity : genesis.engine.entities.base_entity.Entity
|
|
60
|
+
The Genesis entity the ray hit.
|
|
61
|
+
link_name : str
|
|
62
|
+
Name of the hit link or sub-geometry. For URDF entities this is the
|
|
63
|
+
link name; for MJCF entities it is ``"<link>_<vgeom_idx>"``;
|
|
64
|
+
empty string for all other morph types.
|
|
65
|
+
position : tuple of float
|
|
66
|
+
World-space hit point ``(x, y, z)`` in Genesis Z-up coordinates.
|
|
67
|
+
"""
|
|
68
|
+
|
|
69
|
+
entity: "Entity"
|
|
70
|
+
link_name: str
|
|
71
|
+
position: tuple[float, float, float]
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
# ========================== Nyx Camera Sensor ==========================
|
|
75
|
+
class NyxCameraSensor(
|
|
76
|
+
BaseCameraSensor, Sensor[NyxCameraOptions, NyxCameraSharedMetadata, NyxCameraData]
|
|
77
|
+
):
|
|
78
|
+
"""GPU-accelerated camera sensor backed by the Nyx renderer.
|
|
79
|
+
|
|
80
|
+
Registers with the Genesis sensor manager, contributes one camera (plus
|
|
81
|
+
any configured lights / env maps / light fields) to a renderer instance
|
|
82
|
+
shared across all :class:`NyxCameraSensor` sensors in the scene, and
|
|
83
|
+
produces RGB images on demand via the standard Genesis
|
|
84
|
+
:meth:`~genesis.engine.sensors.base_sensor.Sensor.read` API.
|
|
85
|
+
|
|
86
|
+
Configure with :class:`NyxCameraOptions`. The first ``NyxCameraSensor``
|
|
87
|
+
to finish building drives the actual renderer setup (scene export +
|
|
88
|
+
native renderer startup); subsequent sensors slot into the same shared
|
|
89
|
+
instance.
|
|
90
|
+
|
|
91
|
+
See the user guide for a configured example; this class is rarely
|
|
92
|
+
instantiated directly — pass :class:`NyxCameraOptions` to
|
|
93
|
+
:meth:`Scene.add_sensor` instead.
|
|
94
|
+
"""
|
|
95
|
+
|
|
96
|
+
def __init__(
|
|
97
|
+
self,
|
|
98
|
+
options: NyxCameraOptions,
|
|
99
|
+
idx: int,
|
|
100
|
+
manager: "gs.SensorManager",
|
|
101
|
+
):
|
|
102
|
+
"""Bind the sensor to a Genesis sensor manager.
|
|
103
|
+
|
|
104
|
+
Called by :meth:`Scene.add_sensor` — users should not invoke this
|
|
105
|
+
directly. The native renderer is **not** started here; setup runs
|
|
106
|
+
inside :meth:`build` once the Genesis scene is built and every
|
|
107
|
+
``NyxCameraSensor`` in the scene has been added.
|
|
108
|
+
|
|
109
|
+
Parameters
|
|
110
|
+
----------
|
|
111
|
+
options : NyxCameraOptions
|
|
112
|
+
Per-camera configuration (resolution, FOV, pose, render mode,
|
|
113
|
+
lights / env maps, etc.).
|
|
114
|
+
idx : int
|
|
115
|
+
Genesis-side sensor index assigned by the manager.
|
|
116
|
+
manager : genesis.SensorManager
|
|
117
|
+
The sensor manager that owns this sensor.
|
|
118
|
+
"""
|
|
119
|
+
super().__init__(options, idx, manager)
|
|
120
|
+
self._options: NyxCameraOptions
|
|
121
|
+
self._camera_idx: Optional[int] = None
|
|
122
|
+
|
|
123
|
+
# Per-env camera poses from move_to_attach(), shape (B, 3) each, or None if not attached
|
|
124
|
+
self._attached_pos: Optional[torch.Tensor] = None
|
|
125
|
+
self._attached_lookat: Optional[torch.Tensor] = None
|
|
126
|
+
self._attached_up: Optional[torch.Tensor] = None
|
|
127
|
+
|
|
128
|
+
# ========================== Light Conversion ==========================
|
|
129
|
+
|
|
130
|
+
def _convert_light_dict_to_asset(self, light_dict: Dict[str, Any]) -> Any:
|
|
131
|
+
"""
|
|
132
|
+
Convert a light configuration dict to an Nyx LightAsset. Handles coordinate conversion from Genesis Z-up to Nyx Y-up.
|
|
133
|
+
|
|
134
|
+
Parameters
|
|
135
|
+
----------
|
|
136
|
+
light_dict : dict
|
|
137
|
+
Light configuration with keys:
|
|
138
|
+
- type: "point", "directional", or "spot"
|
|
139
|
+
- pos: (x, y, z) position in Z-up coordinates (for point/spot)
|
|
140
|
+
- dir: (x, y, z) direction in Z-up coordinates (for directional/spot)
|
|
141
|
+
- color: (r, g, b) RGB color [0-1]
|
|
142
|
+
- intensity: Light intensity
|
|
143
|
+
- shadow: Whether to cast shadows (default True)
|
|
144
|
+
- inner_angle, outer_angle: For spot lights (degrees)
|
|
145
|
+
|
|
146
|
+
Returns
|
|
147
|
+
-------
|
|
148
|
+
LightAsset
|
|
149
|
+
Nyx LightAsset object configured with Y-up coordinates.
|
|
150
|
+
"""
|
|
151
|
+
light = nps.LightAsset()
|
|
152
|
+
light_type = light_dict.get("type", "directional")
|
|
153
|
+
|
|
154
|
+
if light_type == "point":
|
|
155
|
+
light.type = nps.ELightType.Point
|
|
156
|
+
# Get the position and convert it.
|
|
157
|
+
light.point_position = nps.float3(*light_dict.get("pos", (0, 0, 5)))
|
|
158
|
+
light.point_position = nps.float3_z_up_to_y_up_a(light.point_position)
|
|
159
|
+
light.point_range = light_dict.get("range", 100.0)
|
|
160
|
+
light.unit = nps.ELightUnit.Lumen
|
|
161
|
+
|
|
162
|
+
elif light_type == "directional":
|
|
163
|
+
light.type = nps.ELightType.Directional
|
|
164
|
+
# Get the direction and convert it.
|
|
165
|
+
light.directional_direction = nps.float3(*light_dict.get("dir", (0, 0, -1)))
|
|
166
|
+
light.directional_direction = nps.float3_z_up_to_y_up_a(
|
|
167
|
+
light.directional_direction
|
|
168
|
+
)
|
|
169
|
+
light.unit = nps.ELightUnit.Lux
|
|
170
|
+
|
|
171
|
+
elif light_type == "spot":
|
|
172
|
+
light.type = nps.ELightType.Spot
|
|
173
|
+
# Get the position and convert it.
|
|
174
|
+
light.spot_position = nps.float3(*light_dict.get("pos", (0, 0, 5)))
|
|
175
|
+
light.spot_position = nps.float3_z_up_to_y_up_a(light.spot_position)
|
|
176
|
+
# Get the direction and convert it.
|
|
177
|
+
light.spot_direction = nps.float3(*light_dict.get("dir", (0, 0, -1)))
|
|
178
|
+
light.spot_direction = nps.float3_z_up_to_y_up_a(light.spot_direction)
|
|
179
|
+
light.spot_innerAngle = light_dict.get("inner_angle", 15.0)
|
|
180
|
+
light.spot_outerAngle = light_dict.get("outer_angle", 30.0)
|
|
181
|
+
light.spot_range = light_dict.get("range", 100.0)
|
|
182
|
+
light.unit = nps.ELightUnit.Lumen
|
|
183
|
+
|
|
184
|
+
else:
|
|
185
|
+
gs.raise_exception(f"Unknown light type: {light_type}")
|
|
186
|
+
|
|
187
|
+
# Common light properties
|
|
188
|
+
color = light_dict.get("color", (1, 1, 1))
|
|
189
|
+
light.color = nps.float3(color[0], color[1], color[2])
|
|
190
|
+
light.intensity = light_dict.get("intensity", 1.0)
|
|
191
|
+
light.shadow = light_dict.get("shadow", True)
|
|
192
|
+
|
|
193
|
+
return light
|
|
194
|
+
|
|
195
|
+
def _apply_camera_transform(self, camera_T: torch.Tensor):
|
|
196
|
+
"""
|
|
197
|
+
Store per-env camera transforms from the attachment system.
|
|
198
|
+
|
|
199
|
+
Called by BaseCameraSensor.move_to_attach() with the computed world transform.
|
|
200
|
+
The stored values are sent to the renderer in _render_current_state().
|
|
201
|
+
|
|
202
|
+
Parameters
|
|
203
|
+
----------
|
|
204
|
+
camera_T : torch.Tensor
|
|
205
|
+
Transform matrix, shape (B, 4, 4) for batched envs or (4, 4) for single.
|
|
206
|
+
"""
|
|
207
|
+
if len(camera_T.shape) == 2:
|
|
208
|
+
camera_T = camera_T.unsqueeze(0)
|
|
209
|
+
|
|
210
|
+
# Store per-env pos / lookat / up, all (B, 3)
|
|
211
|
+
self._attached_pos = camera_T[:, :3, 3]
|
|
212
|
+
self._attached_lookat = self._attached_pos + (-camera_T[:, :3, 2])
|
|
213
|
+
self._attached_up = camera_T[:, :3, 1]
|
|
214
|
+
|
|
215
|
+
self._stale = True
|
|
216
|
+
|
|
217
|
+
def build(self):
|
|
218
|
+
"""Finalize the sensor and lazily start the shared Nyx renderer.
|
|
219
|
+
|
|
220
|
+
Called by the Genesis sensor manager once per sensor after the scene
|
|
221
|
+
has been built. Each call:
|
|
222
|
+
|
|
223
|
+
1. Resolves the optional rig attachment via :class:`RigidSensorMixin`.
|
|
224
|
+
2. Verifies CUDA is available (Nyx is GPU-only).
|
|
225
|
+
3. Registers this camera in the shared metadata, assigns it a Nyx
|
|
226
|
+
camera index, and captures its configuration as a definition dict.
|
|
227
|
+
4. If this is the final ``NyxCameraSensor`` left to build, collects
|
|
228
|
+
lights / env maps / light fields from **all** sibling sensors,
|
|
229
|
+
writes the scene-description JSON via :class:`NyxSceneExporter`,
|
|
230
|
+
and constructs the shared :class:`NyxPyRenderer`.
|
|
231
|
+
5. Pre-allocates this sensor's slot in the shared image cache:
|
|
232
|
+
``(B, H, W, 3)`` ``uint8`` CUDA tensor.
|
|
233
|
+
|
|
234
|
+
Raises
|
|
235
|
+
------
|
|
236
|
+
genesis.GenesisException
|
|
237
|
+
If CUDA is unavailable, or if sibling
|
|
238
|
+
:class:`NyxCameraSensor` instances disagree on ``render_mode``
|
|
239
|
+
(the shared renderer can serve only one mode).
|
|
240
|
+
"""
|
|
241
|
+
super().build() # Sets up _link from RigidSensorMixin
|
|
242
|
+
|
|
243
|
+
if not torch.cuda.is_available():
|
|
244
|
+
gs.raise_exception("NyxCameraSensor requires CUDA to be available.")
|
|
245
|
+
|
|
246
|
+
scene = self._manager._sim.scene
|
|
247
|
+
|
|
248
|
+
if self._shared_metadata.sensors is None:
|
|
249
|
+
self._shared_metadata.sensors = []
|
|
250
|
+
self._shared_metadata.camera_defs = []
|
|
251
|
+
self._shared_metadata.image_cache = {}
|
|
252
|
+
|
|
253
|
+
# Register this sensor and assign a camera index
|
|
254
|
+
self._camera_idx = len(self._shared_metadata.sensors)
|
|
255
|
+
self._shared_metadata.sensors.append(self)
|
|
256
|
+
|
|
257
|
+
# Store camera configuration as a dict
|
|
258
|
+
camera_def = {
|
|
259
|
+
"pos": self._options.pos,
|
|
260
|
+
"lookat": self._options.lookat,
|
|
261
|
+
"up": self._options.up,
|
|
262
|
+
"fov": self._options.fov,
|
|
263
|
+
"res": self._options.res,
|
|
264
|
+
"spp": self._options.spp,
|
|
265
|
+
"denoise": self._options.denoise,
|
|
266
|
+
"aperture": getattr(self._options, "aperture", 2.8),
|
|
267
|
+
"focal_len": getattr(self._options, "focal_length", 10.0),
|
|
268
|
+
"near": self._options.near,
|
|
269
|
+
"far": self._options.far,
|
|
270
|
+
"tone_mapper": self._options.tone_mapper,
|
|
271
|
+
"anti_aliasing": self._options.anti_aliasing,
|
|
272
|
+
}
|
|
273
|
+
self._shared_metadata.camera_defs.append(camera_def)
|
|
274
|
+
|
|
275
|
+
# Initialize renderer once all cameras are registered
|
|
276
|
+
all_nyx_sensors = self._manager._sensors_by_type[type(self)]
|
|
277
|
+
all_sensors_built = len(self._shared_metadata.sensors) == len(all_nyx_sensors)
|
|
278
|
+
|
|
279
|
+
if self._shared_metadata.renderer is None and all_sensors_built:
|
|
280
|
+
# Preload Nvidia compiler runtime if available (i.e. torch is not
|
|
281
|
+
# built from source). Must happen after the CUDA check and before
|
|
282
|
+
# the renderer is created, loading it at package-import time via
|
|
283
|
+
# RTLD_GLOBAL interferes with torch's CUDA detection.
|
|
284
|
+
try:
|
|
285
|
+
nvrtc_dist = distribution("nvidia_cuda_nvrtc")
|
|
286
|
+
for file in nvrtc_dist.files:
|
|
287
|
+
if file.name.startswith("libnvrtc-builtins.so.1"):
|
|
288
|
+
ctypes.CDLL(nvrtc_dist.locate_file(file), ctypes.RTLD_GLOBAL)
|
|
289
|
+
break
|
|
290
|
+
except (PackageNotFoundError, OSError):
|
|
291
|
+
pass
|
|
292
|
+
|
|
293
|
+
# Collect lights from all sensors and convert to LightAssets
|
|
294
|
+
lights = []
|
|
295
|
+
for sensor in self._shared_metadata.sensors:
|
|
296
|
+
for light_dict in sensor._options.lights:
|
|
297
|
+
lights.append(sensor._convert_light_dict_to_asset(light_dict))
|
|
298
|
+
|
|
299
|
+
# Collect env_maps from all sensors
|
|
300
|
+
env_maps = []
|
|
301
|
+
for sensor in self._shared_metadata.sensors:
|
|
302
|
+
for env_map in sensor._options.env_maps:
|
|
303
|
+
env_maps.append(env_map)
|
|
304
|
+
|
|
305
|
+
# Collect light_fields from all sensors
|
|
306
|
+
light_fields = []
|
|
307
|
+
for sensor in self._shared_metadata.sensors:
|
|
308
|
+
for light_field in sensor._options.light_fields:
|
|
309
|
+
light_fields.append(light_field)
|
|
310
|
+
|
|
311
|
+
# When there is no lights or env maps, we notify the user.
|
|
312
|
+
if len(lights) == 0 and len(env_maps) == 0:
|
|
313
|
+
print(
|
|
314
|
+
"\033[93m [WARNING] NyxCameraSensor: The current scene has no lights or environment maps. "
|
|
315
|
+
"If this is unintentional, please add a light source or an environment map.\033[0m"
|
|
316
|
+
)
|
|
317
|
+
|
|
318
|
+
# Generate the complete export path.
|
|
319
|
+
complete_path = _nyx_make_complete_path(
|
|
320
|
+
self._shared_metadata.scene_description_export_path
|
|
321
|
+
)
|
|
322
|
+
|
|
323
|
+
# Build the scene description
|
|
324
|
+
self._shared_metadata.scene_exporter = NyxSceneExporter(
|
|
325
|
+
scene,
|
|
326
|
+
cameras=self._shared_metadata.camera_defs,
|
|
327
|
+
export_folder=complete_path,
|
|
328
|
+
lights=lights,
|
|
329
|
+
env_maps=env_maps,
|
|
330
|
+
light_fields=light_fields,
|
|
331
|
+
)
|
|
332
|
+
|
|
333
|
+
# Export the scene to a json file that the Nyx renderer can read.
|
|
334
|
+
self._shared_metadata.scene_exporter.export_to_file(
|
|
335
|
+
os.path.join(
|
|
336
|
+
complete_path,
|
|
337
|
+
"nyx_scene.json",
|
|
338
|
+
)
|
|
339
|
+
)
|
|
340
|
+
|
|
341
|
+
# Get max resolution across all cameras
|
|
342
|
+
max_width, max_height = self._get_max_resolution(
|
|
343
|
+
[s._options for s in all_nyx_sensors]
|
|
344
|
+
)
|
|
345
|
+
|
|
346
|
+
# Verify all sensors use the same render_mode
|
|
347
|
+
render_mode = self._shared_metadata.sensors[0]._options.render_mode
|
|
348
|
+
for sensor in self._shared_metadata.sensors[1:]:
|
|
349
|
+
if sensor._options.render_mode != render_mode:
|
|
350
|
+
gs.raise_exception(
|
|
351
|
+
f"All Nyx cameras must use the same render_mode. "
|
|
352
|
+
f"Found '{render_mode}' and '{sensor._options.render_mode}'. "
|
|
353
|
+
f"Please ensure all NyxCameraOptions use the same render_mode."
|
|
354
|
+
)
|
|
355
|
+
|
|
356
|
+
# Create the Nyx renderer instance.
|
|
357
|
+
n_envs = max(self._manager._sim._B, 1)
|
|
358
|
+
debug_view = self._shared_metadata.sensors[0]._options.debug_view
|
|
359
|
+
self._shared_metadata.renderer = NyxPyRenderer(
|
|
360
|
+
scene,
|
|
361
|
+
os.path.join(
|
|
362
|
+
complete_path,
|
|
363
|
+
"nyx_scene.json",
|
|
364
|
+
),
|
|
365
|
+
max_width,
|
|
366
|
+
max_height,
|
|
367
|
+
n_envs=n_envs,
|
|
368
|
+
open_window=self._options.open_window,
|
|
369
|
+
camera_defs=self._shared_metadata.camera_defs,
|
|
370
|
+
render_mode=render_mode,
|
|
371
|
+
debug_view=debug_view,
|
|
372
|
+
)
|
|
373
|
+
self._shared_metadata.renderer.build(
|
|
374
|
+
self._shared_metadata.scene_exporter._entity_uuid_pairs
|
|
375
|
+
)
|
|
376
|
+
|
|
377
|
+
# Initialize image cache for this camera (always on CUDA for Nyx renderer)
|
|
378
|
+
n_envs = max(self._manager._sim._B, 1)
|
|
379
|
+
h, w = self._options.res[1], self._options.res[0]
|
|
380
|
+
self._shared_metadata.image_cache[self._idx] = torch.zeros(
|
|
381
|
+
(n_envs, h, w, 3), dtype=torch.uint8, device="cuda"
|
|
382
|
+
)
|
|
383
|
+
|
|
384
|
+
def pick_pixel(
|
|
385
|
+
self, camera_index: int, x: int, y: int
|
|
386
|
+
) -> Optional[NyxPickPixelResult]:
|
|
387
|
+
"""
|
|
388
|
+
Cast a ray through pixel (x, y) of the given camera and return what it hits.
|
|
389
|
+
|
|
390
|
+
Parameters
|
|
391
|
+
----------
|
|
392
|
+
camera_index : int
|
|
393
|
+
Index of the camera within this sensor's shared metadata. For a single
|
|
394
|
+
NyxCameraSensor this is 0; multi-camera setups index in registration order.
|
|
395
|
+
x, y : int
|
|
396
|
+
Pixel coordinates in image space, with the origin at the top-left of the
|
|
397
|
+
framebuffer. Must lie within the camera's configured resolution.
|
|
398
|
+
|
|
399
|
+
Returns
|
|
400
|
+
-------
|
|
401
|
+
NyxPickPixelResult or None
|
|
402
|
+
``None`` if the ray missed all scene geometry (e.g. background / sky).
|
|
403
|
+
Otherwise a NyxPickPixelResult with fields:
|
|
404
|
+
|
|
405
|
+
- ``entity``: the Genesis Entity that was hit
|
|
406
|
+
- ``link_name``: name of the hit link (URDF) or ``<link>_<vgeom_idx>`` (MJCF);
|
|
407
|
+
empty string for other morph types
|
|
408
|
+
- ``position``: world-space hit point as ``(x, y, z)`` in Genesis Z-up coordinates
|
|
409
|
+
"""
|
|
410
|
+
# Pick a pixel using the Nyx renderer's picking functionality.
|
|
411
|
+
res = self._shared_metadata.renderer.pick_pixel(
|
|
412
|
+
self._shared_metadata.scene_exporter._entity_uuid_pairs, camera_index, x, y
|
|
413
|
+
)
|
|
414
|
+
|
|
415
|
+
# If res is None, it means no entity was picked at that pixel.
|
|
416
|
+
if res is None:
|
|
417
|
+
return None
|
|
418
|
+
|
|
419
|
+
# If not, convert the picked result to Genesis entities and return as NyxPickPixelResult.
|
|
420
|
+
return NyxPickPixelResult(*res)
|
|
421
|
+
|
|
422
|
+
def _get_image_cache_entry(self):
|
|
423
|
+
"""Return this sensor's entry in the shared image cache."""
|
|
424
|
+
return self._shared_metadata.image_cache[self._idx]
|
|
425
|
+
|
|
426
|
+
def _render_current_state(self):
|
|
427
|
+
"""Perform the actual render for all cameras."""
|
|
428
|
+
# Shorthand
|
|
429
|
+
sensors = self._shared_metadata.sensors or [self]
|
|
430
|
+
renderer = self._shared_metadata.renderer
|
|
431
|
+
scene = self._manager._sim.scene
|
|
432
|
+
|
|
433
|
+
# Update visualization transforms
|
|
434
|
+
scene.visualizer.update_visual_states()
|
|
435
|
+
|
|
436
|
+
# Compute per-env attachment transforms for ALL attached sensors
|
|
437
|
+
for sensor in sensors:
|
|
438
|
+
if sensor._link is not None:
|
|
439
|
+
sensor.move_to_attach()
|
|
440
|
+
|
|
441
|
+
# Update window events, delta time, etc.
|
|
442
|
+
renderer.update()
|
|
443
|
+
|
|
444
|
+
# Nyx Renderer can only render a single env at a time
|
|
445
|
+
# So we loop over all envs and render all the sensors in this env
|
|
446
|
+
for env_idx in range(max(self._manager._sim._B, 1)):
|
|
447
|
+
# Update camera tensors for this env
|
|
448
|
+
for sensor in sensors:
|
|
449
|
+
if sensor._attached_pos is not None:
|
|
450
|
+
# Attached camera: use per-env transform from move_to_attach
|
|
451
|
+
renderer.update_camera_tensor(
|
|
452
|
+
sensor._camera_idx,
|
|
453
|
+
sensor._attached_pos[env_idx],
|
|
454
|
+
sensor._attached_lookat[env_idx],
|
|
455
|
+
sensor._attached_up[env_idx],
|
|
456
|
+
)
|
|
457
|
+
else:
|
|
458
|
+
# Static camera: same pose for all envs
|
|
459
|
+
renderer.update_camera_tensor(
|
|
460
|
+
sensor._camera_idx,
|
|
461
|
+
sensor._options.pos,
|
|
462
|
+
sensor._options.lookat,
|
|
463
|
+
sensor._options.up,
|
|
464
|
+
)
|
|
465
|
+
|
|
466
|
+
# Sync geometry data from Genesis to Nyx for this env
|
|
467
|
+
renderer.update_scene(env_idx)
|
|
468
|
+
|
|
469
|
+
# Render all cameras
|
|
470
|
+
for sensor in sensors:
|
|
471
|
+
rgb_tensor = renderer.render(camera_index=sensor._camera_idx)
|
|
472
|
+
sensor._shared_metadata.image_cache[sensor._idx][env_idx] = rgb_tensor
|
|
473
|
+
|
|
474
|
+
# Mark all sensors as fresh
|
|
475
|
+
for sensor in sensors:
|
|
476
|
+
sensor._stale = False
|
|
477
|
+
|
|
478
|
+
self._shared_metadata.last_render_timestep = self._manager._sim.scene.t
|
|
479
|
+
|
|
480
|
+
def update_camera_pose(self, pos=None, lookat=None, up=None):
|
|
481
|
+
"""Update camera pose dynamically (applies to all envs on next render).
|
|
482
|
+
|
|
483
|
+
Detaches the camera from any rig so the specified world-space pose
|
|
484
|
+
is used uniformly across all environments.
|
|
485
|
+
|
|
486
|
+
Parameters
|
|
487
|
+
----------
|
|
488
|
+
pos : tuple, optional
|
|
489
|
+
New camera position (x, y, z) in Z-up coordinates.
|
|
490
|
+
lookat : tuple, optional
|
|
491
|
+
New look-at point (x, y, z) in Z-up coordinates.
|
|
492
|
+
up : tuple, optional
|
|
493
|
+
New up vector (x, y, z) in Z-up coordinates.
|
|
494
|
+
"""
|
|
495
|
+
if pos is not None:
|
|
496
|
+
self._options.pos = tuple(pos)
|
|
497
|
+
if lookat is not None:
|
|
498
|
+
self._options.lookat = tuple(lookat)
|
|
499
|
+
if up is not None:
|
|
500
|
+
self._options.up = tuple(up)
|
|
501
|
+
|
|
502
|
+
# Detach from rig
|
|
503
|
+
self._link = None
|
|
504
|
+
self._attached_pos = None
|
|
505
|
+
self._attached_lookat = None
|
|
506
|
+
self._attached_up = None
|
|
507
|
+
|
|
508
|
+
self._stale = True
|
|
509
|
+
|
|
510
|
+
@staticmethod
|
|
511
|
+
def _get_max_resolution(options_list: List[NyxCameraOptions]) -> tuple[int, int]:
|
|
512
|
+
"""Get the maximum resolution across all camera options."""
|
|
513
|
+
if not options_list:
|
|
514
|
+
return 1024, 1024
|
|
515
|
+
max_width = max(opt.res[0] for opt in options_list)
|
|
516
|
+
max_height = max(opt.res[1] for opt in options_list)
|
|
517
|
+
return max_width, max_height
|