gs-nyx-plugin 0.1.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,13 @@
1
+ """Nyx Genesis Plugin - GPU-accelerated rendering for Genesis simulation."""
2
+
3
+ from ._version import __version__
4
+
5
+ try:
6
+ import torch # noqa: F401
7
+ except ImportError as e:
8
+ raise ImportError(
9
+ "'torch' module not available. Please install pytorch manually: "
10
+ "https://pytorch.org/get-started/locally/"
11
+ ) from e
12
+
13
+ __all__ = ["__version__"]
@@ -0,0 +1,24 @@
1
+ # file generated by vcs-versioning
2
+ # don't change, don't track in version control
3
+ from __future__ import annotations
4
+
5
+ __all__ = [
6
+ "__version__",
7
+ "__version_tuple__",
8
+ "version",
9
+ "version_tuple",
10
+ "__commit_id__",
11
+ "commit_id",
12
+ ]
13
+
14
+ version: str
15
+ __version__: str
16
+ __version_tuple__: tuple[int | str, ...]
17
+ version_tuple: tuple[int | str, ...]
18
+ commit_id: str | None
19
+ __commit_id__: str | None
20
+
21
+ __version__ = version = '0.1.2'
22
+ __version_tuple__ = version_tuple = (0, 1, 2)
23
+
24
+ __commit_id__ = commit_id = None
@@ -0,0 +1,98 @@
1
+ """
2
+ Nyx Camera Options for Genesis - GPU-accelerated rendering via Nyx renderer.
3
+ """
4
+
5
+ # External imports
6
+ from pydantic import ConfigDict, Field
7
+ from typing import TYPE_CHECKING, Literal, Annotated
8
+
9
+ # Internal imports
10
+ import genesis as gs
11
+ from genesis.options.sensors.camera import BaseCameraOptions
12
+ from genesis.typing import PositiveInt, PositiveFloat, PositiveVec2IType
13
+ from gs_nyx import nyx_py_sdk as nps
14
+ from gs_nyx import nyx_py_renderer as npr
15
+
16
+ if TYPE_CHECKING:
17
+ pass
18
+
19
+
20
+ __all__ = ["NyxCameraOptions"]
21
+
22
+
23
+ # Register init and destroy functions for the Nyx sdk plugin.
24
+ gs.register_external_module(nps.startup, nps.shutdown)
25
+
26
+
27
+ # ========================== Nyx Camera Options ==========================
28
+ class NyxCameraOptions(BaseCameraOptions["NyxCameraSensor"]):
29
+ """
30
+ Options for the Nyx Camera Sensor plugin.
31
+
32
+ Inherits from BaseCameraOptions which provides:
33
+ - res, pos, lookat, up, fov, lights, offset_T (from BaseCameraOptions)
34
+ - entity_idx, link_idx_local, pos_offset, euler_offset (from RigidSensorOptionsMixin)
35
+
36
+ Nyx-specific parameters are defined below.
37
+
38
+ Parameters
39
+ ----------
40
+ model : Literal
41
+ Camera model: "pinhole". Default is "pinhole".
42
+ spp : PositiveInt
43
+ Samples per pixel for path tracing. Default is 16.
44
+ denoise : bool
45
+ Whether to apply denoising. Default is True.
46
+ near : PositiveFloat
47
+ Near clipping plane distance. Default is 0.1.
48
+ far : PositiveFloat
49
+ Far clipping plane distance. Default is 100.0.
50
+ render_mode : ~gs_nyx.nyx_py_renderer.ERenderMode
51
+ Rendering mode for Nyx. Default is
52
+ :attr:`~gs_nyx.nyx_py_renderer.ERenderMode.FastPathTracer`.
53
+ debug_view : ~gs_nyx.nyx_py_renderer.EDebugView
54
+ Debug view mode for Nyx. Default is
55
+ :attr:`~gs_nyx.nyx_py_renderer.EDebugView.Meshlet`.
56
+ tone_mapper : ~gs_nyx.nyx_py_sdk.EToneMapper
57
+ Tone mapper to apply post-render. Default is
58
+ :attr:`~gs_nyx.nyx_py_sdk.EToneMapper.Reinhard`.
59
+ anti_aliasing : ~gs_nyx.nyx_py_sdk.EAntiAliasing
60
+ Anti-aliasing technique. Default is
61
+ :attr:`~gs_nyx.nyx_py_sdk.EAntiAliasing.SMAA`.
62
+ open_window : bool
63
+ Whether to open a GUI window. Default is False.
64
+ window_size : PositiveVec2IType
65
+ Window size if open_window is enabled. Default is (1024, 1024).
66
+ """
67
+
68
+ model_config = ConfigDict(arbitrary_types_allowed=True)
69
+
70
+ # Nyx-specific rendering parameters
71
+ model: Literal["pinhole"] = "pinhole"
72
+ spp: PositiveInt = 16
73
+ denoise: bool = True
74
+ near: PositiveFloat = 0.1
75
+ far: PositiveFloat = 100.0
76
+ render_mode: npr.ERenderMode = npr.ERenderMode.FastPathTracer
77
+ debug_view: npr.EDebugView = npr.EDebugView.Meshlet
78
+ tone_mapper: nps.EToneMapper = nps.EToneMapper.Reinhard
79
+ anti_aliasing: nps.EAntiAliasing = nps.EAntiAliasing.SMAA
80
+
81
+ # Window/export settings
82
+ open_window: bool = False
83
+ window_size: PositiveVec2IType = (1920, 1080)
84
+
85
+ # Sensor manager integration
86
+ update_ground_truth_only: bool = True
87
+
88
+ # Nyx additional params
89
+ env_maps: Annotated[tuple[nps.EnvironmentMapAsset, ...], Field(strict=False)] = ()
90
+ light_fields: Annotated[tuple[nps.LightFieldAsset, ...], Field(strict=False)] = ()
91
+
92
+ # Post-init validation.
93
+ def model_post_init(self, _):
94
+ super().model_post_init(_)
95
+
96
+ # We require far to be greater than near for valid depth range.
97
+ if self.far <= self.near:
98
+ gs.raise_exception("far must be greater than near")
@@ -0,0 +1,517 @@
1
+ """
2
+ Nyx Camera Sensor for Genesis - GPU-accelerated rendering via Nyx renderer.
3
+ """
4
+
5
+ # External imports
6
+ import ctypes
7
+ import os
8
+ from importlib.metadata import PackageNotFoundError, distribution
9
+ from typing import TYPE_CHECKING, Any, Dict, List, NamedTuple, Optional
10
+ import torch
11
+
12
+ # Internal imports
13
+ import genesis as gs
14
+
15
+ # Required for the pick result
16
+ if TYPE_CHECKING:
17
+ from genesis.engine.entities.base_entity import Entity
18
+ from genesis.engine.sensors.camera import BaseCameraSensor
19
+ from genesis.engine.sensors.base_sensor import (
20
+ Sensor,
21
+ )
22
+ from gs_nyx import nyx_py_sdk as nps
23
+
24
+ # Relative imports
25
+ from .nyx_renderer import NyxPyRenderer
26
+ from .nyx_scene_exporter import NyxSceneExporter
27
+ from .nyx_camera_options import NyxCameraOptions
28
+ from .nyx_camera_shared_metadata import NyxCameraSharedMetadata, _nyx_make_complete_path
29
+
30
+
31
+ __all__ = ["NyxCameraSensor", "NyxCameraData", "NyxPickPixelResult"]
32
+
33
+
34
+ # ========================== Data Class ==========================
35
+ class NyxCameraData(NamedTuple):
36
+ """Return type of :meth:`NyxCameraSensor.read`.
37
+
38
+ Attributes
39
+ ----------
40
+ rgb : torch.Tensor
41
+ Rendered RGB image, shape ``(B, H, W, 3)`` with ``dtype=torch.uint8``,
42
+ resident on the CUDA device. ``B`` is the number of parallel Genesis
43
+ environments (``1`` for non-batched scenes); ``H`` and ``W`` match
44
+ the camera's configured :attr:`~NyxCameraOptions.res`. Channel order
45
+ is RGB, values in ``[0, 255]``.
46
+ """
47
+
48
+ rgb: torch.Tensor
49
+
50
+
51
+ # ========================== Nyx Pick Pixel Result ==========================
52
+
53
+
54
+ class NyxPickPixelResult(NamedTuple):
55
+ """Hit information returned by :meth:`NyxCameraSensor.pick_pixel`.
56
+
57
+ Attributes
58
+ ----------
59
+ entity : genesis.engine.entities.base_entity.Entity
60
+ The Genesis entity the ray hit.
61
+ link_name : str
62
+ Name of the hit link or sub-geometry. For URDF entities this is the
63
+ link name; for MJCF entities it is ``"<link>_<vgeom_idx>"``;
64
+ empty string for all other morph types.
65
+ position : tuple of float
66
+ World-space hit point ``(x, y, z)`` in Genesis Z-up coordinates.
67
+ """
68
+
69
+ entity: "Entity"
70
+ link_name: str
71
+ position: tuple[float, float, float]
72
+
73
+
74
+ # ========================== Nyx Camera Sensor ==========================
75
+ class NyxCameraSensor(
76
+ BaseCameraSensor, Sensor[NyxCameraOptions, NyxCameraSharedMetadata, NyxCameraData]
77
+ ):
78
+ """GPU-accelerated camera sensor backed by the Nyx renderer.
79
+
80
+ Registers with the Genesis sensor manager, contributes one camera (plus
81
+ any configured lights / env maps / light fields) to a renderer instance
82
+ shared across all :class:`NyxCameraSensor` sensors in the scene, and
83
+ produces RGB images on demand via the standard Genesis
84
+ :meth:`~genesis.engine.sensors.base_sensor.Sensor.read` API.
85
+
86
+ Configure with :class:`NyxCameraOptions`. The first ``NyxCameraSensor``
87
+ to finish building drives the actual renderer setup (scene export +
88
+ native renderer startup); subsequent sensors slot into the same shared
89
+ instance.
90
+
91
+ See the user guide for a configured example; this class is rarely
92
+ instantiated directly — pass :class:`NyxCameraOptions` to
93
+ :meth:`Scene.add_sensor` instead.
94
+ """
95
+
96
+ def __init__(
97
+ self,
98
+ options: NyxCameraOptions,
99
+ idx: int,
100
+ manager: "gs.SensorManager",
101
+ ):
102
+ """Bind the sensor to a Genesis sensor manager.
103
+
104
+ Called by :meth:`Scene.add_sensor` — users should not invoke this
105
+ directly. The native renderer is **not** started here; setup runs
106
+ inside :meth:`build` once the Genesis scene is built and every
107
+ ``NyxCameraSensor`` in the scene has been added.
108
+
109
+ Parameters
110
+ ----------
111
+ options : NyxCameraOptions
112
+ Per-camera configuration (resolution, FOV, pose, render mode,
113
+ lights / env maps, etc.).
114
+ idx : int
115
+ Genesis-side sensor index assigned by the manager.
116
+ manager : genesis.SensorManager
117
+ The sensor manager that owns this sensor.
118
+ """
119
+ super().__init__(options, idx, manager)
120
+ self._options: NyxCameraOptions
121
+ self._camera_idx: Optional[int] = None
122
+
123
+ # Per-env camera poses from move_to_attach(), shape (B, 3) each, or None if not attached
124
+ self._attached_pos: Optional[torch.Tensor] = None
125
+ self._attached_lookat: Optional[torch.Tensor] = None
126
+ self._attached_up: Optional[torch.Tensor] = None
127
+
128
+ # ========================== Light Conversion ==========================
129
+
130
+ def _convert_light_dict_to_asset(self, light_dict: Dict[str, Any]) -> Any:
131
+ """
132
+ Convert a light configuration dict to an Nyx LightAsset. Handles coordinate conversion from Genesis Z-up to Nyx Y-up.
133
+
134
+ Parameters
135
+ ----------
136
+ light_dict : dict
137
+ Light configuration with keys:
138
+ - type: "point", "directional", or "spot"
139
+ - pos: (x, y, z) position in Z-up coordinates (for point/spot)
140
+ - dir: (x, y, z) direction in Z-up coordinates (for directional/spot)
141
+ - color: (r, g, b) RGB color [0-1]
142
+ - intensity: Light intensity
143
+ - shadow: Whether to cast shadows (default True)
144
+ - inner_angle, outer_angle: For spot lights (degrees)
145
+
146
+ Returns
147
+ -------
148
+ LightAsset
149
+ Nyx LightAsset object configured with Y-up coordinates.
150
+ """
151
+ light = nps.LightAsset()
152
+ light_type = light_dict.get("type", "directional")
153
+
154
+ if light_type == "point":
155
+ light.type = nps.ELightType.Point
156
+ # Get the position and convert it.
157
+ light.point_position = nps.float3(*light_dict.get("pos", (0, 0, 5)))
158
+ light.point_position = nps.float3_z_up_to_y_up_a(light.point_position)
159
+ light.point_range = light_dict.get("range", 100.0)
160
+ light.unit = nps.ELightUnit.Lumen
161
+
162
+ elif light_type == "directional":
163
+ light.type = nps.ELightType.Directional
164
+ # Get the direction and convert it.
165
+ light.directional_direction = nps.float3(*light_dict.get("dir", (0, 0, -1)))
166
+ light.directional_direction = nps.float3_z_up_to_y_up_a(
167
+ light.directional_direction
168
+ )
169
+ light.unit = nps.ELightUnit.Lux
170
+
171
+ elif light_type == "spot":
172
+ light.type = nps.ELightType.Spot
173
+ # Get the position and convert it.
174
+ light.spot_position = nps.float3(*light_dict.get("pos", (0, 0, 5)))
175
+ light.spot_position = nps.float3_z_up_to_y_up_a(light.spot_position)
176
+ # Get the direction and convert it.
177
+ light.spot_direction = nps.float3(*light_dict.get("dir", (0, 0, -1)))
178
+ light.spot_direction = nps.float3_z_up_to_y_up_a(light.spot_direction)
179
+ light.spot_innerAngle = light_dict.get("inner_angle", 15.0)
180
+ light.spot_outerAngle = light_dict.get("outer_angle", 30.0)
181
+ light.spot_range = light_dict.get("range", 100.0)
182
+ light.unit = nps.ELightUnit.Lumen
183
+
184
+ else:
185
+ gs.raise_exception(f"Unknown light type: {light_type}")
186
+
187
+ # Common light properties
188
+ color = light_dict.get("color", (1, 1, 1))
189
+ light.color = nps.float3(color[0], color[1], color[2])
190
+ light.intensity = light_dict.get("intensity", 1.0)
191
+ light.shadow = light_dict.get("shadow", True)
192
+
193
+ return light
194
+
195
+ def _apply_camera_transform(self, camera_T: torch.Tensor):
196
+ """
197
+ Store per-env camera transforms from the attachment system.
198
+
199
+ Called by BaseCameraSensor.move_to_attach() with the computed world transform.
200
+ The stored values are sent to the renderer in _render_current_state().
201
+
202
+ Parameters
203
+ ----------
204
+ camera_T : torch.Tensor
205
+ Transform matrix, shape (B, 4, 4) for batched envs or (4, 4) for single.
206
+ """
207
+ if len(camera_T.shape) == 2:
208
+ camera_T = camera_T.unsqueeze(0)
209
+
210
+ # Store per-env pos / lookat / up, all (B, 3)
211
+ self._attached_pos = camera_T[:, :3, 3]
212
+ self._attached_lookat = self._attached_pos + (-camera_T[:, :3, 2])
213
+ self._attached_up = camera_T[:, :3, 1]
214
+
215
+ self._stale = True
216
+
217
+ def build(self):
218
+ """Finalize the sensor and lazily start the shared Nyx renderer.
219
+
220
+ Called by the Genesis sensor manager once per sensor after the scene
221
+ has been built. Each call:
222
+
223
+ 1. Resolves the optional rig attachment via :class:`RigidSensorMixin`.
224
+ 2. Verifies CUDA is available (Nyx is GPU-only).
225
+ 3. Registers this camera in the shared metadata, assigns it a Nyx
226
+ camera index, and captures its configuration as a definition dict.
227
+ 4. If this is the final ``NyxCameraSensor`` left to build, collects
228
+ lights / env maps / light fields from **all** sibling sensors,
229
+ writes the scene-description JSON via :class:`NyxSceneExporter`,
230
+ and constructs the shared :class:`NyxPyRenderer`.
231
+ 5. Pre-allocates this sensor's slot in the shared image cache:
232
+ ``(B, H, W, 3)`` ``uint8`` CUDA tensor.
233
+
234
+ Raises
235
+ ------
236
+ genesis.GenesisException
237
+ If CUDA is unavailable, or if sibling
238
+ :class:`NyxCameraSensor` instances disagree on ``render_mode``
239
+ (the shared renderer can serve only one mode).
240
+ """
241
+ super().build() # Sets up _link from RigidSensorMixin
242
+
243
+ if not torch.cuda.is_available():
244
+ gs.raise_exception("NyxCameraSensor requires CUDA to be available.")
245
+
246
+ scene = self._manager._sim.scene
247
+
248
+ if self._shared_metadata.sensors is None:
249
+ self._shared_metadata.sensors = []
250
+ self._shared_metadata.camera_defs = []
251
+ self._shared_metadata.image_cache = {}
252
+
253
+ # Register this sensor and assign a camera index
254
+ self._camera_idx = len(self._shared_metadata.sensors)
255
+ self._shared_metadata.sensors.append(self)
256
+
257
+ # Store camera configuration as a dict
258
+ camera_def = {
259
+ "pos": self._options.pos,
260
+ "lookat": self._options.lookat,
261
+ "up": self._options.up,
262
+ "fov": self._options.fov,
263
+ "res": self._options.res,
264
+ "spp": self._options.spp,
265
+ "denoise": self._options.denoise,
266
+ "aperture": getattr(self._options, "aperture", 2.8),
267
+ "focal_len": getattr(self._options, "focal_length", 10.0),
268
+ "near": self._options.near,
269
+ "far": self._options.far,
270
+ "tone_mapper": self._options.tone_mapper,
271
+ "anti_aliasing": self._options.anti_aliasing,
272
+ }
273
+ self._shared_metadata.camera_defs.append(camera_def)
274
+
275
+ # Initialize renderer once all cameras are registered
276
+ all_nyx_sensors = self._manager._sensors_by_type[type(self)]
277
+ all_sensors_built = len(self._shared_metadata.sensors) == len(all_nyx_sensors)
278
+
279
+ if self._shared_metadata.renderer is None and all_sensors_built:
280
+ # Preload Nvidia compiler runtime if available (i.e. torch is not
281
+ # built from source). Must happen after the CUDA check and before
282
+ # the renderer is created, loading it at package-import time via
283
+ # RTLD_GLOBAL interferes with torch's CUDA detection.
284
+ try:
285
+ nvrtc_dist = distribution("nvidia_cuda_nvrtc")
286
+ for file in nvrtc_dist.files:
287
+ if file.name.startswith("libnvrtc-builtins.so.1"):
288
+ ctypes.CDLL(nvrtc_dist.locate_file(file), ctypes.RTLD_GLOBAL)
289
+ break
290
+ except (PackageNotFoundError, OSError):
291
+ pass
292
+
293
+ # Collect lights from all sensors and convert to LightAssets
294
+ lights = []
295
+ for sensor in self._shared_metadata.sensors:
296
+ for light_dict in sensor._options.lights:
297
+ lights.append(sensor._convert_light_dict_to_asset(light_dict))
298
+
299
+ # Collect env_maps from all sensors
300
+ env_maps = []
301
+ for sensor in self._shared_metadata.sensors:
302
+ for env_map in sensor._options.env_maps:
303
+ env_maps.append(env_map)
304
+
305
+ # Collect light_fields from all sensors
306
+ light_fields = []
307
+ for sensor in self._shared_metadata.sensors:
308
+ for light_field in sensor._options.light_fields:
309
+ light_fields.append(light_field)
310
+
311
+ # When there is no lights or env maps, we notify the user.
312
+ if len(lights) == 0 and len(env_maps) == 0:
313
+ print(
314
+ "\033[93m [WARNING] NyxCameraSensor: The current scene has no lights or environment maps. "
315
+ "If this is unintentional, please add a light source or an environment map.\033[0m"
316
+ )
317
+
318
+ # Generate the complete export path.
319
+ complete_path = _nyx_make_complete_path(
320
+ self._shared_metadata.scene_description_export_path
321
+ )
322
+
323
+ # Build the scene description
324
+ self._shared_metadata.scene_exporter = NyxSceneExporter(
325
+ scene,
326
+ cameras=self._shared_metadata.camera_defs,
327
+ export_folder=complete_path,
328
+ lights=lights,
329
+ env_maps=env_maps,
330
+ light_fields=light_fields,
331
+ )
332
+
333
+ # Export the scene to a json file that the Nyx renderer can read.
334
+ self._shared_metadata.scene_exporter.export_to_file(
335
+ os.path.join(
336
+ complete_path,
337
+ "nyx_scene.json",
338
+ )
339
+ )
340
+
341
+ # Get max resolution across all cameras
342
+ max_width, max_height = self._get_max_resolution(
343
+ [s._options for s in all_nyx_sensors]
344
+ )
345
+
346
+ # Verify all sensors use the same render_mode
347
+ render_mode = self._shared_metadata.sensors[0]._options.render_mode
348
+ for sensor in self._shared_metadata.sensors[1:]:
349
+ if sensor._options.render_mode != render_mode:
350
+ gs.raise_exception(
351
+ f"All Nyx cameras must use the same render_mode. "
352
+ f"Found '{render_mode}' and '{sensor._options.render_mode}'. "
353
+ f"Please ensure all NyxCameraOptions use the same render_mode."
354
+ )
355
+
356
+ # Create the Nyx renderer instance.
357
+ n_envs = max(self._manager._sim._B, 1)
358
+ debug_view = self._shared_metadata.sensors[0]._options.debug_view
359
+ self._shared_metadata.renderer = NyxPyRenderer(
360
+ scene,
361
+ os.path.join(
362
+ complete_path,
363
+ "nyx_scene.json",
364
+ ),
365
+ max_width,
366
+ max_height,
367
+ n_envs=n_envs,
368
+ open_window=self._options.open_window,
369
+ camera_defs=self._shared_metadata.camera_defs,
370
+ render_mode=render_mode,
371
+ debug_view=debug_view,
372
+ )
373
+ self._shared_metadata.renderer.build(
374
+ self._shared_metadata.scene_exporter._entity_uuid_pairs
375
+ )
376
+
377
+ # Initialize image cache for this camera (always on CUDA for Nyx renderer)
378
+ n_envs = max(self._manager._sim._B, 1)
379
+ h, w = self._options.res[1], self._options.res[0]
380
+ self._shared_metadata.image_cache[self._idx] = torch.zeros(
381
+ (n_envs, h, w, 3), dtype=torch.uint8, device="cuda"
382
+ )
383
+
384
+ def pick_pixel(
385
+ self, camera_index: int, x: int, y: int
386
+ ) -> Optional[NyxPickPixelResult]:
387
+ """
388
+ Cast a ray through pixel (x, y) of the given camera and return what it hits.
389
+
390
+ Parameters
391
+ ----------
392
+ camera_index : int
393
+ Index of the camera within this sensor's shared metadata. For a single
394
+ NyxCameraSensor this is 0; multi-camera setups index in registration order.
395
+ x, y : int
396
+ Pixel coordinates in image space, with the origin at the top-left of the
397
+ framebuffer. Must lie within the camera's configured resolution.
398
+
399
+ Returns
400
+ -------
401
+ NyxPickPixelResult or None
402
+ ``None`` if the ray missed all scene geometry (e.g. background / sky).
403
+ Otherwise a NyxPickPixelResult with fields:
404
+
405
+ - ``entity``: the Genesis Entity that was hit
406
+ - ``link_name``: name of the hit link (URDF) or ``<link>_<vgeom_idx>`` (MJCF);
407
+ empty string for other morph types
408
+ - ``position``: world-space hit point as ``(x, y, z)`` in Genesis Z-up coordinates
409
+ """
410
+ # Pick a pixel using the Nyx renderer's picking functionality.
411
+ res = self._shared_metadata.renderer.pick_pixel(
412
+ self._shared_metadata.scene_exporter._entity_uuid_pairs, camera_index, x, y
413
+ )
414
+
415
+ # If res is None, it means no entity was picked at that pixel.
416
+ if res is None:
417
+ return None
418
+
419
+ # If not, convert the picked result to Genesis entities and return as NyxPickPixelResult.
420
+ return NyxPickPixelResult(*res)
421
+
422
+ def _get_image_cache_entry(self):
423
+ """Return this sensor's entry in the shared image cache."""
424
+ return self._shared_metadata.image_cache[self._idx]
425
+
426
+ def _render_current_state(self):
427
+ """Perform the actual render for all cameras."""
428
+ # Shorthand
429
+ sensors = self._shared_metadata.sensors or [self]
430
+ renderer = self._shared_metadata.renderer
431
+ scene = self._manager._sim.scene
432
+
433
+ # Update visualization transforms
434
+ scene.visualizer.update_visual_states()
435
+
436
+ # Compute per-env attachment transforms for ALL attached sensors
437
+ for sensor in sensors:
438
+ if sensor._link is not None:
439
+ sensor.move_to_attach()
440
+
441
+ # Update window events, delta time, etc.
442
+ renderer.update()
443
+
444
+ # Nyx Renderer can only render a single env at a time
445
+ # So we loop over all envs and render all the sensors in this env
446
+ for env_idx in range(max(self._manager._sim._B, 1)):
447
+ # Update camera tensors for this env
448
+ for sensor in sensors:
449
+ if sensor._attached_pos is not None:
450
+ # Attached camera: use per-env transform from move_to_attach
451
+ renderer.update_camera_tensor(
452
+ sensor._camera_idx,
453
+ sensor._attached_pos[env_idx],
454
+ sensor._attached_lookat[env_idx],
455
+ sensor._attached_up[env_idx],
456
+ )
457
+ else:
458
+ # Static camera: same pose for all envs
459
+ renderer.update_camera_tensor(
460
+ sensor._camera_idx,
461
+ sensor._options.pos,
462
+ sensor._options.lookat,
463
+ sensor._options.up,
464
+ )
465
+
466
+ # Sync geometry data from Genesis to Nyx for this env
467
+ renderer.update_scene(env_idx)
468
+
469
+ # Render all cameras
470
+ for sensor in sensors:
471
+ rgb_tensor = renderer.render(camera_index=sensor._camera_idx)
472
+ sensor._shared_metadata.image_cache[sensor._idx][env_idx] = rgb_tensor
473
+
474
+ # Mark all sensors as fresh
475
+ for sensor in sensors:
476
+ sensor._stale = False
477
+
478
+ self._shared_metadata.last_render_timestep = self._manager._sim.scene.t
479
+
480
+ def update_camera_pose(self, pos=None, lookat=None, up=None):
481
+ """Update camera pose dynamically (applies to all envs on next render).
482
+
483
+ Detaches the camera from any rig so the specified world-space pose
484
+ is used uniformly across all environments.
485
+
486
+ Parameters
487
+ ----------
488
+ pos : tuple, optional
489
+ New camera position (x, y, z) in Z-up coordinates.
490
+ lookat : tuple, optional
491
+ New look-at point (x, y, z) in Z-up coordinates.
492
+ up : tuple, optional
493
+ New up vector (x, y, z) in Z-up coordinates.
494
+ """
495
+ if pos is not None:
496
+ self._options.pos = tuple(pos)
497
+ if lookat is not None:
498
+ self._options.lookat = tuple(lookat)
499
+ if up is not None:
500
+ self._options.up = tuple(up)
501
+
502
+ # Detach from rig
503
+ self._link = None
504
+ self._attached_pos = None
505
+ self._attached_lookat = None
506
+ self._attached_up = None
507
+
508
+ self._stale = True
509
+
510
+ @staticmethod
511
+ def _get_max_resolution(options_list: List[NyxCameraOptions]) -> tuple[int, int]:
512
+ """Get the maximum resolution across all camera options."""
513
+ if not options_list:
514
+ return 1024, 1024
515
+ max_width = max(opt.res[0] for opt in options_list)
516
+ max_height = max(opt.res[1] for opt in options_list)
517
+ return max_width, max_height