hyperdrone 0.1.2__tar.gz → 0.1.3__tar.gz

{hyperdrone-0.1.2 → hyperdrone-0.1.3}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: hyperdrone
-Version: 0.1.2
+Version: 0.1.3
 Summary: The fastest drone simulator and renderer (raytracer).
 Author-email: Jonas Eschmann <jonas.eschmann@gmail.com>
 License: MIT

{hyperdrone-0.1.2 → hyperdrone-0.1.3}/hyperdrone/_native/dynamics/impl.cpp

@@ -87,6 +87,7 @@ namespace hyperdrone_dynamics_impl {
         if(name == "crazyflie"){ out = pd::registry<pd::REGISTRY::crazyflie, PARAMETERS_SPEC>; return true; }
         if(name == "crazyflie_openmv"){ out = pd::registry<pd::REGISTRY::crazyflie_openmv, PARAMETERS_SPEC>; return true; }
         if(name == "mrs"){ out = pd::registry<pd::REGISTRY::mrs, PARAMETERS_SPEC>; return true; }
+        if(name == "x500"){ out = pd::registry<pd::REGISTRY::x500, PARAMETERS_SPEC>; return true; }
         if(name == "x500_real"){ out = pd::registry<pd::REGISTRY::x500_real, PARAMETERS_SPEC>; return true; }
         if(name == "x500_sim"){ out = pd::registry<pd::REGISTRY::x500_sim, PARAMETERS_SPEC>; return true; }
         if(name == "arpl"){ out = pd::registry<pd::REGISTRY::arpl, PARAMETERS_SPEC>; return true; }

{hyperdrone-0.1.2 → hyperdrone-0.1.3}/hyperdrone/_vendor/rl-tools/include/rl_tools/rendering/raytracing/README.md

@@ -27,8 +27,9 @@ horizontal = Y, image vertical = Z. GLB meshes (Y-up) are swizzled to FLU at loa
   per-sample overlay transforms, sample-major so slab `s` shares the `transforms()` layout),
   `transforms_pair()` (dynamic motion blur producer input: shutter-open/close entries),
   `frame_buffer()`, `depth_buffer()`, `segmentation_buffer()`, `normals_buffer()`,
-  `collision_results()`, `observation()`. Residency is a backend property: CUDA device memory
-  on OptiX, host on generic, shared/mapped on Metal/Vulkan.
+  `flow_buffer()`, `flow_deltas()`, `collision_results()`, `observation()`. Residency is a
+  backend property: CUDA device memory on OptiX, host on generic, shared/mapped on
+  Metal/Vulkan.
 - **Data moves via typed copies and kernels.** No raw backend handles or `cudaStream_t` appear in
   public signatures. Device producers (extraction kernels) write the input tensors in place;
   device consumers read the output tensors in place; host readers/writers stage through
@@ -47,6 +48,15 @@ horizontal = Y, image vertical = Z. GLB meshes (Y-up) are swizzled to FLU at loa
   of the hit surface, oriented against the ray, as float3 per pixel to `normals_buffer()`
   (miss = zero). Unit normals cannot be averaged, so anti-aliasing and motion-blur settings do
   not apply, and the pass stays out of the dynamic-motion-blur accumulate/resolve machinery.
+- **Flow output.** With `OUTPUT_FLOW`, another single-sample pass writes the backward optical
+  flow of the shutter-close frame in pixels to `flow_buffer()` (float2, miss = zero): the hit
+  point is carried to shutter open by its instance's shutter motion and projected through the
+  shutter-open camera. The two instants are the motion-blur pair inputs — `cameras_open()`
+  (whose storage flow enables independently of motion blur) and the `set_transform_pair`
+  shutter poses, composed per overlay slot into `flow_deltas()` (world_open ∘ world_close⁻¹;
+  single-pose verbs replicate, so held objects contribute pure camera flow). A producer drives
+  frame-to-frame flow by writing last frame's state into the open slots. On the device-resident
+  pair path the deltas are expanded on-device alongside `expand_motion_transforms`.
 - **Determinism is a contract.** Fixed seed ⇒ identical results, no atomics. The global
   instance-id layout (scene instances `[0,S)`, overlay `o` slot `s` at `S + o*MAX + s`) is
   cross-backend API surface consumed by segmentation.

{hyperdrone-0.1.2 → hyperdrone-0.1.3}/hyperdrone/_vendor/rl-tools/include/rl_tools/rendering/raytracing/backends/generic/operations_cpu.h

@@ -117,7 +117,7 @@ namespace rl_tools {
         static_assert(utils::typing::is_same_v<typename SPEC::T, float>, "The generic raytracing backend requires T = float");
 
         malloc(device, renderer.cameras);
-        if constexpr (SPEC::ENABLE_MOTION_BLUR) {
+        if constexpr (SPEC::HAS_CAMERA_PAIR) {
             malloc(device, renderer.cameras_open);
         }
         if constexpr (SPEC::HAS_RGB) {
@@ -150,7 +150,7 @@ namespace rl_tools {
         renderer.backend = backend_state;
         // the renderer-owned camera tensors are the render input — no staging copy
         backend_state->scene.cameras_close = data(renderer.cameras);
-        if constexpr (SPEC::ENABLE_MOTION_BLUR) {
+        if constexpr (SPEC::HAS_CAMERA_PAIR) {
             backend_state->scene.cameras_open = data(renderer.cameras_open);
         }
         else {
@@ -171,6 +171,14 @@ namespace rl_tools {
             malloc(device, renderer.normals_buffer);
             backend_state->scene.normals_buffer = data(renderer.normals_buffer);
         }
+        if constexpr (SPEC::HAS_FLOW) {
+            malloc(device, renderer.flow_buffer);
+            backend_state->scene.flow_buffer = data(renderer.flow_buffer);
+            if constexpr (SPEC::ENABLE_OVERLAYS) {
+                malloc(device, renderer.flow_deltas);
+                backend_state->scene.flow_deltas = data(renderer.flow_deltas);
+            }
+        }
         if constexpr (SPEC::HAS_OBSERVATION) {
             static_assert(utils::typing::is_same_v<typename SPEC::OBSERVATION_T, float>, "The generic raytracing backend requires OBSERVATION_T = float");
             malloc(device, renderer.observation);
@@ -360,6 +368,7 @@ namespace rl_tools {
         backend_state.scene.instance_classes = backend_state.instance_classes.data();
         backend_state.scene.instances = backend_state.instances.data();
         backend_state.scene.num_instances = (TI)backend_state.instances.size();
+        backend_state.scene.first_overlay_instance = backend_state.num_scene_instances;
 
         // the main TLAS spans only the shared-world instances; overlay slots live in their own TLASes
         const size_t num_instances = backend_state.num_scene_instances;
@@ -482,6 +491,9 @@ namespace rl_tools {
         namespace generic = rendering::raytracing::backends::generic;
         using TI = typename SPEC::TI;
         auto& backend_state = generic::state(renderer);
+        if constexpr (SPEC::HAS_FLOW){
+            rendering::raytracing::detail::compose_flow_deltas(renderer, data(renderer.flow_deltas));
+        }
         rendering::raytracing::detail::flush_overlay_transforms(renderer);
         if constexpr (SPEC::ENABLE_DYNAMIC_MOTION_BLUR){
             rendering::raytracing::detail::flush_overlay_motion_transforms(renderer);
@@ -528,7 +540,7 @@ namespace rl_tools {
                           const typename SPEC::T center[3], typename SPEC::T radius,
                           const typename SPEC::T up[3], typename SPEC::T fov){
         rendering::raytracing::detail::generate_camera_poses<SPEC>(device, data(renderer.cameras), center, radius, up, fov);
-        if constexpr (SPEC::ENABLE_MOTION_BLUR) {
+        if constexpr (SPEC::HAS_CAMERA_PAIR) {
             std::memcpy(data(renderer.cameras_open), data(renderer.cameras), SPEC::NUM_CAMERAS * sizeof(rendering::raytracing::Camera<typename SPEC::T>));
         }
     }
@@ -598,6 +610,9 @@ namespace rl_tools {
             if constexpr (SPEC::HAS_NORMALS) {
                 generic::render_normals_frame<DEVICE, SPEC>(device, generic::state(renderer).scene);
             }
+            if constexpr (SPEC::HAS_FLOW) {
+                generic::render_flow_frame<DEVICE, SPEC>(device, generic::state(renderer).scene);
+            }
             return;
         }
         if constexpr (SPEC::HAS_RGB) {
@@ -612,6 +627,9 @@ namespace rl_tools {
         if constexpr (SPEC::HAS_NORMALS) {
             generic::render_normals_frame<DEVICE, SPEC>(device, generic::state(renderer).scene);
         }
+        if constexpr (SPEC::HAS_FLOW) {
+            generic::render_flow_frame<DEVICE, SPEC>(device, generic::state(renderer).scene);
+        }
     }
 
     template <typename DEVICE, typename SPEC>
@@ -654,6 +672,12 @@ namespace rl_tools {
         rendering::raytracing::detail::write_normals_grid_png<SPEC>(data(renderer.normals_buffer), filename);
     }
 
+    template <typename DEVICE, typename SPEC>
+    void save_flow_image(DEVICE& device, rendering::raytracing::Renderer<SPEC, rendering::raytracing::backends::Generic>& renderer, const char* filename){
+        static_assert(SPEC::HAS_FLOW, "save_flow_image requires a flow-capable renderer specification");
+        rendering::raytracing::detail::write_flow_grid_png<SPEC>(data(renderer.flow_buffer), filename);
+    }
+
     template <typename DEVICE, typename SPEC>
     void save_image(DEVICE& device, rendering::raytracing::Renderer<SPEC, rendering::raytracing::backends::Generic>& renderer, const char* filename){
         static_assert(SPEC::HAS_RGB, "save_image requires an RGB-capable renderer specification");
@@ -695,7 +719,7 @@ namespace rl_tools {
             renderer.backend = nullptr;
         }
         free(device, renderer.cameras);
-        if constexpr (SPEC::ENABLE_MOTION_BLUR) {
+        if constexpr (SPEC::HAS_CAMERA_PAIR) {
             free(device, renderer.cameras_open);
         }
         if constexpr (SPEC::HAS_RGB) {
@@ -710,6 +734,12 @@ namespace rl_tools {
         if constexpr (SPEC::HAS_NORMALS) {
             free(device, renderer.normals_buffer);
         }
+        if constexpr (SPEC::HAS_FLOW) {
+            free(device, renderer.flow_buffer);
+            if constexpr (SPEC::ENABLE_OVERLAYS) {
+                free(device, renderer.flow_deltas);
+            }
+        }
         if constexpr (SPEC::HAS_OBSERVATION) {
             free(device, renderer.observation);
         }

{hyperdrone-0.1.2 → hyperdrone-0.1.3}/hyperdrone/_vendor/rl-tools/include/rl_tools/rendering/raytracing/backends/generic/operations_generic.h

@@ -136,6 +136,9 @@ namespace rl_tools {
             float* depth_buffer = nullptr;
             unsigned int* segmentation_buffer = nullptr;
             float* normals_buffer = nullptr; // 3 per pixel, world-frame unit normal or zero on miss
+            float* flow_buffer = nullptr;    // 2 per pixel, backward flow of the shutter-close frame in pixels
+            const float* flow_deltas = nullptr; // 12 per overlay slot: world_open ∘ world_close⁻¹
+            TI first_overlay_instance = 0;   // global instance ids >= this index the flow-delta table
             const unsigned int* instance_classes = nullptr; // indexed by global instance id
             CollisionResult* collision_results = nullptr;
             float* observation = nullptr; // 3 per pixel, written pre-quantization when set
@@ -1095,6 +1098,72 @@ namespace rl_tools {
             }
         }
 
+        // world point → screen fraction through the linear camera model: solves
+        // dir_00 + screen_x·dir_du + screen_y·dir_dv = lambda·(point − pos) by Cramer's rule;
+        // returns false for a degenerate basis or a point at/behind the camera (lambda <= 0)
+        template <typename T>
+        RL_TOOLS_FUNCTION_PLACEMENT bool project_camera(const Camera<T>& cam, Vec3<T> point, T& screen_x, T& screen_y){
+            const Vec3<T> direction = point - to_vec3(cam.pos);
+            const Vec3<T> dir_du = to_vec3(cam.dir_du);
+            const Vec3<T> dir_dv = to_vec3(cam.dir_dv);
+            const Vec3<T> negative_direction = -direction;
+            const Vec3<T> cross_dv_negative_direction = cross(dir_dv, negative_direction);
+            const T det = dot(dir_du, cross_dv_negative_direction);
+            if(det > (T)-1e-12 && det < (T)1e-12){
+                return false;
+            }
+            const T inv_det = (T)1 / det;
+            const Vec3<T> b = -to_vec3(cam.dir_00);
+            screen_x = dot(b, cross_dv_negative_direction) * inv_det;
+            screen_y = dot(dir_du, cross(b, negative_direction)) * inv_det;
+            const T lambda = dot(dir_du, cross(dir_dv, b)) * inv_det;
+            return lambda > (T)0;
+        }
+
+        // single-sample by design (shutter-close camera, pixel-center ray): backward flow of the
+        // shutter-close frame in pixels — the hit point is carried to shutter open by its
+        // instance's shutter delta (identity for the static world) and projected through the
+        // shutter-open camera; miss (and behind-the-open-camera projections) write (0, 0)
+        template <typename DEVICE, typename SPEC>
+        RL_TOOLS_FUNCTION_PLACEMENT void render_flow_frame(DEVICE& device, const SceneView<typename SPEC::T, typename SPEC::TI>& scene){
+            using T = typename SPEC::T;
+            using TI = typename SPEC::TI;
+            const auto& math_device = device.math;
+            for(TI camera_i = 0; camera_i < SPEC::NUM_CAMERAS; camera_i++){
+                const Camera<T>& cam = scene.cameras_close[camera_i];
+                const Camera<T>& cam_open = scene.cameras_open[camera_i];
+                const Vec3<T> pos = to_vec3(cam.pos);
+                const Vec3<T> dir_00 = to_vec3(cam.dir_00);
+                const Vec3<T> dir_du = to_vec3(cam.dir_du);
+                const Vec3<T> dir_dv = to_vec3(cam.dir_dv);
+                for(TI y = 0; y < SPEC::CAM_HEIGHT; y++){
+                    for(TI x = 0; x < SPEC::CAM_WIDTH; x++){
+                        const TI fb_offset = camera_i * SPEC::CAM_PIXELS + y * SPEC::CAM_WIDTH + x;
+                        const T screen_x = ((T)x + (T)0.5) / (T)SPEC::CAM_WIDTH;
+                        const T screen_y = ((T)y + (T)0.5) / (T)SPEC::CAM_HEIGHT;
+                        const Vec3<T> direction = normalize(math_device, dir_00 + screen_x * dir_du + screen_y * dir_dv);
+                        const Hit<T, TI> hit = trace_closest_composed<SPEC>(scene, camera_i, pos, direction, (T)0, (T)1e30);
+                        T flow_u = 0, flow_v = 0;
+                        if(hit.valid){
+                            Vec3<T> point = pos + direction * hit.t;
+                            if constexpr (SPEC::ENABLE_OVERLAYS){
+                                if(hit.instance >= scene.first_overlay_instance){
+                                    point = transform_point(&scene.flow_deltas[(TI)(hit.instance - scene.first_overlay_instance) * 12], point);
+                                }
+                            }
+                            T open_x, open_y;
+                            if(project_camera(cam_open, point, open_x, open_y)){
+                                flow_u = ((T)x + (T)0.5) - open_x * (T)SPEC::CAM_WIDTH;
+                                flow_v = ((T)y + (T)0.5) - open_y * (T)SPEC::CAM_HEIGHT;
+                            }
+                        }
+                        scene.flow_buffer[fb_offset * 2 + 0] = (float)flow_u;
+                        scene.flow_buffer[fb_offset * 2 + 1] = (float)flow_v;
+                    }
+                }
+            }
+        }
+
         // single-sample by design: unit normals cannot be averaged, so anti-aliasing and motion
         // blur do not apply (shutter-close camera, pixel-center ray); the geometric normal is
         // world-frame (FLU), oriented against the ray, zero on miss

{hyperdrone-0.1.2 → hyperdrone-0.1.3}/hyperdrone/_vendor/rl-tools/include/rl_tools/rendering/raytracing/backends/metal/context.h

@@ -44,6 +44,7 @@ namespace rl_tools::rendering::raytracing::backends::metal{
         constexpr int DEPTH_ACCUMULATOR = 16;
         constexpr int SHUTTER = 17;
         constexpr int DEPTH_OUTPUT = 18;
+        constexpr int FLOW_DELTAS = 19;
     }
     namespace function_constants{
         constexpr int SRGB_OUTPUT = 0;
@@ -78,7 +79,8 @@ namespace rl_tools::rendering::raytracing::backends::metal{
         float ambient_color[3];
         float miss_color_0[3];
         float miss_color_1[3];
-        float padding[3];
+        uint32_t first_overlay_instance; // global instance ids >= this index the flow-delta table
+        float padding[2];
     };
     static_assert(sizeof(LaunchParams) == 88, "LaunchParams layout must match the MSL declaration in device.metal");
 
@@ -133,6 +135,7 @@ namespace rl_tools::rendering::raytracing::backends::metal{
         NS::SharedPtr<MTL::ComputePipelineState> collision_pipeline;
         NS::SharedPtr<MTL::ComputePipelineState> segmentation_pipeline;
         NS::SharedPtr<MTL::ComputePipelineState> normals_pipeline;
+        NS::SharedPtr<MTL::ComputePipelineState> flow_pipeline;
         NS::SharedPtr<MTL::ComputePipelineState> resolve_pipeline;
         NS::SharedPtr<MTL::AccelerationStructure> acceleration_structure; // instance (top-level) AS
         std::vector<NS::SharedPtr<MTL::AccelerationStructure>> object_acceleration_structures;
@@ -162,6 +165,8 @@ namespace rl_tools::rendering::raytracing::backends::metal{
         NS::SharedPtr<MTL::Buffer> depth_accumulator;
         NS::SharedPtr<MTL::Buffer> segmentation_buffer;
         NS::SharedPtr<MTL::Buffer> normals_buffer;
+        NS::SharedPtr<MTL::Buffer> flow_buffer;
+        NS::SharedPtr<MTL::Buffer> flow_deltas;
         NS::SharedPtr<MTL::Buffer> observation;
         NS::SharedPtr<MTL::Buffer> collision_results;
         NS::SharedPtr<MTL::Buffer> probe_directions;

{hyperdrone-0.1.2 → hyperdrone-0.1.3}/hyperdrone/_vendor/rl-tools/include/rl_tools/rendering/raytracing/backends/metal/operations_cpu.h

@@ -86,6 +86,9 @@ namespace rl_tools {
             if(ctx.observation.get() != nullptr){
                 encoder->setBuffer(ctx.observation.get(), 0, bindings::OBSERVATION);
             }
+            if(ctx.flow_deltas.get() != nullptr){
+                encoder->setBuffer(ctx.flow_deltas.get(), 0, bindings::FLOW_DELTAS);
+            }
             for(auto& object_acceleration_structure : ctx.object_acceleration_structures){
                 encoder->useResource(object_acceleration_structure.get(), MTL::ResourceUsageRead);
             }
@@ -199,6 +202,14 @@ namespace rl_tools {
             ctx->normals_buffer = NS::TransferPtr(ctx->device->newBuffer((size_t)SPEC::NUM_CAMERAS * cam_pixels * 3 * sizeof(float), MTL::ResourceStorageModeShared));
             renderer.normals_buffer._data = (float*)ctx->normals_buffer->contents();
         }
+        if constexpr (SPEC::HAS_FLOW) {
+            ctx->flow_buffer = NS::TransferPtr(ctx->device->newBuffer((size_t)SPEC::NUM_CAMERAS * cam_pixels * 2 * sizeof(float), MTL::ResourceStorageModeShared));
+            renderer.flow_buffer._data = (float*)ctx->flow_buffer->contents();
+            if constexpr (SPEC::ENABLE_OVERLAYS) {
+                ctx->flow_deltas = NS::TransferPtr(ctx->device->newBuffer((size_t)SPEC::NUM_OVERLAYS * SPEC::MAX_OVERLAY_INSTANCES * 12 * sizeof(float), MTL::ResourceStorageModeShared));
+                renderer.flow_deltas._data = (float*)ctx->flow_deltas->contents();
+            }
+        }
         if constexpr (SPEC::HAS_OBSERVATION) {
             static_assert(utils::typing::is_same_v<typename SPEC::OBSERVATION_T, float>, "The Metal raytracing backend requires OBSERVATION_T = float");
             ctx->observation = NS::TransferPtr(ctx->device->newBuffer((size_t)SPEC::NUM_CAMERAS * cam_pixels * SPEC::OBSERVATION_CHANNELS * sizeof(float), MTL::ResourceStorageModeShared));
@@ -227,7 +238,7 @@ namespace rl_tools {
         constexpr size_t camera_bytes = (size_t)SPEC::NUM_CAMERAS * sizeof(rendering::raytracing::Camera<typename SPEC::T>);
         ctx->cameras = NS::TransferPtr(ctx->device->newBuffer(camera_bytes, MTL::ResourceStorageModeShared));
         renderer.cameras._data = (rendering::raytracing::Camera<typename SPEC::T>*)ctx->cameras->contents();
-        if constexpr (SPEC::ENABLE_MOTION_BLUR) {
+        if constexpr (SPEC::HAS_CAMERA_PAIR) {
             ctx->cameras_open = NS::TransferPtr(ctx->device->newBuffer(camera_bytes, MTL::ResourceStorageModeShared));
             renderer.cameras_open._data = (rendering::raytracing::Camera<typename SPEC::T>*)ctx->cameras_open->contents();
         }
@@ -538,6 +549,7 @@ namespace rl_tools {
             params->miss_color_0[0] = .8f; params->miss_color_0[1] = 0.f; params->miss_color_0[2] = 0.f;
             params->miss_color_1[0] = .8f; params->miss_color_1[1] = .8f; params->miss_color_1[2] = .8f;
         }
+        params->first_overlay_instance = (uint32_t)scene.instances.size();
 
         if(!ctx.pipelines_built){
             auto constants = NS::TransferPtr(MTL::FunctionConstantValues::alloc()->init());
@@ -602,6 +614,9 @@ namespace rl_tools {
             if constexpr (SPEC::HAS_NORMALS) {
                 ctx.normals_pipeline = make_pipeline("render_normals");
             }
+            if constexpr (SPEC::HAS_FLOW) {
+                ctx.flow_pipeline = make_pipeline("render_flow");
+            }
             if constexpr (SPEC::ENABLE_DYNAMIC_MOTION_BLUR) {
                 ctx.resolve_pipeline = make_pipeline("resolve_outputs");
             }
@@ -649,6 +664,9 @@ namespace rl_tools {
         }
         NS::Array* object_array = NS::Array::array((const NS::Object* const*)object_acceleration_structure_pointers.data(), object_acceleration_structure_pointers.size());
 
+        if constexpr (SPEC::HAS_FLOW){
+            rendering::raytracing::detail::compose_flow_deltas(renderer, data(renderer.flow_deltas));
+        }
         rendering::raytracing::detail::flush_overlay_transforms(renderer);
         // rebuilt unconditionally: producers may write the transforms tensor directly, which
         // leaves no host-observable dirty flag
@@ -815,7 +833,7 @@ namespace rl_tools {
         auto& ctx = metal::context(renderer);
         metal::wait_in_flight(ctx);
         rendering::raytracing::detail::generate_camera_poses<SPEC>(device, data(renderer.cameras), center, radius, up, fov);
-        if constexpr (SPEC::ENABLE_MOTION_BLUR) {
+        if constexpr (SPEC::HAS_CAMERA_PAIR) {
             std::memcpy(data(renderer.cameras_open), data(renderer.cameras), (size_t)SPEC::NUM_CAMERAS * sizeof(rendering::raytracing::Camera<typename SPEC::T>));
         }
     }
@@ -918,6 +936,9 @@ namespace rl_tools {
             if constexpr (SPEC::HAS_NORMALS) {
                 metal::encode_fullscreen_pass<SPEC>(ctx, command_buffer, ctx.normals_pipeline.get(), ctx.normals_buffer.get());
             }
+            if constexpr (SPEC::HAS_FLOW) {
+                metal::encode_fullscreen_pass<SPEC>(ctx, command_buffer, ctx.flow_pipeline.get(), ctx.flow_buffer.get());
+            }
             metal::encode_resolve_pass<SPEC>(ctx, command_buffer);
             command_buffer->commit();
             ctx.in_flight = NS::RetainPtr(command_buffer);
@@ -936,6 +957,9 @@ namespace rl_tools {
         if constexpr (SPEC::HAS_NORMALS) {
             metal::encode_fullscreen_pass<SPEC>(ctx, command_buffer, ctx.normals_pipeline.get(), ctx.normals_buffer.get());
         }
+        if constexpr (SPEC::HAS_FLOW) {
+            metal::encode_fullscreen_pass<SPEC>(ctx, command_buffer, ctx.flow_pipeline.get(), ctx.flow_buffer.get());
+        }
         command_buffer->commit();
         ctx.in_flight = NS::RetainPtr(command_buffer);
         pool->release();
@@ -1009,6 +1033,13 @@ namespace rl_tools {
         rendering::raytracing::detail::write_normals_grid_png<SPEC>(data(renderer.normals_buffer), filename);
     }
 
+    template <typename DEVICE, typename SPEC>
+    void save_flow_image(DEVICE& device, rendering::raytracing::Renderer<SPEC, rendering::raytracing::backends::Metal>& renderer, const char* filename){
+        static_assert(SPEC::HAS_FLOW, "save_flow_image requires a flow-capable renderer specification");
+        namespace metal = rendering::raytracing::backends::metal;
+        rendering::raytracing::detail::write_flow_grid_png<SPEC>(data(renderer.flow_buffer), filename);
+    }
+
     template <typename DEVICE, typename SPEC>
     void save_depth_image(DEVICE& device, rendering::raytracing::Renderer<SPEC, rendering::raytracing::backends::Metal>& renderer, const char* filename){
         static_assert(SPEC::HAS_DEPTH, "save_depth_image requires a depth-capable renderer specification");
@@ -1051,7 +1082,7 @@ namespace rl_tools {
         }
         // the input and output tensors alias shared MTLBuffers destroyed with the context
         renderer.cameras._data = nullptr;
-        if constexpr (SPEC::ENABLE_MOTION_BLUR) {
+        if constexpr (SPEC::HAS_CAMERA_PAIR) {
             renderer.cameras_open._data = nullptr;
         }
         if constexpr (SPEC::HAS_RGB) {
@@ -1066,6 +1097,12 @@ namespace rl_tools {
         if constexpr (SPEC::HAS_NORMALS) {
             renderer.normals_buffer._data = nullptr;
         }
+        if constexpr (SPEC::HAS_FLOW) {
+            renderer.flow_buffer._data = nullptr;
+            if constexpr (SPEC::ENABLE_OVERLAYS) {
+                renderer.flow_deltas._data = nullptr;
+            }
+        }
         if constexpr (SPEC::HAS_OBSERVATION) {
             renderer.observation._data = nullptr;
         }

{hyperdrone-0.1.2 → hyperdrone-0.1.3}/hyperdrone/_vendor/rl-tools/include/rl_tools/rendering/raytracing/backends/optix/device.h

@@ -194,6 +194,20 @@ namespace rl_tools {
         OptixCameraData *cameras;
     };
 
+    struct FlowRayGenData
+    {
+        float *flow_ptr;
+        owl::vec2i  fb_size;
+        owl::vec2i  cam_size;
+        int    grid_cols;
+        int    num_cameras;
+        OptixTraversableHandle world;
+        OptixCameraData *cameras_open;
+        OptixCameraData *cameras_close;
+        const float *flow_deltas;            // 12 per overlay slot (world_open ∘ world_close⁻¹), null without overlays
+        unsigned int first_overlay_instance; // global instance ids >= this index the delta table
+    };
+
     struct CollisionRayGenData
     {
         CollisionResult *results;    // [num_cameras * num_probes] output

{hyperdrone-0.1.2 → hyperdrone-0.1.3}/hyperdrone/_vendor/rl-tools/include/rl_tools/rendering/raytracing/backends/optix/device_impl.h

@@ -796,6 +796,105 @@ namespace rl_tools
     // Payloads stay as initialized by caller: distance = max_dist, hit = 0, instance = sentinel
   }
 
+  // world point → screen fraction through the linear camera model: solves
+  // dir_00 + screen_x·dir_du + screen_y·dir_dv = lambda·(point − pos) by Cramer's rule;
+  // returns false for a degenerate basis or a point at/behind the camera (lambda <= 0)
+  inline __device__ bool project_camera(const OptixCameraData &cam, const owl::vec3f &point, float &screen_x, float &screen_y)
+  {
+    const owl::vec3f direction = point - cam.pos;
+    const owl::vec3f negative_direction = -direction;
+    const owl::vec3f cross_dv_negative_direction = cross(cam.dir_dv, negative_direction);
+    const float det = dot(cam.dir_du, cross_dv_negative_direction);
+    if (det > -1e-12f && det < 1e-12f)
+      return false;
+    const float inv_det = 1.f / det;
+    const owl::vec3f b = -cam.dir_00;
+    screen_x = dot(b, cross_dv_negative_direction) * inv_det;
+    screen_y = dot(cam.dir_du, cross(b, negative_direction)) * inv_det;
+    const float lambda = dot(cam.dir_du, cross(cam.dir_dv, b)) * inv_det;
+    return lambda > 0.f;
+  }
+
+  inline __device__ owl::vec3f transform_point_rows(const float *m, const owl::vec3f &p)
+  {
+    return owl::vec3f(
+        m[0]*p.x + m[1]*p.y + m[2]*p.z + m[3],
+        m[4]*p.x + m[5]*p.y + m[6]*p.z + m[7],
+        m[8]*p.x + m[9]*p.y + m[10]*p.z + m[11]);
+  }
+
+  // single-sample by design (shutter-close camera, pixel-center ray): backward flow of the
+  // shutter-close frame in pixels. Traces the collision ray type, whose hit program reports the
+  // distance (payload 0) and global instance id (payload 2) — the hit point is carried to
+  // shutter open by its instance's shutter delta and projected through the shutter-open camera;
+  // miss (and behind-the-open-camera projections) write (0, 0). Compiled into every PTX variant
+  // (like the collision programs): no extra ray type or hit program, so no variant is needed.
+  OPTIX_RAYGEN_PROGRAM(flowRayGen)()
+  {
+    const FlowRayGenData &self = owl::getProgramData<FlowRayGenData>();
+    const owl::vec2i pixel = owl::getLaunchIndex();
+    const int tile_col = pixel.x / self.cam_size.x;
+    const int tile_row = pixel.y / self.cam_size.y;
+    const int cam_idx = tile_row * self.grid_cols + tile_col;
+    if (cam_idx >= self.num_cameras)
+      return;
+    const int local_x = pixel.x - tile_col * self.cam_size.x;
+    const int local_y = pixel.y - tile_row * self.cam_size.y;
+    const int fb_offset = cam_idx * self.cam_size.x * self.cam_size.y + local_y * self.cam_size.x + local_x;
+
+    const OptixCameraData &cam = self.cameras_close[cam_idx];
+    const float screen_x = ((float)local_x + 0.5f) / (float)self.cam_size.x;
+    const float screen_y = ((float)local_y + 0.5f) / (float)self.cam_size.y;
+    const owl::vec3f direction = normalize(cam.dir_00 + screen_x * cam.dir_du + screen_y * cam.dir_dv);
+
+    unsigned int u0 = __float_as_uint(1e30f);
+    unsigned int u1 = 0;
+    unsigned int u2 = 0xFFFFFFFFu;
+    optixTrace(
+        self.world,
+        (const float3&)cam.pos,
+        (const float3&)direction,
+        0.f,
+        1e30f,
+        0.0f,
+        OptixVisibilityMask(255),
+        OPTIX_RAY_FLAG_DISABLE_ANYHIT,
+        1, NUM_RAY_TYPES, 1, // collision ray type
+        u0, u1, u2);
+    const int overlay_count = optixLaunchParams.overlay_count;
+    for (int k = 0; k < overlay_count; k++) {
+      const uint32_t overlay = optixLaunchParams.attachments[cam_idx * overlay_count + k];
+      if (overlay == 0xFFFFFFFFu) continue;
+      optixTrace(
+          (OptixTraversableHandle)optixLaunchParams.overlays[overlay],
+          (const float3&)cam.pos,
+          (const float3&)direction,
+          0.f,
+          __uint_as_float(u0),
+          0.0f,
+          OptixVisibilityMask(255),
+          OPTIX_RAY_FLAG_DISABLE_ANYHIT,
+          1, NUM_RAY_TYPES, 1,
+          u0, u1, u2);
+    }
+    float flow_u = 0.f, flow_v = 0.f;
+    if (u1 != 0u) {
+      const float t = __uint_as_float(u0);
+      owl::vec3f point = owl::vec3f(cam.pos) + direction * t;
+      if (self.flow_deltas != nullptr && u2 >= self.first_overlay_instance) {
+        point = transform_point_rows(self.flow_deltas + (size_t)(u2 - self.first_overlay_instance) * 12, point);
+      }
+      const OptixCameraData &cam_open = self.cameras_open[cam_idx];
+      float open_x, open_y;
+      if (project_camera(cam_open, point, open_x, open_y)) {
+        flow_u = ((float)local_x + 0.5f) - open_x * (float)self.cam_size.x;
+        flow_v = ((float)local_y + 0.5f) - open_y * (float)self.cam_size.y;
+      }
+    }
+    self.flow_ptr[fb_offset * 2 + 0] = flow_u;
+    self.flow_ptr[fb_offset * 2 + 1] = flow_v;
+  }
+
 #if RL_TOOLS_RENDERING_RAYTRACING_ENABLE_NORMALS_PROGRAMS
   // normals ray type (2): the hit program reports the hit distance in payload 0 (consumed by
   // the overlay min-t composition) and the world-frame geometric normal, oriented against the