hyperray 0.0.1__py3-none-any.whl

hyperdrone/_vendor/rl-tools/src/rendering/raytracing/yaw_prediction/scene.cpp

@@ -0,0 +1,222 @@
+#include <rl_tools/operations/cpu_mux.h>
+
+#include "scene.h"
+
+#include <random>
+#include <cmath>
+
+namespace rlt = rl_tools;
+
+namespace rl_tools::rendering::raytracing::yaw_prediction {
+
+    SceneHandle* create_scene(const char* scene_path) {
+        auto* handle = new SceneHandle();
+
+        rlt::init(handle->device);
+        handle->env.scene_path = scene_path;
+
+        rlt::malloc(handle->device, handle->env);
+        rlt::init(handle->device, handle->env);
+
+        rlt::initial_parameters(handle->device, handle->env, handle->default_params);
+
+        handle->data_rng.seed(42);
+        rlt::malloc(handle->device, handle->sampling_rng);
+        rlt::init(handle->device, handle->sampling_rng, 123);
+
+        return handle;
+    }
+
+    void destroy_scene(SceneHandle* handle) {
+        if (handle) {
+            rlt::free(handle->device, handle->sampling_rng);
+            rlt::free(handle->device, handle->env);
+            delete handle;
+        }
+    }
+
+    using Camera = rlt::rendering::raytracing::Camera<float>;
+
+    void sample_camera_batch(
+        SceneHandle* handle,
+        Camera* cameras_out,
+        float* targets_out,
+        unsigned long batch_size,
+        float max_angle,
+        float fov_min,
+        float fov_max
+    ) {
+        using T = float;
+        constexpr T PI = static_cast<T>(3.14159265358979323846);
+
+        std::uniform_real_distribution<T> yaw_dist(0.0f, 2.0f * PI);
+        std::uniform_real_distribution<T> unit_dist(-1.0f, 1.0f);
+        std::uniform_real_distribution<T> swap_dist(0.0f, 1.0f);
+        std::uniform_real_distribution<T> fov_dist(fov_min, fov_max);
+
+        const T aspect = static_cast<T>(SCENE_CAM_WIDTH) / static_cast<T>(SCENE_CAM_HEIGHT);
+
+        for (unsigned long i = 0; i < batch_size; i++) {
+            rlt::rl::environments::raytracing_example::State<SceneHandle::SPEC> state;
+            auto params = handle->default_params;
+            rlt::sample_initial_state(handle->device, handle->env, params, state, handle->sampling_rng);
+
+            const T fov = fov_dist(handle->data_rng);
+            const T fov_radians = rlt::rendering::raytracing::degrees_to_radians(fov);
+            const T tan_half_hfov = std::tan(fov_radians / T{2});
+            const T tan_half_vfov = std::tan(fov_radians / T{2}) / aspect;
+            const T half_hfov = std::atan(tan_half_hfov);
+            const T half_vfov = std::atan(tan_half_vfov);
+
+            // Clamp displacement angles to FOV
+            const T max_delta_yaw = std::min(max_angle, half_hfov);
+            const T max_delta_pitch = std::min(max_angle, half_vfov);
+            const T max_roll = max_angle;
+
+            // Sample displacement angles
+            const T delta_yaw = unit_dist(handle->data_rng) * max_delta_yaw;
+            const T delta_pitch = unit_dist(handle->data_rng) * max_delta_pitch;
+            const T roll = unit_dist(handle->data_rng) * max_roll;
+
+            // Build rotation matrix R = R_roll * R_pitch * R_yaw
+            // Camera local frame: x=right, y=up, z=forward
+            // R_yaw: rotation around y-axis
+            // R_pitch: rotation around x-axis
+            // R_roll: rotation around z-axis
+            const T cdy = std::cos(delta_yaw), sdy = std::sin(delta_yaw);
+            const T cdp = std::cos(delta_pitch), sdp = std::sin(delta_pitch);
+            const T cdr = std::cos(roll), sdr = std::sin(roll);
+
+            // R = R_roll * R_pitch * R_yaw (computed analytically)
+            T R[3][3] = {
+                {cdr*cdy - sdr*sdp*sdy,   -sdr*cdp,   cdr*sdy + sdr*sdp*cdy},
+                {sdr*cdy + cdr*sdp*sdy,    cdr*cdp,   sdr*sdy - cdr*sdp*cdy},
+                {-cdp*sdy,                 sdp,        cdp*cdy              }
+            };
+
+            // Swap augmentation: randomly swap A and B
+            const bool do_swap = swap_dist(handle->data_rng) < 0.5f;
+
+            // Effective rotation for target computation (transposed if swapped)
+            T eff[3][3];
+            if (do_swap) {
+                for (int r = 0; r < 3; r++)
+                    for (int c = 0; c < 3; c++)
+                        eff[r][c] = R[c][r];
+            } else {
+                for (int r = 0; r < 3; r++)
+                    for (int c = 0; c < 3; c++)
+                        eff[r][c] = R[r][c];
+            }
+
+            // Base camera orientation (random horizontal yaw)
+            const T base_yaw = yaw_dist(handle->data_rng);
+            const T cby = std::cos(base_yaw), sby = std::sin(base_yaw);
+
+            // Camera A's local frame in FLU world coords
+            // forward = (cby, sby, 0), right = (sby, -cby, 0), up = (0, 0, 1)
+            const T forward_a[3] = {cby, sby, 0.0f};
+            const T right_a[3] = {sby, -cby, 0.0f};
+            const T up_a[3] = {0.0f, 0.0f, 1.0f};
+
+            const T position[3] = {
+                params.scene_translation[0] + state.position[0],
+                params.scene_translation[1] + state.position[1],
+                params.scene_translation[2] + state.position[2] + handle->env.eye_height
+            };
+            const T world_up[3] = {0, 0, 1};
+
+            auto make_base_cam = [&]() -> Camera {
+                const T look_at[3] = {
+                    position[0] + handle->env.look_ahead * cby,
+                    position[1] + handle->env.look_ahead * sby,
+                    position[2]
+                };
+                return rlt::make_camera_data(position, look_at, world_up, fov, aspect);
+            };
+
+            auto make_rotated_cam = [&]() -> Camera {
+                const T dir_b[3] = {
+                    R[0][2]*right_a[0] + R[1][2]*up_a[0] + R[2][2]*forward_a[0],
+                    R[0][2]*right_a[1] + R[1][2]*up_a[1] + R[2][2]*forward_a[1],
+                    R[0][2]*right_a[2] + R[1][2]*up_a[2] + R[2][2]*forward_a[2]
+                };
+                const T up_b[3] = {
+                    R[0][1]*right_a[0] + R[1][1]*up_a[0] + R[2][1]*forward_a[0],
+                    R[0][1]*right_a[1] + R[1][1]*up_a[1] + R[2][1]*forward_a[1],
+                    R[0][1]*right_a[2] + R[1][1]*up_a[2] + R[2][1]*forward_a[2]
+                };
+                const T look_at[3] = {
+                    position[0] + handle->env.look_ahead * dir_b[0],
+                    position[1] + handle->env.look_ahead * dir_b[1],
+                    position[2] + handle->env.look_ahead * dir_b[2]
+                };
+                return rlt::make_camera_data(position, look_at, up_b, fov, aspect);
+            };
+
+            if (do_swap) {
+                cameras_out[i] = make_rotated_cam();
+                cameras_out[batch_size + i] = make_base_cam();
+            } else {
+                cameras_out[i] = make_base_cam();
+                cameras_out[batch_size + i] = make_rotated_cam();
+            }
+
+            // Compute targets from effective rotation
+            // z2 = third column of eff (camera B's optical axis in camera A's frame)
+            const T z2x = eff[0][2];
+            const T z2y = eff[1][2];
+            const T z2z = eff[2][2];
+
+            // FOV-normalized projection
+            const T px = z2x / (z2z * tan_half_hfov);
+            const T py = z2y / (z2z * tan_half_vfov);
+
+            // Roll extraction via shortest-arc decomposition
+            // Normalize z2 to get direction n
+            const T z2_len = std::sqrt(z2x*z2x + z2y*z2y + z2z*z2z);
+            const T nx = z2x / z2_len;
+            const T ny = z2y / z2_len;
+            const T nz = z2z / z2_len;
+
+            // R_pan = shortest-arc rotation from e_z to n (Rodrigues formula)
+            // R_pan^T row 0: [1 - nx²*k, -nx*ny*k, -nx]  where k = 1/(1+nz)
+            // R_pan^T row 1: [-nx*ny*k, 1 - ny²*k, -ny]
+            const T k = static_cast<T>(1) / (static_cast<T>(1) + nz);
+
+            // R_roll_ext = R_pan^T * eff, extract angle from first column
+            const T eff00 = eff[0][0], eff10 = eff[1][0], eff20 = eff[2][0];
+            const T re00 = (static_cast<T>(1) - nx*nx*k)*eff00 + (-nx*ny*k)*eff10 + (-nx)*eff20;
+            const T re10 = (-nx*ny*k)*eff00 + (static_cast<T>(1) - ny*ny*k)*eff10 + (-ny)*eff20;
+            const T phi = std::atan2(re10, re00);
+
+            targets_out[i * 3 + 0] = px;
+            targets_out[i * 3 + 1] = py;
+            targets_out[i * 3 + 2] = phi / PI;
+        }
+    }
+
+    template <bool ASYNC>
+    void render_batch(SceneHandle* handle, const Camera* cameras) {
+#if defined(RL_TOOLS_RENDERING_RAYTRACING_BACKEND_OPTIX)
+        cudaMemcpy(rlt::data(rlt::cameras(handle->device, *handle->env.renderer)), cameras, SCENE_NUM_CAMERAS * sizeof(Camera), cudaMemcpyHostToDevice);
+#else
+        std::memcpy(rlt::data(rlt::cameras(handle->device, *handle->env.renderer)), cameras, SCENE_NUM_CAMERAS * sizeof(Camera));
+#endif
+        if constexpr (ASYNC){
+            rlt::render_launch(handle->device, *handle->env.renderer);
+        } else {
+            rlt::render(handle->device, *handle->env.renderer);
+        }
+    }
+    template void render_batch<true>(SceneHandle* handle, const Camera* cameras);
+    template void render_batch<false>(SceneHandle* handle, const Camera* cameras);
+
+    unsigned long get_num_indoor_states(SceneHandle* handle) {
+        return handle->env.num_indoor_initial_states;
+    }
+
+    uint32_t* get_framebuffer_device_ptr(SceneHandle* handle) {
+        return rlt::data(rlt::frame_buffer(handle->device, *handle->env.renderer));
+    }
+}

hyperdrone/_vendor/rl-tools/src/rendering/raytracing/yaw_prediction/scene.h

@@ -0,0 +1,68 @@
+#pragma once
+
+#include <rl_tools/rendering/raytracing/backends/optix/device.h>
+#include <cstdint>
+
+#include "../example/environment/environment.h"
+#include "../example/environment/operations_cpu.h"
+
+
+namespace rl_tools::rendering::raytracing::yaw_prediction {
+
+    // Shared configuration constants
+    static constexpr unsigned long SCENE_NUM_CAMERAS = 1024; // 2 * BATCH_SIZE
+#ifndef ABLATION_RESOLUTION
+#define ABLATION_RESOLUTION 64
+#endif
+    static constexpr unsigned long SCENE_CAM_WIDTH = ABLATION_RESOLUTION;
+    static constexpr unsigned long SCENE_CAM_HEIGHT = ABLATION_RESOLUTION;
+
+    struct SceneHandle {
+        using T = float;
+        using TI = unsigned long;
+
+        using SPEC = rl_tools::rl::environments::raytracing_example::Specification<T, TI, SCENE_NUM_CAMERAS, SCENE_CAM_WIDTH, SCENE_CAM_HEIGHT, 64>;
+        using ENV = rl_tools::rl::environments::raytracing_example::Environment<SPEC>;
+        using DEVICE = rl_tools::devices::DEVICE_FACTORY<>;
+
+        DEVICE device;
+        ENV env;
+
+        std::mt19937 data_rng;
+        DEVICE::SPEC::RANDOM::ENGINE<> sampling_rng;
+        rl_tools::rl::environments::raytracing_example::Parameters<SPEC> default_params;
+    };
+
+
+    SceneHandle* create_scene(const char* scene_path);
+    void destroy_scene(SceneHandle* handle);
+
+    // Sample a batch of camera pairs for rotation prediction training
+    // cameras_out: array of SCENE_NUM_CAMERAS Camera structs
+    //   [0..batch_size-1] = image A cameras (reference)
+    //   [batch_size..2*batch_size-1] = image B cameras (displaced by random rotation)
+    // targets_out: array of batch_size * 3 floats (p_x, p_y, phi/pi per sample)
+    //   p_x: FOV-normalized horizontal projection of camera B's axis in A's frame
+    //   p_y: FOV-normalized vertical projection
+    //   phi/pi: roll angle normalized by pi
+    // fov_min, fov_max: range of the random horizontal FOV in degrees
+    void sample_camera_batch(
+        SceneHandle* handle,
+        rl_tools::rendering::raytracing::Camera<float>* cameras_out,
+        float* targets_out,
+        unsigned long batch_size,
+        float max_angle,
+        float fov_min,
+        float fov_max
+    );
+
+    // Set cameras and render (blocking)
+    template <bool ASYNC>
+    void render_batch(SceneHandle* handle, const rl_tools::rendering::raytracing::Camera<float>* cameras);
+
+    // Get number of indoor initial states found
+    unsigned long get_num_indoor_states(SceneHandle* handle);
+
+    // Get device pointer to framebuffer (for direct GPU access from CUDA kernels)
+    uint32_t* get_framebuffer_device_ptr(SceneHandle* handle);
+}

hyperdrone/_vendor/rl-tools/src/rendering/raytracing/yaw_prediction/swift_app/.gitignore

@@ -0,0 +1,4 @@
+.build/
+YawPredictor.app/
+build
+YawPredictor.xcodeproj

hyperdrone/_vendor/rl-tools/src/rendering/raytracing/yaw_prediction/swift_app/CMakeLists.txt

@@ -0,0 +1,4 @@
+add_executable(rendering_raytracing_yaw_prediction_swift_app_h5_to_tar h5_to_tar.cpp)
+set_target_properties(rendering_raytracing_yaw_prediction_swift_app_h5_to_tar PROPERTIES CXX_STANDARD 17)
+target_include_directories(rendering_raytracing_yaw_prediction_swift_app_h5_to_tar PRIVATE ${CMAKE_SOURCE_DIR}/include)
+target_link_libraries(rendering_raytracing_yaw_prediction_swift_app_h5_to_tar PRIVATE rl_tools_full)

hyperdrone/_vendor/rl-tools/src/rendering/raytracing/yaw_prediction/swift_app/Info.plist

@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+    <key>CFBundleIdentifier</key>
+    <string>com.rltools.yaw-predictor</string>
+    <key>CFBundleName</key>
+    <string>YawPredictor</string>
+    <key>CFBundleExecutable</key>
+    <string>YawPredictor</string>
+    <key>CFBundlePackageType</key>
+    <string>APPL</string>
+    <key>CFBundleVersion</key>
+    <string>1</string>
+    <key>CFBundleShortVersionString</key>
+    <string>1.0</string>
+    <key>LSMinimumSystemVersion</key>
+    <string>13.0</string>
+    <key>NSCameraUsageDescription</key>
+    <string>Camera access is needed to capture images for yaw angle prediction.</string>
+    <key>NSHighResolutionCapable</key>
+    <true/>
+</dict>
+</plist>

hyperdrone/_vendor/rl-tools/src/rendering/raytracing/yaw_prediction/swift_app/Info_ios.plist

@@ -0,0 +1,36 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+    <key>CFBundleIdentifier</key>
+    <string>$(PRODUCT_BUNDLE_IDENTIFIER)</string>
+    <key>CFBundleName</key>
+    <string>YawPredictor</string>
+    <key>CFBundleExecutable</key>
+    <string>$(EXECUTABLE_NAME)</string>
+    <key>CFBundlePackageType</key>
+    <string>APPL</string>
+    <key>CFBundleVersion</key>
+    <string>1</string>
+    <key>CFBundleShortVersionString</key>
+    <string>1.0</string>
+    <key>MinimumOSVersion</key>
+    <string>16.0</string>
+    <key>NSCameraUsageDescription</key>
+    <string>Camera access is needed to capture images for yaw angle prediction.</string>
+    <key>UIDeviceFamily</key>
+    <array>
+        <integer>1</integer>
+    </array>
+    <key>UISupportedInterfaceOrientations</key>
+    <array>
+        <string>UIInterfaceOrientationPortrait</string>
+    </array>
+    <key>UILaunchScreen</key>
+    <dict/>
+    <key>UIFileSharingEnabled</key>
+    <true/>
+    <key>LSSupportsOpeningDocumentsInPlace</key>
+    <true/>
+</dict>
+</plist>

hyperdrone/_vendor/rl-tools/src/rendering/raytracing/yaw_prediction/swift_app/Sources/CameraManager.swift

@@ -0,0 +1,120 @@
+import AVFoundation
+import CoreImage
+#if os(iOS)
+import ARKit
+import simd
+#endif
+
+#if os(iOS)
+struct CapturedARFrame {
+    let image: CGImage
+    let transform: simd_float4x4
+    let timestamp: TimeInterval
+}
+#endif
+
+final class CameraManager: NSObject, ObservableObject {
+    @Published var currentFrame: CGImage?
+    @Published var horizontalFOV: Double = 65.0
+    @Published var trackingReady = false
+    #if os(iOS)
+    @Published var latestARFrame: CapturedARFrame?
+    var currentTransform: simd_float4x4?
+    #endif
+
+    private let context = CIContext()
+
+    #if os(iOS)
+    private let arSession = ARSession()
+    #else
+    private let session = AVCaptureSession()
+    private let output = AVCaptureVideoDataOutput()
+    private let queue = DispatchQueue(label: "camera.queue")
+    #endif
+
+    func start() {
+        #if os(iOS)
+        arSession.delegate = self
+        let config = ARWorldTrackingConfiguration()
+        config.isAutoFocusEnabled = true
+        arSession.run(config)
+        #else
+        session.sessionPreset = .medium
+        guard let camera = AVCaptureDevice.default(for: .video),
+              let input = try? AVCaptureDeviceInput(device: camera) else {
+            print("No camera available")
+            return
+        }
+        if session.canAddInput(input) {
+            session.addInput(input)
+        }
+        output.videoSettings = [
+            kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA
+        ]
+        output.setSampleBufferDelegate(self, queue: queue)
+        if session.canAddOutput(output) {
+            session.addOutput(output)
+        }
+        queue.async {
+            self.session.startRunning()
+        }
+        #endif
+    }
+
+    func stop() {
+        #if os(iOS)
+        arSession.pause()
+        #else
+        session.stopRunning()
+        #endif
+    }
+}
+
+#if os(iOS)
+extension CameraManager: ARSessionDelegate {
+    func session(_ session: ARSession, didUpdate frame: ARFrame) {
+        let tracking = frame.camera.trackingState
+
+        let pixelBuffer = frame.capturedImage
+        let ciImage = CIImage(cvPixelBuffer: pixelBuffer)
+            .oriented(.right)
+        guard let cgImage = context.createCGImage(ciImage, from: ciImage.extent) else { return }
+
+        DispatchQueue.main.async {
+            if case .normal = tracking {
+                self.trackingReady = true
+                self.currentTransform = frame.camera.transform
+                self.latestARFrame = CapturedARFrame(
+                    image: cgImage,
+                    transform: frame.camera.transform,
+                    timestamp: frame.timestamp
+                )
+            }
+            if self.trackingReady {
+                self.currentFrame = cgImage
+            }
+        }
+    }
+}
+#endif
+
+extension CameraManager: AVCaptureVideoDataOutputSampleBufferDelegate {
+    func captureOutput(_ output: AVCaptureOutput,
+                       didOutput sampleBuffer: CMSampleBuffer,
+                       from connection: AVCaptureConnection) {
+        guard let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return }
+        let ciImage = CIImage(cvPixelBuffer: pixelBuffer)
+
+        #if os(macOS)
+        let mirrored = ciImage.transformed(by: CGAffineTransform(scaleX: -1, y: 1)
+            .translatedBy(x: -ciImage.extent.width, y: 0))
+        guard let cgImage = context.createCGImage(mirrored, from: mirrored.extent) else { return }
+        #else
+        guard let cgImage = context.createCGImage(ciImage, from: ciImage.extent) else { return }
+        #endif
+
+        DispatchQueue.main.async {
+            self.currentFrame = cgImage
+        }
+    }
+}