hyperray 0.0.1__py3-none-any.whl

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/environments/pendulum/operations_cpu.h

@@ -0,0 +1,146 @@
+#include "../../../version.h"
+#if (defined(RL_TOOLS_DISABLE_INCLUDE_GUARDS) || !defined(RL_TOOLS_RL_ENVIRONMENTS_PENDULUM_OPERATIONS_CPU_H)) && (RL_TOOLS_USE_THIS_VERSION == 1)
+#pragma once
+#define RL_TOOLS_RL_ENVIRONMENTS_PENDULUM_OPERATIONS_CPU_H
+
+#include "pendulum.h"
+#include "operations_generic.h"
+
+#include <string>
+
+RL_TOOLS_NAMESPACE_WRAPPER_START
+namespace rl_tools{
+    template <typename DEVICE, typename SPEC>
+    std::string json(DEVICE&, rl::environments::Pendulum<SPEC>& env){
+        return "{}";
+    }
+    template <typename DEVICE, typename SPEC>
+    std::string json(DEVICE&, rl::environments::Pendulum<SPEC>& env, typename rl::environments::Pendulum<SPEC>::Parameters& parameters){
+        return "{}";
+    }
+    template <typename DEVICE, typename SPEC, typename STATE_SPEC>
+    std::string json(DEVICE&, rl::environments::Pendulum<SPEC>& env, typename rl::environments::Pendulum<SPEC>::Parameters& parameters, typename rl::environments::pendulum::State<STATE_SPEC>& state){
+        std::string json = "{";
+        json += "\"theta\":" + std::to_string(state.theta) + ",";
+        json += "\"theta_dot\":" + std::to_string(state.theta_dot);
+        json += "}";
+        return json;
+    }
+    template <typename DEVICE, typename SPEC, typename STATE_SPEC>
+    std::string json(DEVICE&, rl::environments::Pendulum<SPEC>& env, typename rl::environments::Pendulum<SPEC>::Parameters& parameters, typename rl::environments::pendulum::StateMultiTask<STATE_SPEC>& state){
+        std::string json = "{";
+        json += "\"theta\":" + std::to_string(state.theta) + ",";
+        json += "\"theta_dot\":" + std::to_string(state.theta_dot);
+        json += "\"invert_action\":" + std::to_string(state.invert_action);
+        json += "}";
+        return json;
+    }
+    template <typename DEVICE, typename SPEC, typename STATE_SPEC>
+    std::string json(DEVICE&, rl::environments::Pendulum<SPEC>& env, typename rl::environments::Pendulum<SPEC>::Parameters& parameters, typename rl::environments::pendulum::StateMeta<STATE_SPEC>& state){
+        std::string json = "{";
+        json += "\"theta\":" + std::to_string(state.theta) + ",";
+        json += "\"theta_dot\":" + std::to_string(state.theta_dot) + ",";
+        json += "\"invert_action\":" + std::to_string(state.invert_action) + ",";
+        json += "\"last_action\":" + std::to_string(state.last_action);
+        json += "}";
+        return json;
+    }
+
+    template <typename DEVICE, typename SPEC>
+    std::string get_ui(DEVICE& device, rl::environments::Pendulum<SPEC>& env){
+        // Implement the functions `export async function render(ui_state, parameters, state, action)` and `export async function init(canvas, parameters, options)` and `export` them so that they are available as ES6 imports
+        // Please have a look at https://studio.rl.tools which helps you create render functions interactively
+        std::string ui = R"RL_TOOLS_LITERAL(
+export async function init(canvas, options){
+    // Simply saving the context for 2D environments
+    return {
+        ctx: canvas.getContext('2d')
+    }
+}
+export async function render(ui_state, parameters, state, action) {
+    const ctx = ui_state.ctx
+    ctx.clearRect(0, 0, ctx.canvas.width, ctx.canvas.height);
+
+    const centerX = ctx.canvas.width / 2;
+    const centerY = ctx.canvas.height / 2;
+    const canvasWidth = ctx.canvas.width;
+
+    const pendulumLength = canvasWidth * 0.2;
+    const bobRadius = canvasWidth * 0.02;
+    const pivotRadius = canvasWidth * 0.01;
+
+    // Draw the Pendulum
+    const adjustedTheta = state.theta - Math.PI;
+
+    const pendulumX = centerX + pendulumLength * Math.sin(adjustedTheta);
+    const pendulumY = centerY + pendulumLength * Math.cos(adjustedTheta);
+
+    ctx.beginPath();
+    ctx.moveTo(centerX, centerY);
+    ctx.lineTo(pendulumX, pendulumY);
+    ctx.lineWidth = canvasWidth * 0.008;
+    ctx.strokeStyle = 'black';
+    ctx.stroke();
+
+    ctx.beginPath();
+    ctx.arc(pendulumX, pendulumY, bobRadius, 0, 2 * Math.PI);
+    ctx.fillStyle = '#7DB9B6';
+    ctx.fill();
+    ctx.stroke();
+
+    ctx.beginPath();
+    ctx.arc(centerX, centerY, pivotRadius, 0, 2 * Math.PI);
+    ctx.fillStyle = 'black';
+    ctx.fill();
+    ctx.stroke();
+
+    // Draw torque arc
+    const torqueMagnitude = -Math.max(-1, Math.min(action[0], 1));
+    const arrowRadius = canvasWidth * 0.08;
+    const magnitudeRadians = (Math.PI * 2 / 3 * torqueMagnitude);
+    const startAngle = Math.PI / 2 + (torqueMagnitude > 0 ? 0 : magnitudeRadians);
+    const endAngle = Math.PI / 2 + (torqueMagnitude < 0 ? 0 : magnitudeRadians);
+
+    ctx.beginPath();
+    ctx.arc(centerX, centerY, arrowRadius, startAngle, endAngle);
+    ctx.strokeStyle = 'black';
+    ctx.lineWidth = canvasWidth * 0.008;
+    ctx.stroke();
+
+    // Draw arrowhead
+    const arrowAngle = torqueMagnitude > 0 ? endAngle : startAngle;
+    const arrowHeadAngularOffset = torqueMagnitude * Math.PI/180*20
+    const arrowX = centerX + arrowRadius * Math.cos(arrowAngle + arrowHeadAngularOffset);
+    const arrowY = centerY + arrowRadius * Math.sin(arrowAngle + arrowHeadAngularOffset);
+
+    const headlen = canvasWidth * 0.04 * Math.min(Math.abs(torqueMagnitude)*2, 1);
+    const angleOffset = Math.PI / 6;
+    const rotationAngle = Math.PI / 2 + (torqueMagnitude > 0 ? 0 : Math.PI);
+
+    ctx.beginPath();
+    ctx.moveTo(arrowX, arrowY);
+    ctx.lineTo(
+        arrowX - headlen * Math.cos(arrowAngle + arrowHeadAngularOffset/2 - angleOffset + rotationAngle),
+        arrowY - headlen * Math.sin(arrowAngle + arrowHeadAngularOffset/2 - angleOffset + rotationAngle)
+    );
+    ctx.lineTo(
+        arrowX - headlen * Math.cos(arrowAngle + arrowHeadAngularOffset/2 + angleOffset + rotationAngle),
+        arrowY - headlen * Math.sin(arrowAngle + arrowHeadAngularOffset/2 + angleOffset + rotationAngle)
+    );
+    ctx.lineTo(arrowX, arrowY);
+    ctx.fillStyle = 'black';
+    ctx.fill();
+}
+        )RL_TOOLS_LITERAL";
+        return ui;
+    }
+    template <typename DEVICE, typename SPEC>
+    std::string get_description(DEVICE& device, rl::environments::Pendulum<SPEC>& env){
+        std::string description;
+        description += "C implementation of Pendulum-v1.";
+        return description;
+    }
+}
+RL_TOOLS_NAMESPACE_WRAPPER_END
+
+#endif

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/environments/pendulum/operations_generic.h

@@ -0,0 +1,262 @@
+#include "../../../version.h"
+#if (defined(RL_TOOLS_DISABLE_INCLUDE_GUARDS) || !defined(RL_TOOLS_RL_ENVIRONMENTS_PENDULUM_OPERATIONS_GENERIC_H)) && (RL_TOOLS_USE_THIS_VERSION == 1)
+#pragma once
+#define RL_TOOLS_RL_ENVIRONMENTS_PENDULUM_OPERATIONS_GENERIC_H
+#include "pendulum.h"
+#include "../operations_generic.h"
+RL_TOOLS_NAMESPACE_WRAPPER_START
+namespace rl_tools::rl::environments::pendulum {
+    template <typename T>
+    RL_TOOLS_FUNCTION_PLACEMENT T clip(T x, T min, T max){
+        x = x < min ? min : (x > max ? max : x);
+        return x;
+    }
+    template <typename DEVICE, typename T>
+    RL_TOOLS_FUNCTION_PLACEMENT T f_mod_python(const DEVICE& dev, T a, T b){
+        return a - b * math::floor(dev, a / b);
+    }
+
+    template <typename DEVICE, typename T>
+    RL_TOOLS_FUNCTION_PLACEMENT T angle_normalize(const DEVICE& dev, T x){
+        return f_mod_python(dev, (x + math::PI<T>), (2 * math::PI<T>)) - math::PI<T>;
+    }
+}
+RL_TOOLS_NAMESPACE_WRAPPER_END
+RL_TOOLS_NAMESPACE_WRAPPER_START
+namespace rl_tools{
+    template<typename DEVICE, typename SPEC>
+    RL_TOOLS_FUNCTION_PLACEMENT static void malloc(DEVICE& device, const rl::environments::Pendulum<SPEC>& env){ }
+    template<typename DEVICE, typename SPEC>
+    RL_TOOLS_FUNCTION_PLACEMENT static void free(DEVICE& device, const rl::environments::Pendulum<SPEC>& env){ }
+    template<typename DEVICE, typename SPEC>
+    RL_TOOLS_FUNCTION_PLACEMENT static void init(DEVICE& device, const rl::environments::Pendulum<SPEC>& env){ }
+    template<typename DEVICE, typename SPEC, typename RNG>
+    RL_TOOLS_FUNCTION_PLACEMENT static void sample_initial_parameters(DEVICE& device, const rl::environments::Pendulum<SPEC>& env, typename rl::environments::Pendulum<SPEC>::Parameters& parameters, RNG& rng){ }
+    template<typename DEVICE, typename SPEC>
+    RL_TOOLS_FUNCTION_PLACEMENT static void initial_parameters(DEVICE& device, const rl::environments::Pendulum<SPEC>& env, typename rl::environments::Pendulum<SPEC>::Parameters& parameters){ }
+    template<typename DEVICE, typename SPEC, typename STATE_SPEC>
+    RL_TOOLS_FUNCTION_PLACEMENT static void initial_state(DEVICE& device, const rl::environments::Pendulum<SPEC>& env, typename rl::environments::Pendulum<SPEC>::Parameters& parameters, typename rl::environments::pendulum::State<STATE_SPEC>& state){
+        state.theta = -math::PI<typename SPEC::T>;
+        state.theta_dot = 0;
+    }
+    template<typename DEVICE, typename SPEC, typename STATE_SPEC>
+    RL_TOOLS_FUNCTION_PLACEMENT static void initial_state(DEVICE& device, const rl::environments::Pendulum<SPEC>& env, typename rl::environments::Pendulum<SPEC>::Parameters& parameters, typename rl::environments::pendulum::StateLastAction<STATE_SPEC>& state){
+        initial_state(device, env, parameters, static_cast<typename rl::environments::pendulum::State<STATE_SPEC>&>(state));
+        state.last_action = 0;
+    }
+    template<typename DEVICE, typename SPEC, typename STATE_SPEC>
+    RL_TOOLS_FUNCTION_PLACEMENT static void initial_state(DEVICE& device, const rl::environments::Pendulum<SPEC>& env, typename rl::environments::Pendulum<SPEC>::Parameters& parameters, typename rl::environments::pendulum::StateMultiTask<STATE_SPEC>& state){
+        initial_state(device, env, parameters, static_cast<typename rl::environments::pendulum::State<STATE_SPEC>&>(state));
+        state.invert_action = false;
+    }
+    template<typename DEVICE, typename SPEC, typename STATE_SPEC, typename RNG>
+    RL_TOOLS_FUNCTION_PLACEMENT static void initial_state(DEVICE& device, const rl::environments::Pendulum<SPEC>& env, typename rl::environments::Pendulum<SPEC>::Parameters& parameters, typename rl::environments::pendulum::StateMeta<STATE_SPEC>& state){
+        sample_initial_state(device, env, parameters, static_cast<typename rl::environments::pendulum::StateLastAction<STATE_SPEC>&>(state));
+        state.invert_action = false;
+    }
+    template<typename DEVICE, typename SPEC, typename STATE_SPEC, typename RNG>
+    RL_TOOLS_FUNCTION_PLACEMENT static void sample_initial_state(DEVICE& device, const rl::environments::Pendulum<SPEC>& env, typename rl::environments::Pendulum<SPEC>::Parameters& parameters, typename rl::environments::pendulum::State<STATE_SPEC>& state, RNG& rng){
+        state.theta     = random::uniform_real_distribution(device.random, SPEC::PARAMETERS::initial_state_min_angle, SPEC::PARAMETERS::initial_state_max_angle, rng);
+        state.theta_dot = random::uniform_real_distribution(device.random, SPEC::PARAMETERS::initial_state_min_speed, SPEC::PARAMETERS::initial_state_max_speed, rng);
+    }
+    template<typename DEVICE, typename SPEC, typename STATE_SPEC, typename RNG>
+    RL_TOOLS_FUNCTION_PLACEMENT static void sample_initial_state(DEVICE& device, const rl::environments::Pendulum<SPEC>& env, typename rl::environments::Pendulum<SPEC>::Parameters& parameters, typename rl::environments::pendulum::StateLastAction<STATE_SPEC>& state, RNG& rng){
+        sample_initial_state(device, env, parameters, static_cast<typename rl::environments::pendulum::State<STATE_SPEC>&>(state), rng);
+        state.last_action = 0;
+    }
+    template<typename DEVICE, typename SPEC, typename STATE_SPEC, typename RNG>
+    RL_TOOLS_FUNCTION_PLACEMENT static void sample_initial_state(DEVICE& device, const rl::environments::Pendulum<SPEC>& env, typename rl::environments::Pendulum<SPEC>::Parameters& parameters, typename rl::environments::pendulum::StateMultiTask<STATE_SPEC>& state, RNG& rng){
+        sample_initial_state(device, env, parameters, static_cast<typename rl::environments::pendulum::State<STATE_SPEC>&>(state), rng);
+        state.invert_action = random::uniform_int_distribution(device.random, 0, 1, rng) == 1;
+    }
+    template<typename DEVICE, typename SPEC, typename STATE_SPEC, typename RNG>
+    RL_TOOLS_FUNCTION_PLACEMENT static void sample_initial_state(DEVICE& device, const rl::environments::Pendulum<SPEC>& env, typename rl::environments::Pendulum<SPEC>::Parameters& parameters, typename rl::environments::pendulum::StateMeta<STATE_SPEC>& state, RNG& rng){
+        sample_initial_state(device, env, parameters, static_cast<typename rl::environments::pendulum::StateLastAction<STATE_SPEC>&>(state), rng);
+        // initial_state(device, env, parameters, static_cast<typename rl::environments::pendulum::StateLastAction<STATE_SPEC>&>(state));
+        state.invert_action = random::uniform_int_distribution(device.random, 0, 1, rng) == 1;
+    }
+    template<typename DEVICE, typename SPEC, typename STATE_SPEC, typename ACTION_SPEC, typename RNG>
+    RL_TOOLS_FUNCTION_PLACEMENT typename SPEC::T step(DEVICE& device, const rl::environments::Pendulum<SPEC>& env, typename rl::environments::Pendulum<SPEC>::Parameters& parameters, const typename rl::environments::pendulum::State<STATE_SPEC>& state, const Matrix<ACTION_SPEC>& action, typename rl::environments::pendulum::State<STATE_SPEC>& next_state, RNG& rng) {
+        static_assert(ACTION_SPEC::ROWS == 1);
+        static_assert(ACTION_SPEC::COLS == 1);
+        using namespace rl::environments::pendulum;
+        typedef typename SPEC::T T;
+        typedef typename SPEC::PARAMETERS PARAMS;
+        T u_normalised = math::clamp(device.math, get(action, 0, 0), (T)-1, (T)1);
+        T u = PARAMS::max_torque * u_normalised;
+        T g = PARAMS::g;
+        T m = PARAMS::m;
+        T l = PARAMS::l;
+        T dt = PARAMS::dt;
+
+        u = clip(u, -PARAMS::max_torque, PARAMS::max_torque);
+
+        T newthdot = state.theta_dot + (3 * g / (2 * l) * math::sin(device.math, state.theta) + 3.0 / (m * l * l) * u) * dt;
+        newthdot = clip(newthdot, -PARAMS::max_speed, PARAMS::max_speed);
+        T newth = state.theta + newthdot * dt;
+
+        next_state.theta = newth;
+        next_state.theta_dot = newthdot;
+        return SPEC::PARAMETERS::dt;
+    }
+    template<typename DEVICE, typename SPEC, typename STATE_SPEC, typename ACTION_SPEC, typename RNG>
+    RL_TOOLS_FUNCTION_PLACEMENT typename SPEC::T step(DEVICE& device, const rl::environments::Pendulum<SPEC>& env, typename rl::environments::Pendulum<SPEC>::Parameters& parameters, const typename rl::environments::pendulum::StateLastAction<STATE_SPEC>& state, const Matrix<ACTION_SPEC>& action, typename rl::environments::pendulum::StateLastAction<STATE_SPEC>& next_state, RNG& rng){
+        using T = typename SPEC::T;
+        T dt = step(device, env, parameters, static_cast<const typename rl::environments::pendulum::State<STATE_SPEC>&>(state), action, static_cast<typename rl::environments::pendulum::State<STATE_SPEC>&>(next_state), rng);
+        next_state.last_action = get(action, 0, 0);
+        return dt;
+    }
+    template<typename DEVICE, typename SPEC, typename STATE_SPEC, typename ACTION_SPEC, typename RNG>
+    RL_TOOLS_FUNCTION_PLACEMENT typename SPEC::T step(DEVICE& device, const rl::environments::Pendulum<SPEC>& env, typename rl::environments::Pendulum<SPEC>::Parameters& parameters, const typename rl::environments::pendulum::StateMultiTask<STATE_SPEC>& state, const Matrix<ACTION_SPEC>& action, typename rl::environments::pendulum::StateMultiTask<STATE_SPEC>& next_state, RNG& rng){
+        using T = typename SPEC::T;
+        using TI = typename SPEC::TI;
+        Matrix<matrix::Specification<T, TI, 1, 1, false>> modified_action;
+        set(modified_action, 0, 0, state.invert_action ? -get(action, 0, 0) : get(action, 0, 0));
+        T dt = step(device, env, parameters, static_cast<const typename rl::environments::pendulum::State<STATE_SPEC>&>(state), modified_action, static_cast<typename rl::environments::pendulum::State<STATE_SPEC>&>(next_state), rng);
+        next_state.invert_action = state.invert_action;
+        return dt;
+    }
+    template<typename DEVICE, typename SPEC, typename STATE_SPEC, typename ACTION_SPEC, typename RNG>
+    RL_TOOLS_FUNCTION_PLACEMENT typename SPEC::T step(DEVICE& device, const rl::environments::Pendulum<SPEC>& env, typename rl::environments::Pendulum<SPEC>::Parameters& parameters, const typename rl::environments::pendulum::StateMeta<STATE_SPEC>& state, const Matrix<ACTION_SPEC>& action, typename rl::environments::pendulum::StateMeta<STATE_SPEC>& next_state, RNG& rng){
+        using T = typename SPEC::T;
+        using TI = typename SPEC::TI;
+        Matrix<matrix::Specification<T, TI, 1, 1, false>> modified_action;
+        set(modified_action, 0, 0, state.invert_action ? -get(action, 0, 0) : get(action, 0, 0));
+        T dt = step(device, env, parameters, static_cast<const typename rl::environments::pendulum::State<STATE_SPEC>&>(state), modified_action, static_cast<typename rl::environments::pendulum::State<STATE_SPEC>&>(next_state), rng);
+        next_state.last_action = get(action, 0, 0);
+        next_state.invert_action = state.invert_action;
+        return dt;
+    }
+    template<typename DEVICE, typename SPEC, typename ACTION_SPEC, typename STATE_SPEC, typename RNG>
+    RL_TOOLS_FUNCTION_PLACEMENT static typename SPEC::T reward(DEVICE& device, const rl::environments::Pendulum<SPEC>& env, typename rl::environments::Pendulum<SPEC>::Parameters& parameters, const typename rl::environments::pendulum::State<STATE_SPEC>& state, const Matrix<ACTION_SPEC>& action, const typename rl::environments::pendulum::State<STATE_SPEC>& next_state, RNG& rng){
+        using namespace rl::environments::pendulum;
+        typedef typename SPEC::T T;
+        T angle_norm = angle_normalize(device.math, state.theta);
+        T u_normalised = get(action, 0, 0);
+        T u = SPEC::PARAMETERS::max_torque * u_normalised;
+        T costs = angle_norm * angle_norm + 0.1 * state.theta_dot * state.theta_dot + 0.001 * (u * u);
+        return -costs;
+    }
+
+    template<typename DEVICE, typename SPEC, typename STATE_SPEC, typename OBS_TYPE_SPEC, typename OBS_SPEC, typename RNG>
+    RL_TOOLS_FUNCTION_PLACEMENT static void observe(DEVICE& device, const rl::environments::Pendulum<SPEC>& env, const typename rl::environments::Pendulum<SPEC>::Parameters& parameters, const typename rl::environments::pendulum::State<STATE_SPEC>& state, const typename rl::environments::pendulum::ObservationFourier<OBS_TYPE_SPEC>&, Matrix<OBS_SPEC>& observation, RNG& rng){
+        static_assert(OBS_SPEC::ROWS == 1);
+        static_assert(OBS_SPEC::COLS == 3);
+        typedef typename SPEC::T T;
+        using PARAMETERS = utils::typing::remove_reference_t<decltype(parameters)>;
+        set(observation, 0, 0, math::cos(device.math, state.theta));
+        set(observation, 0, 1, math::sin(device.math, state.theta));
+        set(observation, 0, 2, state.theta_dot);
+        if constexpr(PARAMETERS::OBSERVATION_NOISE_POSITION > 0){
+            T noise_cos = random::normal_distribution::sample(device.random, (T)0, PARAMETERS::OBSERVATION_NOISE_POSITION, rng);
+            increment(observation, 0, 0, noise_cos);
+            T noise_sin = random::normal_distribution::sample(device.random, (T)0, PARAMETERS::OBSERVATION_NOISE_POSITION, rng);
+            increment(observation, 0, 1, noise_sin);
+        }
+        if constexpr(PARAMETERS::OBSERVATION_NOISE_VELOCITY > 0){
+            T noise_velocity = random::normal_distribution::sample(device.random, (T)0, PARAMETERS::OBSERVATION_NOISE_VELOCITY, rng);
+            increment(observation, 0, 2, noise_velocity);
+        }
+    }
+    template<typename DEVICE, typename SPEC, typename STATE_SPEC, typename OBS_TYPE_SPEC, typename OBS_SPEC, typename RNG>
+    RL_TOOLS_FUNCTION_PLACEMENT static void observe(DEVICE& device, const rl::environments::Pendulum<SPEC>& env, const typename rl::environments::Pendulum<SPEC>::Parameters& parameters, const typename rl::environments::pendulum::State<STATE_SPEC>& state, const typename rl::environments::pendulum::ObservationRaw<OBS_TYPE_SPEC>&, Matrix<OBS_SPEC>& observation, RNG& rng){
+        static_assert(OBS_SPEC::ROWS == 1);
+        static_assert(OBS_SPEC::COLS == 2);
+        typedef typename SPEC::T T;
+        set(observation, 0, 0, rl::environments::pendulum::angle_normalize(device.math, state.theta));
+        set(observation, 0, 1, state.theta_dot);
+    }
+    template<typename DEVICE, typename SPEC, typename STATE_SPEC, typename OBS_TYPE_SPEC, typename OBS_SPEC, typename RNG>
+    RL_TOOLS_FUNCTION_PLACEMENT static void observe(DEVICE& device, const rl::environments::Pendulum<SPEC>& env, const typename rl::environments::Pendulum<SPEC>::Parameters& parameters, const typename rl::environments::pendulum::State<STATE_SPEC>& state, const typename rl::environments::pendulum::ObservationPosition<OBS_TYPE_SPEC>&, Matrix<OBS_SPEC>& observation, RNG& rng){
+        static_assert(OBS_SPEC::ROWS == 1);
+        static_assert(OBS_SPEC::COLS == 2);
+        typedef typename SPEC::T T;
+        using PARAMETERS = utils::typing::remove_reference_t<decltype(parameters)>;
+        set(observation, 0, 0, math::cos(device.math, state.theta));
+        set(observation, 0, 1, math::sin(device.math, state.theta));
+        if constexpr(PARAMETERS::OBSERVATION_NOISE_POSITION > 0){
+            T noise_cos = random::normal_distribution::sample(device.random, (T)0, PARAMETERS::OBSERVATION_NOISE_POSITION, rng);
+            increment(observation, 0, 0, noise_cos);
+            T noise_sin = random::normal_distribution::sample(device.random, (T)0, PARAMETERS::OBSERVATION_NOISE_POSITION, rng);
+            increment(observation, 0, 1, noise_sin);
+        }
+    }
+    template<typename DEVICE, typename SPEC, typename STATE_SPEC, typename OBS_TYPE_SPEC, typename OBS_SPEC, typename RNG>
+    RL_TOOLS_FUNCTION_PLACEMENT static void observe(DEVICE& device, const rl::environments::Pendulum<SPEC>& env, const typename rl::environments::Pendulum<SPEC>::Parameters& parameters, const typename rl::environments::pendulum::State<STATE_SPEC>& state, const typename rl::environments::pendulum::ObservationVelocity<OBS_TYPE_SPEC>&, Matrix<OBS_SPEC>& observation, RNG& rng){
+        using ENVIRONMENT = rl::environments::Pendulum<SPEC>;
+        using PARAMETERS = typename ENVIRONMENT::Parameters;
+        static_assert(OBS_SPEC::ROWS == 1);
+        static_assert(OBS_SPEC::COLS == 1);
+        using T = typename SPEC::T;
+        set(observation, 0, 0, state.theta_dot);
+        if constexpr(PARAMETERS::OBSERVATION_NOISE_VELOCITY > 0){
+            T noise_velocity = random::normal_distribution::sample(device.random, (T)0, PARAMETERS::OBSERVATION_NOISE_VELOCITY, rng);
+            increment(observation, 0, 0, noise_velocity);
+        }
+    }
+    template<typename DEVICE, typename SPEC, typename STATE_SPEC, typename OBS_TYPE_SPEC, typename OBS_SPEC, typename RNG>
+    RL_TOOLS_FUNCTION_PLACEMENT static void observe(DEVICE& device, const rl::environments::Pendulum<SPEC>& env, const typename rl::environments::Pendulum<SPEC>::Parameters& parameters, const typename rl::environments::pendulum::StateLastAction<STATE_SPEC>& state, const typename rl::environments::pendulum::ObservationVelocityLastAction<OBS_TYPE_SPEC>&, Matrix<OBS_SPEC>& observation, RNG& rng){
+        using ENVIRONMENT = rl::environments::Pendulum<SPEC>;
+        using PARAMETERS = typename ENVIRONMENT::Parameters;
+        static_assert(OBS_SPEC::ROWS == 1);
+        static_assert(OBS_SPEC::COLS == 2);
+        using T = typename SPEC::T;
+        set(observation, 0, 0, state.theta_dot);
+        set(observation, 0, 1, state.last_action);
+        if constexpr(PARAMETERS::OBSERVATION_NOISE_VELOCITY > 0){
+            T noise_velocity = random::normal_distribution::sample(device.random, (T)0, PARAMETERS::OBSERVATION_NOISE_VELOCITY, rng);
+            increment(observation, 0, 0, noise_velocity);
+        }
+    }
+    template<typename DEVICE, typename SPEC, typename STATE_SPEC, typename OBS_TYPE_SPEC, typename OBS_SPEC, typename RNG>
+    RL_TOOLS_FUNCTION_PLACEMENT static void observe(DEVICE& device, const rl::environments::Pendulum<SPEC>& env, const typename rl::environments::Pendulum<SPEC>::Parameters& parameters, const typename rl::environments::pendulum::StateMultiTask<STATE_SPEC>& state, const typename rl::environments::pendulum::ObservationMultiTask<OBS_TYPE_SPEC>&, Matrix<OBS_SPEC>& observation, RNG& rng){
+        using ENVIRONMENT = rl::environments::Pendulum<SPEC>;
+        using PARAMETERS = typename ENVIRONMENT::Parameters;
+        static_assert(OBS_SPEC::ROWS == 1);
+        static_assert(OBS_SPEC::COLS == 3 + 1);
+        using T = typename SPEC::T;
+        auto upstream_view = view(device, observation, matrix::ViewSpec<1, 3>{}, 0, 0);
+        observe(device, env, parameters, state, typename rl::environments::pendulum::ObservationFourier<OBS_TYPE_SPEC>{}, upstream_view, rng);
+        set(observation, 0, 3, state.invert_action ? -1 : 1);
+    }
+    template<typename DEVICE, typename SPEC, typename STATE_SPEC, typename OBS_TYPE_SPEC, typename OBS_SPEC, typename RNG>
+    RL_TOOLS_FUNCTION_PLACEMENT static void observe(DEVICE& device, const rl::environments::Pendulum<SPEC>& env, const typename rl::environments::Pendulum<SPEC>::Parameters& parameters, const typename rl::environments::pendulum::StateMeta<STATE_SPEC>& state, const typename rl::environments::pendulum::ObservationMeta<OBS_TYPE_SPEC>&, Matrix<OBS_SPEC>& observation, RNG& rng){
+        using ENVIRONMENT = rl::environments::Pendulum<SPEC>;
+        using PARAMETERS = typename ENVIRONMENT::Parameters;
+        static_assert(OBS_SPEC::ROWS == 1);
+        static_assert(OBS_SPEC::COLS == 3 + 1);
+        using T = typename SPEC::T;
+        auto upstream_view = view(device, observation, matrix::ViewSpec<1, 3>{}, 0, 0);
+        observe(device, env, parameters, state, typename rl::environments::pendulum::ObservationFourier<OBS_TYPE_SPEC>{}, upstream_view, rng);
+        set(observation, 0, 3, state.last_action);
+    }
+    // get_serialized_state is not generally required, it is just used in the WASM demonstration of the project page, where serialization is needed to go from the WASM runtime to the JavaScript UI
+    template<typename DEVICE, typename SPEC, typename STATE_SPEC>
+    RL_TOOLS_FUNCTION_PLACEMENT static typename SPEC::T get_serialized_state(DEVICE& device, const rl::environments::Pendulum<SPEC>& env, const typename rl::environments::pendulum::State<STATE_SPEC>& state, typename DEVICE::index_t index){
+        if(index == 0) {
+            return state.theta;
+        }
+        else{
+            return state.theta_dot;
+        }
+    }
+    template<typename DEVICE, typename SPEC, typename STATE_SPEC, typename RNG>
+    RL_TOOLS_FUNCTION_PLACEMENT static bool terminated(DEVICE& device, const rl::environments::Pendulum<SPEC>& env, typename rl::environments::Pendulum<SPEC>::Parameters& parameters, const typename rl::environments::pendulum::State<STATE_SPEC> state, RNG& rng){
+        using T = typename SPEC::T;
+        return false; //random::uniform_real_distribution(typename DEVICE::SPEC::RANDOM(), (T)0, (T)1, rng) > 0.9;
+    }
+    template<typename DEVICE, typename STATE_SPEC>
+    RL_TOOLS_FUNCTION_PLACEMENT typename STATE_SPEC::T abs_diff(DEVICE& device, const rl::environments::pendulum::State<STATE_SPEC>& s1, const rl::environments::pendulum::State<STATE_SPEC>& s2){
+        using T = typename STATE_SPEC::T;
+        T acc = 0;
+        acc += math::abs(device.math, s1.theta - s2.theta);
+        acc += math::abs(device.math, s1.theta_dot - s2.theta_dot);
+        return acc;
+    }
+    template<typename DEVICE, typename T>
+    RL_TOOLS_FUNCTION_PLACEMENT T abs_diff(DEVICE& device, const rl::environments::pendulum::DefaultParameters<T>& p1, const rl::environments::pendulum::DefaultParameters<T>& p2){
+        return 0;
+    }
+}
+RL_TOOLS_NAMESPACE_WRAPPER_END
+#endif

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/environments/pendulum/pendulum.h

@@ -0,0 +1,197 @@
+#include "../../../version.h"
+#if (defined(RL_TOOLS_DISABLE_INCLUDE_GUARDS) || !defined(RL_TOOLS_RL_ENVIRONMENTS_PENDULUM_PENDULUM_H)) && (RL_TOOLS_USE_THIS_VERSION == 1)
+#pragma once
+#define RL_TOOLS_RL_ENVIRONMENTS_PENDULUM_PENDULUM_H
+
+#include "../../../math/operations_generic.h"
+#include "../environments.h"
+#include "../observation.h"
+
+RL_TOOLS_NAMESPACE_WRAPPER_START
+namespace rl_tools::rl::environments::pendulum {
+    template <typename T>
+    struct DefaultParameters {
+        constexpr static T g = 10;
+        constexpr static T max_speed = 8;
+        constexpr static T max_torque = 2;
+        constexpr static T dt = 0.05;
+        constexpr static T m = 1;
+        constexpr static T l = 1;
+        constexpr static T initial_state_min_angle = -math::PI<T>;
+        constexpr static T initial_state_max_angle = math::PI<T>;
+        constexpr static T initial_state_min_speed = -1;
+        constexpr static T initial_state_max_speed = 1;
+        constexpr static T OBSERVATION_NOISE_POSITION = 0.0;
+        constexpr static T OBSERVATION_NOISE_VELOCITY = 0.0;
+    };
+    template <typename T_T, typename T_TI, typename T_PARAMETERS = DefaultParameters<T_T>>
+    struct Specification{
+        using T = T_T;
+        using TI = T_TI;
+        using PARAMETERS = T_PARAMETERS;
+    };
+
+    template <typename TI>
+    struct ObservationFourier{
+        static constexpr TI DIM = 3;
+        using SHAPE = tensor::Shape<TI, DIM>;
+    };
+    template <typename TI>
+    struct ObservationRaw{
+        static constexpr TI DIM = 2;
+        using SHAPE = tensor::Shape<TI, DIM>;
+    };
+    template <typename TI>
+    struct ObservationPosition{
+        static constexpr TI DIM = 2;
+        using SHAPE = tensor::Shape<TI, DIM>;
+    };
+    template <typename TI>
+    struct ObservationVelocity{
+        static constexpr TI DIM = 1;
+        using SHAPE = tensor::Shape<TI, DIM>;
+    };
+    template <typename TI>
+    struct ObservationVelocityLastAction{
+        static constexpr TI DIM = 2;
+        using SHAPE = tensor::Shape<TI, DIM>;
+    };
+    template <typename TI>
+    struct ObservationMultiTask{
+        static constexpr TI DIM = 3 + 1;
+        using SHAPE = tensor::Shape<TI, DIM>;
+    };
+    template <typename TI>
+    struct ObservationMeta{
+        static constexpr TI DIM = 3 + 1;
+        using SHAPE = tensor::Shape<TI, DIM>;
+    };
+
+
+    template <typename T_T, typename T_TI>
+    struct StateSpecification{
+        using T = T_T;
+        using TI = T_TI;
+    };
+    template <typename T_SPEC>
+    struct State{
+        using SPEC = T_SPEC;
+        using T = typename SPEC::T;
+        using TI = typename SPEC::TI;
+        static constexpr TI DIM = 2;
+        T theta;
+        T theta_dot;
+    };
+
+    template <typename T_SPEC>
+    struct StateLastAction: State<T_SPEC>{
+        using SPEC = T_SPEC;
+        using T = typename SPEC::T;
+        T last_action;
+    };
+    template <typename T_SPEC>
+    struct StateMultiTask: State<T_SPEC>{
+        bool invert_action;
+    };
+    template <typename T_SPEC>
+    struct StateMeta: StateLastAction<T_SPEC>{
+        bool invert_action;
+    };
+
+}
+RL_TOOLS_NAMESPACE_WRAPPER_END
+
+RL_TOOLS_NAMESPACE_WRAPPER_START
+namespace rl_tools::rl::environments{
+    template <typename T_SPEC>
+    struct Pendulum: Environment<typename T_SPEC::T, typename T_SPEC::TI>{
+        using SPEC = T_SPEC;
+        using T = typename SPEC::T;
+        using TI = typename SPEC::TI;
+        using State = pendulum::State<pendulum::StateSpecification<T, TI>>;
+        using Parameters = typename SPEC::PARAMETERS;
+        using Observation = pendulum::ObservationFourier<TI>;
+        using ObservationPrivileged = Observation;
+        static constexpr TI N_AGENTS = 1; // single agent
+        static constexpr TI ACTION_DIM = 1;
+        static constexpr TI EPISODE_STEP_LIMIT = 200;
+    };
+    template <typename T_SPEC>
+    struct PendulumAsymmetric: Pendulum<T_SPEC>{
+        using PENDULUM = Pendulum<T_SPEC>;
+        using Observation = pendulum::ObservationRaw<typename PENDULUM::TI>;
+        using ObservationPrivileged = pendulum::ObservationFourier<typename PENDULUM::TI>;
+    };
+    template <typename T_SPEC>
+    struct PendulumPosition: Pendulum<T_SPEC>{
+        using T = typename T_SPEC::T;
+        using TI = typename T_SPEC::TI;
+        using State = pendulum::StateLastAction<pendulum::StateSpecification<T, TI>>;
+        using Observation = pendulum::ObservationPosition<typename T_SPEC::TI>;
+        using ObservationPrivileged = Observation;
+    };
+    template <typename SPEC>
+    struct PREVENT_DEFAULT_GET_UI<PendulumPosition<SPEC>>: rl_tools::utils::typing::true_type{};
+    template <typename SPEC>
+    struct PREVENT_DEFAULT_GET_DESCRIPTION<PendulumPosition<SPEC>>: rl_tools::utils::typing::true_type{};
+    template <typename T_SPEC>
+    struct PendulumVelocity: Pendulum<T_SPEC>{
+        using T = typename T_SPEC::T;
+        using TI = typename T_SPEC::TI;
+        using State = pendulum::State<pendulum::StateSpecification<T, TI>>;
+        using Observation = pendulum::ObservationVelocity<typename T_SPEC::TI>;
+        using ObservationPrivileged = Observation;
+    };
+    template <typename SPEC>
+    struct PREVENT_DEFAULT_GET_UI<PendulumVelocity<SPEC>>: rl_tools::utils::typing::true_type{};
+    template <typename SPEC>
+    struct PREVENT_DEFAULT_GET_DESCRIPTION<PendulumVelocity<SPEC>>: rl_tools::utils::typing::true_type{};
+    template <typename T_SPEC>
+    struct PendulumVelocityLastAction: Pendulum<T_SPEC>{
+        using T = typename T_SPEC::T;
+        using TI = typename T_SPEC::TI;
+        using State = pendulum::StateLastAction<pendulum::StateSpecification<T, TI>>;
+        using Observation = pendulum::ObservationVelocityLastAction<typename T_SPEC::TI>;
+        using ObservationPrivileged = Observation;
+    };
+    template <typename SPEC>
+    struct PREVENT_DEFAULT_GET_UI<PendulumVelocityLastAction<SPEC>>: rl_tools::utils::typing::true_type{};
+    template <typename SPEC>
+    struct PREVENT_DEFAULT_GET_DESCRIPTION<PendulumVelocityLastAction<SPEC>>: rl_tools::utils::typing::true_type{};
+
+    template <typename T_SPEC>
+    struct PendulumMultiTask: Pendulum<T_SPEC>{
+        using T = typename T_SPEC::T;
+        using TI = typename T_SPEC::TI;
+        using State = pendulum::StateMultiTask<pendulum::StateSpecification<T, TI>>;
+        using Observation = pendulum::ObservationMultiTask<typename T_SPEC::TI>;
+        using ObservationPrivileged = Observation;
+    };
+    template <typename SPEC>
+    struct PREVENT_DEFAULT_GET_UI<PendulumMultiTask<SPEC>>: rl_tools::utils::typing::true_type{};
+    template <typename SPEC>
+    struct PREVENT_DEFAULT_GET_DESCRIPTION<PendulumMultiTask<SPEC>>: rl_tools::utils::typing::true_type{};
+
+    template <typename T_SPEC>
+    struct PendulumMeta: Pendulum<T_SPEC>{
+        using T = typename T_SPEC::T;
+        using TI = typename T_SPEC::TI;
+        using State = pendulum::StateMeta<pendulum::StateSpecification<T, TI>>;
+        using Observation = pendulum::ObservationMeta<typename T_SPEC::TI>;
+        using ObservationPrivileged = Observation;
+    };
+    template <typename SPEC>
+    struct PREVENT_DEFAULT_GET_UI<PendulumMeta<SPEC>>: rl_tools::utils::typing::true_type{};
+    template <typename SPEC>
+    struct PREVENT_DEFAULT_GET_DESCRIPTION<PendulumMeta<SPEC>>: rl_tools::utils::typing::true_type{};
+
+}
+RL_TOOLS_NAMESPACE_WRAPPER_END
+
+
+
+
+
+
+
+#endif