hyperverse 0.0.1__py3-none-any.whl

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/components/replay_buffer/operations_generic.h

@@ -0,0 +1,123 @@
+#include "../../../version.h"
+#if (defined(RL_TOOLS_DISABLE_INCLUDE_GUARDS) || !defined(RL_TOOLS_RL_COMPONENTS_REPLAY_BUFFER_OPERATIONS_GENERIC_H)) && (RL_TOOLS_USE_THIS_VERSION == 1)
+#pragma once
+#define RL_TOOLS_RL_COMPONENTS_REPLAY_BUFFER_OPERATIONS_GENERIC_H
+
+#include "replay_buffer.h"
+#include "../../../utils/generic/memcpy.h"
+
+RL_TOOLS_NAMESPACE_WRAPPER_START
+namespace rl_tools {
+    template <typename DEVICE, typename SPEC>
+    RL_TOOLS_FUNCTION_PLACEMENT void update_views(DEVICE& device, rl::components::ReplayBuffer<SPEC>& rb){
+        typename DEVICE::index_t offset = 0;
+        rb.observations                 = view(device, rb.data, matrix::ViewSpec<SPEC::CAPACITY, SPEC::OBSERVATION_DIM           >{}, 0, offset); offset += SPEC::ASYMMETRIC_OBSERVATIONS ? SPEC::OBSERVATION_DIM : 0;
+        rb.observations_privileged      = view(device, rb.data, matrix::ViewSpec<SPEC::CAPACITY, SPEC::OBSERVATION_DIM_PRIVILEGED>{}, 0, offset); offset += SPEC::OBSERVATION_DIM_PRIVILEGED;
+        rb.actions                      = view(device, rb.data, matrix::ViewSpec<SPEC::CAPACITY, SPEC::ACTION_DIM                >{}, 0, offset); offset += SPEC::ACTION_DIM;
+        rb.rewards                      = view(device, rb.data, matrix::ViewSpec<SPEC::CAPACITY, 1                               >{}, 0, offset); offset += 1;
+        rb.next_observations            = view(device, rb.data, matrix::ViewSpec<SPEC::CAPACITY, SPEC::OBSERVATION_DIM           >{}, 0, offset); offset += SPEC::ASYMMETRIC_OBSERVATIONS ? SPEC::OBSERVATION_DIM : 0;
+        rb.next_observations_privileged = view(device, rb.data, matrix::ViewSpec<SPEC::CAPACITY, SPEC::OBSERVATION_DIM_PRIVILEGED>{}, 0, offset); offset += SPEC::OBSERVATION_DIM_PRIVILEGED;
+        rb.terminated                   = view(device, rb.data, matrix::ViewSpec<SPEC::CAPACITY, 1                               >{}, 0, offset); offset += 1;
+        rb.truncated                    = view(device, rb.data, matrix::ViewSpec<SPEC::CAPACITY, 1                               >{}, 0, offset);
+    }
+    template <typename DEVICE, typename SPEC>
+    RL_TOOLS_FUNCTION_PLACEMENT void malloc(DEVICE& device, rl::components::ReplayBuffer<SPEC>& rb) {
+        malloc(device, rb.data);
+        malloc(device, rb.episode_start);
+        update_views(device, rb);
+    }
+    template <typename DEVICE, typename SPEC>
+    RL_TOOLS_FUNCTION_PLACEMENT void free(DEVICE& device, rl::components::ReplayBuffer<SPEC>& rb) {
+        free(device, rb.data);
+        free(device, rb.episode_start);
+    }
+    template <typename DEVICE, typename SPEC>
+    RL_TOOLS_FUNCTION_PLACEMENT void malloc(DEVICE& device, rl::components::ReplayBufferWithStates<SPEC>& rb) {
+        malloc(device, (rl::components::ReplayBuffer<typename SPEC::BASE_SPEC>&)rb);
+        malloc(device, rb.states);
+        malloc(device, rb.next_states);
+    }
+    template <typename DEVICE, typename SPEC>
+    RL_TOOLS_FUNCTION_PLACEMENT void free(DEVICE& device, rl::components::ReplayBufferWithStates<SPEC>& rb) {
+        free(device, (rl::components::ReplayBuffer<typename SPEC::BASE_SPEC>&)rb);
+        free(device, rb.states);
+        free(device, rb.next_states);
+    }
+    template <typename DEVICE, typename SPEC>
+    RL_TOOLS_FUNCTION_PLACEMENT void init(DEVICE& device, rl::components::ReplayBuffer<SPEC>& rb) {
+        update_views(device, rb);
+        rb.full = false;
+        rb.position = 0;
+        rb.current_episode_start = 0;
+    }
+    template <typename DEVICE, typename SPEC, typename STATE, typename OBSERVATION_SPEC, typename OBSERVATION_PRIVILEGED_SPEC, typename ACTION_SPEC, typename NEXT_OBSERVATION_SPEC, typename NEXT_OBSERVATION_PRIVILEGED_SPEC>
+    RL_TOOLS_FUNCTION_PLACEMENT void add(DEVICE& device, rl::components::ReplayBuffer<SPEC>& buffer, const STATE& state, const Matrix<OBSERVATION_SPEC>& observation, const Matrix<OBSERVATION_PRIVILEGED_SPEC>& observation_privileged, const Matrix<ACTION_SPEC>& action, typename SPEC::TYPE_POLICY::DEFAULT reward, const STATE& next_state, const Matrix<NEXT_OBSERVATION_SPEC>& next_observation, const Matrix<NEXT_OBSERVATION_PRIVILEGED_SPEC>& next_observation_privileged, const bool terminated, const bool truncated) {
+        // todo: change to memcpy?
+        for(typename DEVICE::index_t i = 0; i < SPEC::OBSERVATION_DIM; i++) {
+            set(buffer.observations, buffer.position, i, get(observation, 0, i));
+            set(buffer.next_observations, buffer.position, i, get(next_observation, 0, i));
+        }
+        for(typename DEVICE::index_t i = 0; i < SPEC::OBSERVATION_DIM_PRIVILEGED; i++) {
+            set(buffer.observations_privileged, buffer.position, i, get(observation_privileged, 0, i));
+            set(buffer.next_observations_privileged, buffer.position, i, get(next_observation_privileged, 0, i));
+        }
+        for(typename DEVICE::index_t i = 0; i < SPEC::ACTION_DIM; i++) {
+            set(buffer.actions, buffer.position, i, get(action, 0, i));
+        }
+        set(buffer.rewards, buffer.position, 0, reward);
+        set(buffer.terminated, buffer.position, 0, terminated);
+        set(buffer.truncated, buffer.position, 0, truncated);
+        set(device, buffer.episode_start, buffer.current_episode_start, buffer.position);
+        buffer.position = (buffer.position + 1) % SPEC::CAPACITY;
+        if(truncated){
+            buffer.current_episode_start = buffer.position;
+        }
+        if(buffer.position == 0 && !buffer.full) {
+            buffer.full = true;
+        }
+//        add_scalar(device, device.logger, "replay_buffer/position", (typename SPEC::T)(buffer.full ? SPEC::CAPACITY : buffer.position), 1000);
+    }
+    template <typename DEVICE, typename SPEC, typename STATE, typename OBSERVATION_SPEC, typename OBSERVATION_PRIVILEGED_SPEC, typename ACTION_SPEC, typename NEXT_OBSERVATION_SPEC, typename NEXT_OBSERVATION_PRIVILEGED_SPEC>
+    RL_TOOLS_FUNCTION_PLACEMENT void add(DEVICE& device, rl::components::ReplayBufferWithStates<SPEC>& buffer, const STATE& state, const Matrix<OBSERVATION_SPEC>& observation, const Matrix<OBSERVATION_PRIVILEGED_SPEC>& observation_privileged, const Matrix<ACTION_SPEC>& action, const typename SPEC::T reward, const STATE& next_state, const Matrix<NEXT_OBSERVATION_SPEC>& next_observation, const Matrix<NEXT_OBSERVATION_PRIVILEGED_SPEC>& next_observation_privileged, const bool terminated, const bool truncated) {
+        set(buffer.states, buffer.position, 0, state);
+        set(buffer.next_states, buffer.position, 0, next_state);
+        add(device, (rl::components::ReplayBuffer<typename SPEC::BASE_SPEC>&) buffer, state, observation, observation_privileged, action, reward, next_state, next_observation, next_observation_privileged, terminated, truncated);
+    }
+    template <typename SOURCE_DEVICE, typename TARGET_DEVICE, typename SOURCE_SPEC, typename TARGET_SPEC>
+    RL_TOOLS_FUNCTION_PLACEMENT void copy(SOURCE_DEVICE& source_device, TARGET_DEVICE& target_device, rl::components::ReplayBuffer<SOURCE_SPEC>& source, rl::components::ReplayBuffer<TARGET_SPEC>& target) {
+        copy(source_device, target_device, source.data, target.data);
+        copy(source_device, target_device, source.episode_start, target.episode_start);
+        target.full = source.full;
+        target.position = source.position;
+    }
+
+    template <typename DEVICE, typename SPEC_1, typename SPEC_2>
+    RL_TOOLS_FUNCTION_PLACEMENT typename SPEC_1::TYPE_POLICY::DEFAULT abs_diff(DEVICE& device, rl::components::ReplayBuffer<SPEC_1>& b1, rl::components::ReplayBuffer<SPEC_2>& b2) {
+        typename SPEC_1::TYPE_POLICY::DEFAULT acc = 0;
+        acc += abs_diff(device, b1.observations, b2.observations);
+        acc += abs_diff(device, b1.observations_privileged, b2.observations_privileged);
+        acc += abs_diff(device, b1.actions, b2.actions);
+        acc += abs_diff(device, b1.rewards, b2.rewards);
+        acc += abs_diff(device, b1.next_observations, b2.next_observations);
+        acc += abs_diff(device, b1.next_observations_privileged, b2.next_observations_privileged);
+        acc += abs_diff(device, b1.terminated, b2.terminated);
+        acc += abs_diff(device, b1.truncated, b2.truncated);
+        return acc;
+    }
+    template <typename DEVICE, typename SPEC, typename RNG>
+    RL_TOOLS_FUNCTION_PLACEMENT void recalculate_rewards(DEVICE& device, rl::components::ReplayBufferWithStates<SPEC>& buffer, const typename SPEC::ENVIRONMENT& env, RNG& rng) {
+        using TI = typename DEVICE::index_t;
+        using ENVIRONMENT = typename SPEC::ENVIRONMENT;
+        using STATE = typename ENVIRONMENT::State;
+        for(TI position_i = 0; position_i < SPEC::BASE_SPEC::CAPACITY; position_i++){
+            auto prev_r = get(buffer.rewards, position_i, 0);
+            STATE& state = get(buffer.states, position_i, 0);
+            STATE& next_state = get(buffer.next_states, position_i, 0);
+            auto action = row(device, buffer.actions, position_i);
+            auto r = reward(device, env, state, action, next_state, rng);
+            set(buffer.rewards, position_i, 0, r);
+        }
+    }
+}
+RL_TOOLS_NAMESPACE_WRAPPER_END
+#endif

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/components/replay_buffer/persist.h

@@ -0,0 +1,57 @@
+#include "../../../version.h"
+#if (defined(RL_TOOLS_DISABLE_INCLUDE_GUARDS) || !defined(RL_TOOLS_RL_COMPONENTS_REPLAY_BUFFER_PERSIST_H)) && (RL_TOOLS_USE_THIS_VERSION == 1)
+#pragma once
+#define RL_TOOLS_RL_COMPONENTS_REPLAY_BUFFER_PERSIST_H
+
+#include "replay_buffer.h"
+
+RL_TOOLS_NAMESPACE_WRAPPER_START
+namespace rl_tools{
+    template <typename DEVICE, typename SPEC, typename GROUP>
+    void save(DEVICE& device, rl::components::ReplayBuffer<SPEC>& rb, GROUP& group) {
+        using TI = typename SPEC::TI;
+        static_assert(decltype(rb.rewards)::COLS == 1);
+        static_assert(decltype(rb.terminated)::COLS == 1);
+        static_assert(decltype(rb.truncated)::COLS == 1);
+        save(device, rb.observations, group, "observations");
+        save(device, rb.actions, group, "actions");
+        save(device, rb.rewards, group, "rewards");
+        save(device, rb.next_observations, group, "next_observations");
+        save(device, rb.terminated, group, "terminated");
+        save(device, rb.truncated, group, "truncated");
+
+        save(device, rb.episode_start, group, "episode_start");
+
+        Tensor<tensor::Specification<decltype(rb.position), TI, tensor::Shape<TI, 1>, false>> position, full;
+        set(device, position, rb.position, 0);
+        save(device, position , group, "position");
+
+        set(device, full, rb.full, 0);
+        save(device, full, group, "full");
+    }
+    template <typename DEVICE, typename SPEC, typename GROUP>
+    bool load(DEVICE& device, rl::components::ReplayBuffer<SPEC>& rb, GROUP& group) {
+        using TI = typename SPEC::TI;
+        static_assert(decltype(rb.rewards)::COLS == 1);
+        static_assert(decltype(rb.terminated)::COLS == 1);
+        static_assert(decltype(rb.truncated)::COLS == 1);
+        bool success = load(device, rb.observations, group, "observations");
+        success &= load(device, rb.actions, group, "actions");
+        success &= load(device, rb.rewards, group, "rewards");
+        success &= load(device, rb.next_observations, group, "next_observations");
+        success &= load(device, rb.terminated, group, "terminated");
+        success &= load(device, rb.truncated, group, "truncated");
+        success &= load(device, rb.episode_start, group, "episode_start");
+
+        Tensor<tensor::Specification<decltype(rb.position), TI, tensor::Shape<TI, 1>, false>> position, full;
+        success &= load(device, position, group, "position");
+        rb.position = get(device, position, 0);
+
+        success &= load(device, full, group, "full");
+        rb.full = get(device, full, 0);
+        return success;
+    }
+}
+RL_TOOLS_NAMESPACE_WRAPPER_END
+
+#endif

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/components/replay_buffer/replay_buffer.h

@@ -0,0 +1,93 @@
+#include "../../../version.h"
+#if (defined(RL_TOOLS_DISABLE_INCLUDE_GUARDS) || !defined(RL_TOOLS_RL_COMPONENTS_REPLAY_BUFFER_REPLAY_BUFFER_H)) && (RL_TOOLS_USE_THIS_VERSION == 1)
+#pragma once
+#define RL_TOOLS_RL_COMPONENTS_REPLAY_BUFFER_REPLAY_BUFFER_H
+RL_TOOLS_NAMESPACE_WRAPPER_START
+namespace rl_tools::rl::components::replay_buffer{
+    template<typename T_TYPE_POLICY, typename T_TI, T_TI T_OBSERVATION_DIM, T_TI T_OBSERVATION_DIM_PRIVILEGED, bool T_ASYMMETRIC_OBSERVATIONS, T_TI T_ACTION_DIM, T_TI T_CAPACITY, bool T_DYNAMIC_ALLOCATION=true>
+    struct Specification{
+        using TYPE_POLICY = T_TYPE_POLICY;
+        using TI = T_TI;
+        static constexpr TI OBSERVATION_DIM = T_OBSERVATION_DIM;
+        static constexpr bool ASYMMETRIC_OBSERVATIONS = T_ASYMMETRIC_OBSERVATIONS && T_OBSERVATION_DIM_PRIVILEGED > 0;
+        static_assert(ASYMMETRIC_OBSERVATIONS == T_ASYMMETRIC_OBSERVATIONS, "ASYMMETRIC_OBSERVATIONS set but not available in the environment");
+        static constexpr TI OBSERVATION_DIM_PRIVILEGED = ASYMMETRIC_OBSERVATIONS ? T_OBSERVATION_DIM_PRIVILEGED : OBSERVATION_DIM;
+        static constexpr TI OBSERVATION_DIM_PRIVILEGED_ACTUAL = ASYMMETRIC_OBSERVATIONS ? T_OBSERVATION_DIM_PRIVILEGED : 0;
+        static constexpr TI ACTION_DIM = T_ACTION_DIM;
+        static constexpr TI CAPACITY = T_CAPACITY;
+        static constexpr bool DYNAMIC_ALLOCATION = T_DYNAMIC_ALLOCATION;
+    };
+
+    template<typename T_ENVIRONMENT, typename T_BASE_SPEC>
+    struct SpecificationWithStates: T_BASE_SPEC{
+        using BASE_SPEC = T_BASE_SPEC;
+        using ENVIRONMENT = T_ENVIRONMENT;
+    };
+
+}
+RL_TOOLS_NAMESPACE_WRAPPER_END
+
+RL_TOOLS_NAMESPACE_WRAPPER_START
+namespace rl_tools::rl::components {
+    template <typename T_SPEC>
+    struct ReplayBuffer {
+        using SPEC = T_SPEC;
+        using TYPE_POLICY = typename SPEC::TYPE_POLICY;
+        using T = typename TYPE_POLICY::DEFAULT;
+        using TI = typename SPEC::TI;
+        static constexpr TI CAPACITY = SPEC::CAPACITY;
+        static constexpr TI DATA_COLS = SPEC::OBSERVATION_DIM + SPEC::OBSERVATION_DIM_PRIVILEGED_ACTUAL + SPEC::ACTION_DIM + 1 + SPEC::OBSERVATION_DIM + SPEC::OBSERVATION_DIM_PRIVILEGED_ACTUAL + 1 + 1;
+
+        // mem
+        Matrix<matrix::Specification<T, TI, SPEC::CAPACITY, DATA_COLS, SPEC::DYNAMIC_ALLOCATION>> data;
+
+        TI current_episode_start = 0;
+        TI position = 0;
+        bool full = false;
+
+        // views
+        template<typename SPEC::TI DIM>
+        using DATA_VIEW = typename decltype(data)::template VIEW<CAPACITY, DIM>;
+
+        DATA_VIEW<SPEC::OBSERVATION_DIM> observations;
+        DATA_VIEW<SPEC::OBSERVATION_DIM_PRIVILEGED> observations_privileged;
+        DATA_VIEW<SPEC::ACTION_DIM> actions;
+        DATA_VIEW<1> rewards;
+        DATA_VIEW<SPEC::OBSERVATION_DIM> next_observations;
+        DATA_VIEW<SPEC::OBSERVATION_DIM_PRIVILEGED> next_observations_privileged;
+        DATA_VIEW<1> terminated;
+        DATA_VIEW<1> truncated;
+
+        Tensor<tensor::Specification<TI, TI, tensor::Shape<TI, SPEC::CAPACITY>, SPEC::DYNAMIC_ALLOCATION>> episode_start;
+    };
+
+    template <typename T_SPEC>
+    struct ReplayBufferWithStates: ReplayBuffer<typename T_SPEC::BASE_SPEC> {
+        // mem
+        using SPEC = typename T_SPEC::BASE_SPEC;
+        using ENVIRONMENT = typename T_SPEC::ENVIRONMENT;
+        using TYPE_POLICY = typename SPEC::TYPE_POLICY;
+        using T = typename TYPE_POLICY::DEFAULT;
+        using TI = typename SPEC::TI;
+        Matrix<matrix::Specification<typename ENVIRONMENT::State, TI, SPEC::CAPACITY, 1, SPEC::DYNAMIC_ALLOCATION>> states;
+        Matrix<matrix::Specification<typename ENVIRONMENT::State, TI, SPEC::CAPACITY, 1, SPEC::DYNAMIC_ALLOCATION>> next_states;
+    };
+}
+RL_TOOLS_NAMESPACE_WRAPPER_END
+RL_TOOLS_NAMESPACE_WRAPPER_START
+namespace rl_tools::rl::components::replay_buffer{
+    template <typename T_SPEC, auto T_SIZE>
+    struct SetSpecification{
+        static constexpr auto SIZE = T_SIZE;
+    };
+    template <typename T_SET_SPEC>
+    struct Set{
+        using SET_SPEC = T_SET_SPEC;
+        using SPEC = typename SET_SPEC::SPEC;
+        ReplayBuffer<SPEC> replay_buffers[SET_SPEC::SIZE];
+    };
+}
+RL_TOOLS_NAMESPACE_WRAPPER_END
+
+
+#endif

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/components/running_normalizer/operations_generic.h

@@ -0,0 +1,76 @@
+#include "../../../version.h"
+#if (defined(RL_TOOLS_DISABLE_INCLUDE_GUARDS) || !defined(RL_TOOLS_RL_COMPONENTS_RUNNING_NORMALIZER_OPERATIONS_GENERIC_H)) && (RL_TOOLS_USE_THIS_VERSION == 1)
+#pragma once
+#define RL_TOOLS_RL_COMPONENTS_RUNNING_NORMALIZER_OPERATIONS_GENERIC_H
+
+#include "running_normalizer.h"
+RL_TOOLS_NAMESPACE_WRAPPER_START
+namespace rl_tools{
+    template <typename DEVICE, typename SPEC>
+    RL_TOOLS_FUNCTION_PLACEMENT void malloc(DEVICE& device, rl::components::RunningNormalizer<SPEC>& normalizer){
+        malloc(device, normalizer.mean);
+        malloc(device, normalizer.std);
+    }
+    template <typename DEVICE, typename SPEC>
+    RL_TOOLS_FUNCTION_PLACEMENT void free(DEVICE& device, rl::components::RunningNormalizer<SPEC>& normalizer){
+        free(device, normalizer.mean);
+        free(device, normalizer.std);
+    }
+    template <typename DEVICE, typename SPEC>
+    RL_TOOLS_FUNCTION_PLACEMENT void init(DEVICE& device, rl::components::RunningNormalizer<SPEC>& normalizer){
+        normalizer.age = 0;
+        set_all(device, normalizer.mean, 0);
+        set_all(device, normalizer.std, 1);
+    }
+    template <typename DEVICE, typename SPEC, typename DATA_SPEC>
+    RL_TOOLS_FUNCTION_PLACEMENT void update(DEVICE& device, rl::components::RunningNormalizer<SPEC>& normalizer, Matrix<DATA_SPEC>& data){
+        // Note: data should have >> 2 rows; subsequent calls should have the same number of rows to not skeew the mean/std
+        // todo: take advantage of the data coming in batches
+        static_assert(DATA_SPEC::COLS == SPEC::DIM, "Data dimension must match normalizer dimension");
+        using T = typename SPEC::TYPE_POLICY::DEFAULT;
+        using TI = typename DEVICE::index_t;
+        constexpr TI DATA_SIZE = DATA_SPEC::ROWS;
+        static_assert(DATA_SIZE > 1, "Data size must be greater than 1 and should be much greated than one");
+        normalizer.age++;
+        for(TI col_i = 0; col_i < SPEC::DIM; col_i++){
+            auto column = col(device, data, col_i);
+            T data_mean = mean(device, column);
+            T data_std = std(device, column);
+            T mean = get(normalizer.mean, 0, col_i);
+            T new_mean = mean + (data_mean - mean)/(normalizer.age);
+            T std = get(normalizer.std, 0, col_i);
+            T new_std = std + (data_std - std)/(normalizer.age);
+            set(normalizer.mean, 0, col_i, new_mean);
+            set(normalizer.std, 0, col_i, new_std);
+        }
+    }
+    template <typename DEVICE, typename SPEC, typename DATA_SPEC>
+    RL_TOOLS_FUNCTION_PLACEMENT void normalize(DEVICE& device, rl::components::RunningNormalizer<SPEC>& normalizer, Matrix<DATA_SPEC>& data){
+        static_assert(DATA_SPEC::COLS == SPEC::DIM, "Data dimension must match normalizer dimension");
+        normalize(device, normalizer.mean, normalizer.std, data, data);
+    }
+    template <typename DEVICE, typename SPEC, typename INPUT_SPEC, typename OUTPUT_SPEC>
+    RL_TOOLS_FUNCTION_PLACEMENT void normalize(DEVICE& device, rl::components::RunningNormalizer<SPEC>& normalizer, Matrix<INPUT_SPEC>& input, Matrix<OUTPUT_SPEC>& output){
+        static_assert(containers::check_structure<INPUT_SPEC, OUTPUT_SPEC>);
+        static_assert(INPUT_SPEC::COLS == SPEC::DIM, "Data dimension must match normalizer dimension");
+        normalize(device, normalizer.mean, normalizer.std, input, output);
+    }
+    template <typename SOURCE_DEVICE, typename TARGET_DEVICE, typename SOURCE_SPEC, typename TARGET_SPEC>
+    RL_TOOLS_FUNCTION_PLACEMENT void copy(SOURCE_DEVICE& source_device, TARGET_DEVICE& target_device, rl::components::RunningNormalizer<SOURCE_SPEC>& source, rl::components::RunningNormalizer<TARGET_SPEC>& target){
+        static_assert(SOURCE_SPEC::DIM == TARGET_SPEC::DIM, "copy: source and target normalizers must have the same dimension");
+        copy(source_device, target_device, source.mean, target.mean);
+        copy(source_device, target_device, source.std, target.std);
+        target.age = source.age;
+    }
+    template <typename DEVICE, typename SPEC_1, typename SPEC_2>
+    RL_TOOLS_FUNCTION_PLACEMENT typename SPEC_1::TYPE_POLICY::DEFAULT abs_diff(DEVICE& device, rl::components::RunningNormalizer<SPEC_1>& n1, rl::components::RunningNormalizer<SPEC_2>& n2){
+        using T = typename SPEC_1::TYPE_POLICY::DEFAULT;
+        T acc = 0;
+        acc += abs_diff(device, n1.mean, n2.mean);
+        acc += abs_diff(device, n1.std, n2.std);
+        acc += math::abs(device.math, (T)n1.age - (T)n2.age);
+        return acc;
+    }
+}
+RL_TOOLS_NAMESPACE_WRAPPER_END
+#endif

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/components/running_normalizer/persist.h

@@ -0,0 +1,31 @@
+#include "../../../version.h"
+#if (defined(RL_TOOLS_DISABLE_INCLUDE_GUARDS) || !defined(RL_TOOLS_RL_COMPONENTS_RUNNING_NORMALIZER_PERSIST_H)) && (RL_TOOLS_USE_THIS_VERSION == 1)
+#pragma once
+#define RL_TOOLS_RL_COMPONENTS_RUNNING_NORMALIZER_PERSIST_H
+#include "running_normalizer.h"
+RL_TOOLS_NAMESPACE_WRAPPER_START
+namespace rl_tools{
+    template<typename DEVICE, typename SPEC, typename GROUP>
+    void save(DEVICE& device, rl::components::RunningNormalizer<SPEC>& normalizer, GROUP& group) {
+        save(device, normalizer.mean, group, "mean");
+        save(device, normalizer.std, group, "std");
+        Tensor<tensor::Specification<typename SPEC::TI, typename SPEC::TI, tensor::Shape<typename SPEC::TI, 1>>> age_tensor;
+        malloc(device, age_tensor);
+        set(device, age_tensor, normalizer.age, 0);
+        save(device, age_tensor, group, "age");
+        free(device, age_tensor);
+    }
+    template<typename DEVICE, typename SPEC, typename GROUP>
+    bool load(DEVICE& device, rl::components::RunningNormalizer<SPEC>& normalizer, GROUP& group) {
+        bool success = load(device, normalizer.mean, group, "mean");
+        success &= load(device, normalizer.std, group, "std");
+        Tensor<tensor::Specification<typename SPEC::TI, typename SPEC::TI, tensor::Shape<typename SPEC::TI, 1>>> age_tensor;
+        malloc(device, age_tensor);
+        success &= load(device, age_tensor, group, "age");
+        normalizer.age = get(device, age_tensor, 0);
+        free(device, age_tensor);
+        return success;
+    }
+}
+RL_TOOLS_NAMESPACE_WRAPPER_END
+#endif

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/components/running_normalizer/running_normalizer.h

@@ -0,0 +1,33 @@
+#include "../../../version.h"
+#if (defined(RL_TOOLS_DISABLE_INCLUDE_GUARDS) || !defined(RL_TOOLS_RL_COMPONENTS_RUNNING_NORMALIZER_RUNNING_NORMALIZER_H)) && (RL_TOOLS_USE_THIS_VERSION == 1)
+#pragma once
+#define RL_TOOLS_RL_COMPONENTS_RUNNING_NORMALIZER_RUNNING_NORMALIZER_H
+
+RL_TOOLS_NAMESPACE_WRAPPER_START
+namespace rl_tools::rl::components{
+    namespace running_normalizer{
+        template <typename T_TYPE_POLICY, typename T_TI, T_TI T_DIM, bool T_DYNAMIC_ALLOCATION=true>
+        struct Specification{
+            using TYPE_POLICY = T_TYPE_POLICY;
+            using TI = T_TI;
+            static constexpr TI DIM = T_DIM;
+            static constexpr bool DYNAMIC_ALLOCATION = T_DYNAMIC_ALLOCATION;
+        };
+    }
+    template <typename T_SPEC>
+    struct RunningNormalizer{
+        using SPEC = T_SPEC;
+        using TYPE_POLICY = typename SPEC::TYPE_POLICY;
+        using TI = typename SPEC::TI;
+        static constexpr TI DIM = SPEC::DIM;
+        static constexpr bool DYNAMIC_ALLOCATION = SPEC::DYNAMIC_ALLOCATION;
+
+        using T = typename TYPE_POLICY::DEFAULT;
+        Matrix<matrix::Specification<T, TI, 1, DIM, DYNAMIC_ALLOCATION>> mean;
+        Matrix<matrix::Specification<T, TI, 1, DIM, DYNAMIC_ALLOCATION>> std;
+        TI age = 0;
+    };
+}
+RL_TOOLS_NAMESPACE_WRAPPER_END
+
+#endif

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/environment_wrappers/operations_generic.h

@@ -0,0 +1,58 @@
+#include "../../version.h"
+#if (defined(RL_TOOLS_DISABLE_INCLUDE_GUARDS) || !defined(RL_TOOLS_RL_ENVIRONMENT_WRAPPERS_OPERATIONS_GENERIC_H)) && (RL_TOOLS_USE_THIS_VERSION == 1)
+#pragma once
+#define RL_TOOLS_RL_ENVIRONMENT_WRAPPERS_OPERATIONS_GENERIC_H
+#include "wrappers.h"
+
+RL_TOOLS_NAMESPACE_WRAPPER_START
+namespace rl_tools{
+    template<typename DEVICE, typename ENVIRONMENT>
+    RL_TOOLS_FUNCTION_PLACEMENT static void malloc(DEVICE& device, rl::environment_wrappers::Wrapper<ENVIRONMENT>& env){
+        malloc(device, env.env);
+    }
+    template<typename DEVICE, typename ENVIRONMENT>
+    RL_TOOLS_FUNCTION_PLACEMENT static void free(DEVICE& device, rl::environment_wrappers::Wrapper<ENVIRONMENT>& env){
+        free(device, env.env);
+    }
+    template<typename DEVICE, typename ENVIRONMENT>
+    RL_TOOLS_FUNCTION_PLACEMENT static void init(DEVICE& device, rl::environment_wrappers::Wrapper<ENVIRONMENT>& env){
+        init(device, env.env);
+    }
+    template<typename DEVICE, typename ENVIRONMENT, typename RNG>
+    RL_TOOLS_FUNCTION_PLACEMENT static void sample_initial_parameters(DEVICE& device, const rl::environment_wrappers::Wrapper<ENVIRONMENT>& env, typename rl::environment_wrappers::Wrapper<ENVIRONMENT>::Parameters& parameters, RNG& rng){
+        sample_initial_parameters(device, env.env, parameters, rng);
+    }
+    template<typename DEVICE, typename ENVIRONMENT>
+    RL_TOOLS_FUNCTION_PLACEMENT static void initial_parameters(DEVICE& device, const rl::environment_wrappers::Wrapper<ENVIRONMENT>& env, typename rl::environment_wrappers::Wrapper<ENVIRONMENT>::Parameters& parameters){
+        initial_parameters(device, env.env, parameters);
+    }
+    template<typename DEVICE, typename ENVIRONMENT, typename RNG>
+    RL_TOOLS_FUNCTION_PLACEMENT static void sample_initial_state(DEVICE& device, const rl::environment_wrappers::Wrapper<ENVIRONMENT>& env, typename rl::environment_wrappers::Wrapper<ENVIRONMENT>::Parameters& parameters, typename rl::environment_wrappers::Wrapper<ENVIRONMENT>::State& state, RNG& rng){
+        sample_initial_state(device, env.env, parameters, state, rng);
+    }
+    template<typename DEVICE, typename ENVIRONMENT>
+    RL_TOOLS_FUNCTION_PLACEMENT static void initial_state(DEVICE& device, const rl::environment_wrappers::Wrapper<ENVIRONMENT>& env, typename rl::environment_wrappers::Wrapper<ENVIRONMENT>::Parameters& parameters, typename rl::environment_wrappers::Wrapper<ENVIRONMENT>::State& state){
+        initial_state(device, env.env, parameters, state);
+    }
+    template<typename DEVICE, typename ENVIRONMENT, typename ACTION_SPEC, typename RNG>
+    RL_TOOLS_FUNCTION_PLACEMENT typename ENVIRONMENT::T step(DEVICE& device, const rl::environment_wrappers::Wrapper<ENVIRONMENT>& env, typename rl::environment_wrappers::Wrapper<ENVIRONMENT>::Parameters& parameters, const typename rl::environment_wrappers::Wrapper<ENVIRONMENT>::State& state, const Matrix<ACTION_SPEC>& action, typename rl::environment_wrappers::Wrapper<ENVIRONMENT>::State& next_state, RNG& rng) {
+        static_assert(ACTION_SPEC::ROWS == 1);
+        return step(device, env.env, parameters, state, action, next_state, rng);
+    }
+    template<typename DEVICE, typename ENVIRONMENT, typename ACTION_SPEC, typename RNG>
+    RL_TOOLS_FUNCTION_PLACEMENT static typename ENVIRONMENT::T reward(DEVICE& device, const rl::environment_wrappers::Wrapper<ENVIRONMENT>& env, typename rl::environment_wrappers::Wrapper<ENVIRONMENT>::Parameters& parameters, const typename rl::environment_wrappers::Wrapper<ENVIRONMENT>::State& state, const Matrix<ACTION_SPEC>& action, const typename rl::environment_wrappers::Wrapper<ENVIRONMENT>::State& next_state, RNG& rng){
+        return reward(device, env.env, parameters, state, action, next_state, rng);
+    }
+
+    template<typename DEVICE, typename ENVIRONMENT, typename OBS_TYPE, typename OBS_SPEC, typename RNG>
+    RL_TOOLS_FUNCTION_PLACEMENT static void observe(DEVICE& device, const rl::environment_wrappers::Wrapper<ENVIRONMENT>& env, const typename rl::environment_wrappers::Wrapper<ENVIRONMENT>::Parameters& parameters, const typename rl::environment_wrappers::Wrapper<ENVIRONMENT>::State& state, OBS_TYPE& obs_type, Matrix<OBS_SPEC>& observation, RNG& rng){
+        static_assert(OBS_SPEC::ROWS == 1);
+        observe(device, env.env, parameters, state, obs_type, observation, rng);
+    }
+    template<typename DEVICE, typename ENVIRONMENT, typename RNG>
+    RL_TOOLS_FUNCTION_PLACEMENT static bool terminated(DEVICE& device, const rl::environment_wrappers::Wrapper<ENVIRONMENT>& env, typename rl::environment_wrappers::Wrapper<ENVIRONMENT>::Parameters& parameters, const typename rl::environment_wrappers::Wrapper<ENVIRONMENT>::State state, RNG& rng){
+        return terminated(device, env.env, parameters, state, rng);
+    }
+}
+RL_TOOLS_NAMESPACE_WRAPPER_END
+#endif

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/environment_wrappers/scale_observations/operations_generic.h

@@ -0,0 +1,28 @@
+#include "../../../version.h"
+#if (defined(RL_TOOLS_DISABLE_INCLUDE_GUARDS) || !defined(RL_TOOLS_RL_ENVIRONMENT_WRAPPERS_SCALE_OBSERVATIONS_OPERATIONS_GENERIC_H)) && (RL_TOOLS_USE_THIS_VERSION == 1)
+#pragma once
+#define RL_TOOLS_RL_ENVIRONMENT_WRAPPERS_SCALE_OBSERVATIONS_OPERATIONS_GENERIC_H
+
+#include "../../../math/operations_generic.h"
+#include "wrapper.h"
+#include "../operations_generic.h"
+
+RL_TOOLS_NAMESPACE_WRAPPER_START
+namespace rl_tools{
+    template<typename DEVICE, typename SPEC, typename ENVIRONMENT, typename OBS_TYPE, typename OBS_SPEC, typename RNG>
+    RL_TOOLS_FUNCTION_PLACEMENT static void observe(DEVICE& device, const rl::environment_wrappers::ScaleObservations<SPEC, ENVIRONMENT>& env, const typename rl::environment_wrappers::ScaleObservations<SPEC, ENVIRONMENT>::Parameters& parameters, const typename rl::environment_wrappers::ScaleObservations<SPEC, ENVIRONMENT>::State& state, const OBS_TYPE& obs_type, Matrix<OBS_SPEC>& observation, RNG& rng){
+        using T = typename SPEC::TYPE_POLICY::DEFAULT;
+        using TI = typename DEVICE::index_t;
+        static_assert(OBS_SPEC::ROWS == 1);
+        observe(device, env.env, parameters, state, obs_type, observation, rng);
+        for(TI observation_i = 0; observation_i < OBS_SPEC::COLS; observation_i++){
+            T observation_value = get(observation, 0, observation_i);
+            set(observation, 0, observation_i, observation_value * SPEC::SCALE);
+        }
+    }
+}
+
+RL_TOOLS_NAMESPACE_WRAPPER_END
+
+#endif
+

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/environment_wrappers/scale_observations/wrapper.h

@@ -0,0 +1,26 @@
+#include "../../../version.h"
+#if (defined(RL_TOOLS_DISABLE_INCLUDE_GUARDS) || !defined(RL_TOOLS_RL_ENVIRONMENT_WRAPPERS_SCALE_OBSERVATIONS_WRAPPER_H)) && (RL_TOOLS_USE_THIS_VERSION == 1)
+#pragma once
+#define RL_TOOLS_RL_ENVIRONMENT_WRAPPERS_SCALE_OBSERVATIONS_WRAPPER_H
+
+#include "../wrappers.h"
+
+RL_TOOLS_NAMESPACE_WRAPPER_START
+namespace rl_tools::rl::environment_wrappers::scale_observations {
+    template <typename T_TYPE_POLICY, typename T_TI>
+    struct Specification{
+        using TYPE_POLICY = T_TYPE_POLICY;
+        using TI = T_TI;
+        static constexpr typename TYPE_POLICY::DEFAULT SCALE = 1;
+    };
+}
+namespace rl_tools::rl::environment_wrappers{
+    template <typename T_SPEC, typename T_ENVIRONMENT>
+    struct ScaleObservations: Wrapper<T_ENVIRONMENT>{
+        using SPEC = T_SPEC;
+    };
+}
+RL_TOOLS_NAMESPACE_WRAPPER_END
+
+#endif
+

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/environment_wrappers/wrappers.h

@@ -0,0 +1,29 @@
+#include "../../version.h"
+#if (defined(RL_TOOLS_DISABLE_INCLUDE_GUARDS) || !defined(RL_TOOLS_RL_ENVIRONMENT_WRAPPERS_WRAPPER_H)) && (RL_TOOLS_USE_THIS_VERSION == 1)
+#pragma once
+#define RL_TOOLS_RL_ENVIRONMENT_WRAPPERS_WRAPPER_H
+
+RL_TOOLS_NAMESPACE_WRAPPER_START
+namespace rl_tools::rl::environment_wrappers {
+    template <typename T_ENVIRONMENT>
+    struct Wrapper{
+        using ENVIRONMENT = T_ENVIRONMENT;
+        using T = typename ENVIRONMENT::T;
+        using TI = typename ENVIRONMENT::TI;
+        using Observation = typename ENVIRONMENT::Observation;
+        using ObservationPrivileged = typename ENVIRONMENT::ObservationPrivileged;
+        static constexpr TI N_AGENTS = ENVIRONMENT::N_AGENTS;
+        static constexpr TI ACTION_DIM = ENVIRONMENT::ACTION_DIM;
+        static constexpr TI EPISODE_STEP_LIMIT = ENVIRONMENT::EPISODE_STEP_LIMIT;
+        using Parameters = typename ENVIRONMENT::Parameters;
+        using State = typename ENVIRONMENT::State;
+
+        ENVIRONMENT env;
+    };
+}
+RL_TOOLS_NAMESPACE_WRAPPER_END
+
+#include "scale_observations/wrapper.h"
+
+#endif
+

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/environments/acrobot/acrobot.h

@@ -0,0 +1,88 @@
+#include "../../../version.h"
+#if (defined(RL_TOOLS_DISABLE_INCLUDE_GUARDS) || !defined(RL_TOOLS_RL_ENVIRONMENTS_ACROBOT_ACROBOT_H)) && (RL_TOOLS_USE_THIS_VERSION == 1)
+#pragma once
+#define RL_TOOLS_RL_ENVIRONMENTS_ACROBOT_ACROBOT_H
+
+#include "../environments.h"
+#include "../observation.h"
+
+RL_TOOLS_NAMESPACE_WRAPPER_START
+namespace rl_tools::rl::environments::acrobot {
+    template <typename T>
+    struct DefaultParameters {
+        static constexpr T DT = 0.2;
+        static constexpr T LINK_LENGTH_1 = 1.0;  // [m]
+        static constexpr T LINK_LENGTH_2 = 1.0;  // [m]
+        static constexpr T LINK_MASS_1 = 1.0;  //: [kg] mass of link 1
+        static constexpr T LINK_MASS_2 = 1.0;  //: [kg] mass of link 2
+        static constexpr T LINK_COM_POS_1 = 0.5;  //: [m] position of the center of mass of link 1
+        static constexpr T LINK_COM_POS_2 = 0.5;  //: [m] position of the center of mass of link 2
+        static constexpr T LINK_MOI = 1.0;  //: moments of inertia for both links
+        static constexpr T MAX_VEL_1 = 4 * math::PI<T>;
+        static constexpr T MAX_VEL_2 = 9 * math::PI<T>;
+        static constexpr T MIN_TORQUE = -5;
+        static constexpr T MAX_TORQUE = +5;
+        static constexpr T TORQUE_NOISE_MAX = 0.0;
+        static constexpr T SCREEN_DIM = 500;
+        static constexpr T VEL_PENALTY = 0.2;
+    };
+    template <typename T>
+    struct EasyParameters: DefaultParameters<T> {
+        static constexpr T MIN_TORQUE = -10;
+        static constexpr T MAX_TORQUE = +10;
+    };
+    template <typename T>
+    struct Parameters{};
+    template <typename T_T, typename T_TI, typename T_PARAMETERS = DefaultParameters<T_T>>
+    struct Specification{
+        using T = T_T;
+        using TI = T_TI;
+        using PARAMETERS = T_PARAMETERS;
+    };
+
+    template <typename T, typename TI>
+    struct State{
+        static constexpr TI DIM = 4;
+        T theta_1;
+        T theta_2;
+        T theta_1_dot;
+        T theta_2_dot;
+    };
+
+    template <typename TI>
+    struct Observation{
+        static constexpr TI DIM = 6;
+        using SHAPE = tensor::Shape<TI, DIM>;
+    };
+
+}
+RL_TOOLS_NAMESPACE_WRAPPER_END
+
+RL_TOOLS_NAMESPACE_WRAPPER_START
+namespace rl_tools::rl::environments{
+
+    template <typename T_SPEC>
+    struct Acrobot: Environment<typename T_SPEC::T, typename T_SPEC::TI> {
+        using SPEC = T_SPEC;
+        using T = typename SPEC::T;
+        using TI = typename SPEC::TI;
+        using State = acrobot::State<T, TI>;
+        using Parameters = typename SPEC::PARAMETERS;
+        using Observation = acrobot::Observation<TI>;
+        using ObservationPrivileged = Observation;
+        static constexpr TI ACTION_DIM = 1;
+        acrobot::Parameters<T> parameters;
+        static constexpr TI EPISODE_STEP_LIMIT = 500;
+    };
+    template <typename T_SPEC>
+    struct AcrobotSwingup: Acrobot<T_SPEC>{};
+}
+RL_TOOLS_NAMESPACE_WRAPPER_END
+
+
+
+
+
+
+
+#endif