hyperdrone 0.1.0__py3-none-any.whl

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/algorithms/ppo/operations_generic_extensions.h

@@ -0,0 +1,244 @@
+#include "../../../version.h"
+#if (defined(RL_TOOLS_DISABLE_INCLUDE_GUARDS) || !defined(RL_TOOLS_RL_ALGORITHMS_PPO_OPERATIONS_GENERIC_EXTENSIONS_H)) && (RL_TOOLS_USE_THIS_VERSION == 1)
+#pragma once
+#define RL_TOOLS_RL_ALGORITHMS_PPO_OPERATIONS_GENERIC_EXTENSIONS_H
+
+#include "ppo.h"
+#include "../../../rl/components/on_policy_runner/on_policy_runner.h"
+
+RL_TOOLS_NAMESPACE_WRAPPER_START
+namespace rl_tools{
+    namespace rl::algorithms::ppo{
+
+        template <typename PPO_SPEC>
+        struct TrainingBuffersHybrid{
+            using SPEC = PPO_SPEC;
+            using T = typename SPEC::T;
+            using TI = typename SPEC::TI;
+            static constexpr TI BATCH_SIZE = SPEC::BATCH_SIZE;
+            static constexpr TI ACTION_DIM = SPEC::ENVIRONMENT::ACTION_DIM;
+            static constexpr TI OBSERVATION_DIM = SPEC::ENVIRONMENT::Observation::DIM;
+            typename SPEC::CONTAINER_TYPE_TAG::template type<matrix::Specification<T, TI, BATCH_SIZE, ACTION_DIM>> actions;
+            typename SPEC::CONTAINER_TYPE_TAG::template type<matrix::Specification<T, TI, BATCH_SIZE, OBSERVATION_DIM>> observations;
+            typename SPEC::CONTAINER_TYPE_TAG::template type<matrix::Specification<T, TI, BATCH_SIZE, ACTION_DIM>> d_action_log_prob_d_action;
+            typename SPEC::CONTAINER_TYPE_TAG::template type<matrix::Specification<T, TI, BATCH_SIZE, 1>> target_values;
+            typename SPEC::CONTAINER_TYPE_TAG::template type<matrix::Specification<T, TI, BATCH_SIZE, 1>> d_critic_output;
+        };
+    }
+    template <typename DEVICE, typename SPEC>
+    void malloc(DEVICE& device, rl::algorithms::ppo::TrainingBuffersHybrid<SPEC>& buffers){
+        malloc(device, buffers.actions);
+        malloc(device, buffers.observations);
+        malloc(device, buffers.d_action_log_prob_d_action);
+        malloc(device, buffers.target_values);
+        malloc(device, buffers.d_critic_output);
+    }
+    template <typename DEVICE, typename SPEC>
+    void free(DEVICE& device, rl::algorithms::ppo::TrainingBuffersHybrid<SPEC>& buffers){
+        free(device, buffers.actions);
+        free(device, buffers.observations);
+        free(device, buffers.d_action_log_prob_d_action);
+        free(device, buffers.target_values);
+        free(device, buffers.d_critic_output);
+    }
+    template <typename DEVICE, typename DEVICE_EVALUATION, typename PPO_SPEC, typename OPR_SPEC, auto STEPS_PER_ENV, typename ACTOR_OPTIMIZER, typename CRITIC_OPTIMIZER, typename RNG>
+    void train_hybrid(DEVICE& device,
+        DEVICE_EVALUATION& device_evaluation,
+        rl::algorithms::PPO<PPO_SPEC>& ppo,
+        rl::algorithms::PPO<PPO_SPEC>& ppo_evaluation,
+        rl::components::on_policy_runner::Dataset<rl::components::on_policy_runner::DatasetSpecification<OPR_SPEC, STEPS_PER_ENV>>& dataset,
+        ACTOR_OPTIMIZER& actor_optimizer,
+        CRITIC_OPTIMIZER& critic_optimizer,
+        rl::algorithms::ppo::Buffers<PPO_SPEC>& ppo_buffers,
+        rl::algorithms::ppo::TrainingBuffersHybrid<PPO_SPEC>& hybrid_buffers,
+        typename PPO_SPEC::ACTOR_TYPE::template Buffer<PPO_SPEC::BATCH_SIZE>& actor_buffers,
+        typename PPO_SPEC::CRITIC_TYPE::template Buffer<PPO_SPEC::BATCH_SIZE>& critic_buffers,
+        RNG& rng){
+#ifdef RL_TOOLS_DEBUG_RL_ALGORITHMS_PPO_CHECK_INIT
+        utils::assert_exit(device, ppo.initialized, "PPO not initialized");
+#endif
+        using T = typename PPO_SPEC::T;
+        using TI = typename PPO_SPEC::TI;
+        static_assert(utils::typing::is_same_v<typename PPO_SPEC::ENVIRONMENT, typename OPR_SPEC::ENVIRONMENT>, "environment mismatch");
+        using BUFFER = rl::components::on_policy_runner::Dataset<rl::components::on_policy_runner::DatasetSpecification<OPR_SPEC, STEPS_PER_ENV>>;
+        static_assert(BUFFER::STEPS_TOTAL > 0);
+        constexpr TI N_EPOCHS = PPO_SPEC::PARAMETERS::N_EPOCHS;
+        constexpr TI BATCH_SIZE = PPO_SPEC::BATCH_SIZE;
+        constexpr TI N_BATCHES = BUFFER::STEPS_TOTAL/BATCH_SIZE;
+        static_assert(N_BATCHES > 0);
+        constexpr TI ACTION_DIM = OPR_SPEC::ENVIRONMENT::ACTION_DIM;
+        constexpr TI OBSERVATION_DIM = OPR_SPEC::ENVIRONMENT::Observation::DIM;
+        constexpr bool NORMALIZE_OBSERVATIONS = PPO_SPEC::PARAMETERS::NORMALIZE_OBSERVATIONS;
+//        auto all_observations = NORMALIZE_OBSERVATIONS ? dataset.all_observations_normalized : dataset.all_observations;
+//        auto observations = NORMALIZE_OBSERVATIONS ? dataset.observations_normalized : dataset.observations;
+        // batch needs observations, original log-probs, advantages
+        T policy_kl_divergence = 0; // KL( current || old ) todo: make hyperparameter that swaps the order
+        if(PPO_SPEC::PARAMETERS::ADAPTIVE_LEARNING_RATE) {
+            auto& last_layer = get_last_layer(ppo.actor);
+            copy(device, device, last_layer.log_std.parameters, ppo_buffers.rollout_log_std);
+        }
+        for(TI epoch_i = 0; epoch_i < N_EPOCHS; epoch_i++){
+            // shuffling
+            for(TI dataset_i = 0; dataset_i < BUFFER::STEPS_TOTAL; dataset_i++){
+                TI sample_index = random::uniform_int_distribution(typename DEVICE::SPEC::RANDOM(), dataset_i, BUFFER::STEPS_TOTAL-1, rng);
+                {
+                    auto target_row = row(device, dataset.observations, dataset_i);
+                    auto source_row = row(device, dataset.observations, sample_index);
+                    swap(device, target_row, source_row);
+                }
+                if(PPO_SPEC::PARAMETERS::ADAPTIVE_LEARNING_RATE){
+                    auto target_row = row(device, dataset.actions_mean, dataset_i);
+                    auto source_row = row(device, dataset.actions_mean, sample_index);
+                    swap(device, target_row, source_row);
+                }
+                {
+                    auto target_row = row(device, dataset.actions, dataset_i);
+                    auto source_row = row(device, dataset.actions, sample_index);
+                    swap(device, target_row, source_row);
+                }
+                swap(device, dataset.advantages      , dataset.advantages      , dataset_i, 0, sample_index, 0);
+                swap(device, dataset.action_log_probs, dataset.action_log_probs, dataset_i, 0, sample_index, 0);
+                swap(device, dataset.target_values   , dataset.target_values   , dataset_i, 0, sample_index, 0);
+            }
+            for(TI batch_i = 0; batch_i < N_BATCHES; batch_i++){
+                T batch_policy_kl_divergence = 0; // KL( current || old ) todo: make hyperparameter that swaps the order
+                zero_gradient(device_evaluation, ppo_evaluation.critic);
+                zero_gradient(device_evaluation, ppo_evaluation.actor); // has to be reset before accumulating the action-log-std gradient
+
+                auto batch_offset = batch_i * BATCH_SIZE;
+                auto batch_observations     = view(device, dataset.observations    , matrix::ViewSpec<BATCH_SIZE, OBSERVATION_DIM>(), batch_offset, 0);
+                auto batch_actions_mean     = view(device, dataset.actions_mean    , matrix::ViewSpec<BATCH_SIZE, ACTION_DIM     >(), batch_offset, 0);
+                auto batch_actions          = view(device, dataset.actions         , matrix::ViewSpec<BATCH_SIZE, ACTION_DIM     >(), batch_offset, 0);
+                auto batch_action_log_probs = view(device, dataset.action_log_probs, matrix::ViewSpec<BATCH_SIZE, 1              >(), batch_offset, 0);
+                auto batch_advantages       = view(device, dataset.advantages      , matrix::ViewSpec<BATCH_SIZE, 1              >(), batch_offset, 0);
+                auto batch_target_values    = view(device, dataset.target_values   , matrix::ViewSpec<BATCH_SIZE, 1              >(), batch_offset, 0);
+
+                T advantage_mean = 0;
+                T advantage_std = 0;
+                for(TI batch_step_i = 0; batch_step_i < BATCH_SIZE; batch_step_i++){
+                    T advantage = get(batch_advantages, batch_step_i, 0);
+                    advantage_mean += advantage;
+                    advantage_std += advantage * advantage;
+                }
+                advantage_mean /= BATCH_SIZE;
+                advantage_std /= BATCH_SIZE;
+
+                advantage_std = math::sqrt(device.math, advantage_std - advantage_mean * advantage_mean);
+//                add_scalar(device, device.logger, "ppo/advantage/mean", advantage_mean);
+//                add_scalar(device, device.logger, "ppo/advantage/std", advantage_std);
+
+                copy(device, device_evaluation, batch_observations, hybrid_buffers.observations);
+                forward(device_evaluation, ppo_evaluation.actor, hybrid_buffers.observations, hybrid_buffers.actions, actor_buffers, rng);
+                copy(device_evaluation, device, hybrid_buffers.actions, ppo_buffers.current_batch_actions);
+//                auto abs_diff = abs_diff(device, batch_actions, buffer.actions);
+
+                auto& last_layer = get_last_layer(ppo.actor);
+                auto& last_layer_eval = get_last_layer(ppo_evaluation.actor);
+                copy(device_evaluation, device, last_layer_eval.log_std.parameters, last_layer.log_std.parameters);
+                copy(device_evaluation, device, last_layer_eval.log_std.gradient, last_layer.log_std.gradient);
+                for(TI batch_step_i = 0; batch_step_i < BATCH_SIZE; batch_step_i++){
+                    T action_log_prob = 0;
+                    for(TI action_i = 0; action_i < ACTION_DIM; action_i++){
+
+                        T current_action = get(ppo_buffers.current_batch_actions, batch_step_i, action_i);
+                        T rollout_action = get(batch_actions, batch_step_i, action_i);
+                        auto& last_layer = get_last_layer(ppo.actor);
+                        T current_action_log_std = get(last_layer.log_std.parameters, 0, action_i);
+                        T current_action_std = math::exp(device.math, current_action_log_std);
+                        if(PPO_SPEC::PARAMETERS::ADAPTIVE_LEARNING_RATE){
+                            T rollout_action_log_std = get(ppo_buffers.rollout_log_std, 0, action_i);
+                            T rollout_action_std = math::exp(device.math, rollout_action_log_std);
+                            T rollout_action_mean = get(batch_actions_mean, batch_step_i, action_i);
+                            T action_mean_diff = rollout_action_mean - current_action;
+                            T kl = rollout_action_log_std - current_action_log_std;
+                            kl += (current_action_std * current_action_std + action_mean_diff * action_mean_diff)/(2 * rollout_action_std * rollout_action_std + PPO_SPEC::PARAMETERS::POLICY_KL_EPSILON);
+                            kl += (T)-0.5;
+                            kl = math::max(device.math, kl, (T)0);
+                            policy_kl_divergence += kl;
+                            batch_policy_kl_divergence += kl;
+                        }
+
+                        T action_diff_by_action_std = (current_action - rollout_action) / current_action_std;
+                        action_log_prob += -0.5 * action_diff_by_action_std * action_diff_by_action_std - current_action_log_std - 0.5 * math::log(device.math, 2 * math::PI<T>);
+                        set(ppo_buffers.d_action_log_prob_d_action, batch_step_i, action_i, - action_diff_by_action_std / current_action_std);
+                        T current_entropy = current_action_log_std + math::log(device.math, 2 * math::PI<T>)/(T)2 + (T)1/(T)2;
+                        T current_entropy_loss = -(T)1/BATCH_SIZE * PPO_SPEC::PARAMETERS::ACTION_ENTROPY_COEFFICIENT * current_entropy;
+                        // todo: think about possible implementation detail: clipping entropy bonus as well (because it changes the distribution)
+                        if(PPO_SPEC::PARAMETERS::LEARN_ACTION_STD){
+                            T d_entropy_loss_d_current_action_log_std = -(T)1/BATCH_SIZE * PPO_SPEC::PARAMETERS::ACTION_ENTROPY_COEFFICIENT;
+                            auto& last_layer = get_last_layer(ppo.actor);
+                            increment(last_layer.log_std.gradient, 0, action_i, d_entropy_loss_d_current_action_log_std);
+//                          derivation: d_current_action_log_prob_d_action_log_std
+//                          d_current_action_log_prob_d_action_std =  (-action_diff_by_action_std) * (-action_diff_by_action_std)      / action_std - 1 / action_std)
+//                          d_current_action_log_prob_d_action_std = ((-action_diff_by_action_std) * (-action_diff_by_action_std) - 1) / action_std)
+//                          d_current_action_log_prob_d_action_std = (action_diff_by_action_std * action_diff_by_action_std - 1) / action_std
+//                          d_current_action_log_prob_d_action_log_std = (action_diff_by_action_std * action_diff_by_action_std - 1) / action_std * exp(action_log_std)
+//                          d_current_action_log_prob_d_action_log_std = (action_diff_by_action_std * action_diff_by_action_std - 1) / action_std * action_std
+//                          d_current_action_log_prob_d_action_log_std =  action_diff_by_action_std * action_diff_by_action_std - 1
+                            T d_current_action_log_prob_d_action_log_std = action_diff_by_action_std * action_diff_by_action_std - 1;
+                            set(ppo_buffers.d_action_log_prob_d_action_log_std, batch_step_i, action_i, d_current_action_log_prob_d_action_log_std);
+                        }
+                    }
+                    T rollout_action_log_prob = get(batch_action_log_probs, batch_step_i, 0);
+                    T advantage = get(batch_advantages, batch_step_i, 0);
+                    if(PPO_SPEC::PARAMETERS::NORMALIZE_ADVANTAGE){
+                        advantage = (advantage - advantage_mean) / (advantage_std + PPO_SPEC::PARAMETERS::ADVANTAGE_EPSILON);
+                    }
+                    T log_ratio = action_log_prob - rollout_action_log_prob;
+                    T ratio = math::exp(device.math, log_ratio);
+                    // todo: test relative clipping (clipping in log space makes more sense thatn clipping in exp space)
+                    T clipped_ratio = math::clamp(device.math, ratio, 1 - PPO_SPEC::PARAMETERS::EPSILON_CLIP, 1 + PPO_SPEC::PARAMETERS::EPSILON_CLIP);
+                    T normal_advantage = ratio * advantage;
+                    T clipped_advantage = clipped_ratio * advantage;
+                    T slippage = 0.0;
+                    bool ratio_min_switch = normal_advantage - clipped_advantage <= slippage;
+                    T clipped_surrogate = ratio_min_switch ? normal_advantage : clipped_advantage;
+
+                    T d_loss_d_clipped_surrogate = -(T)1/BATCH_SIZE;
+                    T d_clipped_surrogate_d_ratio = ratio_min_switch ? advantage : 0;
+                    T d_ratio_d_action_log_prob = ratio;
+                    T d_loss_d_action_log_prob = d_loss_d_clipped_surrogate * d_clipped_surrogate_d_ratio * d_ratio_d_action_log_prob;
+                    for(TI action_i = 0; action_i < ACTION_DIM; action_i++){
+                        multiply(ppo_buffers.d_action_log_prob_d_action, batch_step_i, action_i, d_loss_d_action_log_prob);
+                        if(PPO_SPEC::PARAMETERS::LEARN_ACTION_STD){
+                            T current_d_action_log_prob_d_action_log_std = get(ppo_buffers.d_action_log_prob_d_action_log_std, batch_step_i, action_i);
+                            auto& last_layer = get_last_layer(ppo.actor);
+                            increment(last_layer.log_std.gradient, 0, action_i, d_loss_d_action_log_prob * current_d_action_log_prob_d_action_log_std);
+                        }
+                    }
+                }
+                if(PPO_SPEC::PARAMETERS::ADAPTIVE_LEARNING_RATE){
+                    batch_policy_kl_divergence /= BATCH_SIZE;
+                    auto& actor_optimizer_parameters = get_ref(device, actor_optimizer.parameters, 0);
+                    if(batch_policy_kl_divergence > 2 * PPO_SPEC::PARAMETERS::ADAPTIVE_LEARNING_RATE_POLICY_KL_THRESHOLD){
+                        actor_optimizer_parameters.alpha = math::max(device.math, actor_optimizer_parameters.alpha * PPO_SPEC::PARAMETERS::ADAPTIVE_LEARNING_RATE_DECAY, PPO_SPEC::PARAMETERS::ADAPTIVE_LEARNING_RATE_MIN);
+                    }
+                    if(batch_policy_kl_divergence < 0.5 * PPO_SPEC::PARAMETERS::ADAPTIVE_LEARNING_RATE_POLICY_KL_THRESHOLD){
+                        actor_optimizer_parameters.alpha = math::min(device.math, actor_optimizer_parameters.alpha / PPO_SPEC::PARAMETERS::ADAPTIVE_LEARNING_RATE_DECAY, PPO_SPEC::PARAMETERS::ADAPTIVE_LEARNING_RATE_MAX);
+                    }
+                }
+                copy(device, device_evaluation, last_layer.log_std.parameters, last_layer_eval.log_std.parameters);
+                copy(device, device_evaluation, last_layer.log_std.gradient, last_layer_eval.log_std.gradient);
+
+                copy(device, device_evaluation, ppo_buffers.d_action_log_prob_d_action, hybrid_buffers.d_action_log_prob_d_action);
+                backward(device_evaluation, ppo_evaluation.actor, hybrid_buffers.observations, hybrid_buffers.d_action_log_prob_d_action, actor_buffers);
+                copy(device, device_evaluation, batch_target_values, hybrid_buffers.target_values);
+//                forward_backward_mse(device_evaluation, ppo_evaluation.critic, hybrid_buffers.observations, hybrid_buffers.target_values, critic_buffers);
+                {
+                    forward(device_evaluation, ppo_evaluation.critic, hybrid_buffers.observations, critic_buffers, rng);
+                    nn::loss_functions::mse::gradient(device_evaluation, output(ppo_evaluation.critic), hybrid_buffers.target_values, hybrid_buffers.d_critic_output);
+                    backward(device_evaluation, ppo_evaluation.critic, hybrid_buffers.observations, hybrid_buffers.d_critic_output, critic_buffers);
+                }
+                step(device_evaluation, actor_optimizer, ppo_evaluation.actor);
+                step(device_evaluation, critic_optimizer, ppo_evaluation.critic);
+            }
+        }
+        if(PPO_SPEC::PARAMETERS::ADAPTIVE_LEARNING_RATE) {
+            policy_kl_divergence /= N_EPOCHS * N_BATCHES * BATCH_SIZE;
+            add_scalar(device, device.logger, "ppo/policy_kl", policy_kl_divergence);
+        }
+    }
+
+}
+RL_TOOLS_NAMESPACE_WRAPPER_END
+#endif

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/algorithms/ppo/persist.h

@@ -0,0 +1,28 @@
+#include "../../../version.h"
+#if (defined(RL_TOOLS_DISABLE_INCLUDE_GUARDS) || !defined(RL_TOOLS_RL_ALGORITHMS_PPO_PERSIST_H)) && (RL_TOOLS_USE_THIS_VERSION == 1)
+#pragma once
+#define RL_TOOLS_RL_ALGORITHMS_PPO_PERSIST_H
+#include "ppo.h"
+RL_TOOLS_NAMESPACE_WRAPPER_START
+namespace rl_tools{
+    template<typename DEVICE, typename SPEC, typename GROUP>
+    void save(DEVICE& device, rl::algorithms::PPO<SPEC>& ppo, GROUP& group) {
+        auto actor_group = create_group(device, group, "actor");
+        save(device, ppo.actor, actor_group);
+        auto critic_group = create_group(device, group, "critic");
+        save(device, ppo.critic, critic_group);
+    }
+    template<typename DEVICE, typename SPEC, typename GROUP>
+    bool load(DEVICE& device, rl::algorithms::PPO<SPEC>& ppo, GROUP& group) {
+        auto actor_group = get_group(device, group, "actor");
+        bool success = load(device, ppo.actor, actor_group);
+        auto critic_group = get_group(device, group, "critic");
+        success &= load(device, ppo.critic, critic_group);
+#ifdef RL_TOOLS_DEBUG_RL_ALGORITHMS_PPO_CHECK_INIT
+        ppo.initialized = true;
+#endif
+        return success;
+    }
+}
+RL_TOOLS_NAMESPACE_WRAPPER_END
+#endif

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/algorithms/ppo/ppo.h

@@ -0,0 +1,105 @@
+#include "../../../version.h"
+#if (defined(RL_TOOLS_DISABLE_INCLUDE_GUARDS) || !defined(RL_TOOLS_RL_ALGORITHMS_PPO_PPO_H)) && (RL_TOOLS_USE_THIS_VERSION == 1)
+#pragma once
+#define RL_TOOLS_RL_ALGORITHMS_PPO_PPO_H
+
+#include "../../../rl/components/running_normalizer/running_normalizer.h"
+#include "../../../utils/generic/typing.h"
+
+
+RL_TOOLS_NAMESPACE_WRAPPER_START
+namespace rl_tools::rl::algorithms{
+    namespace ppo{
+        template<typename TYPE_POLICY, typename TI, TI T_BATCH_SIZE>
+        struct DefaultParameters {
+            using T = typename TYPE_POLICY::DEFAULT;
+            static constexpr T GAMMA = 0.99;
+            static constexpr T LAMBDA = 0.95;
+            static constexpr T EPSILON_CLIP = 0.2;
+            static constexpr T INITIAL_ACTION_STD = 0.5; // note this is NOT log(std) but actual std (log is applied at init)
+            static constexpr bool LEARN_ACTION_STD = true;
+            static constexpr T ACTION_ENTROPY_COEFFICIENT = 0.01;
+            static constexpr T ADVANTAGE_EPSILON = 1e-8;
+            static constexpr bool NORMALIZE_ADVANTAGE = true;
+            static constexpr bool ADAPTIVE_LEARNING_RATE = false;
+            static constexpr T ADAPTIVE_LEARNING_RATE_POLICY_KL_THRESHOLD = 0.008;
+            static constexpr T POLICY_KL_EPSILON = 1e-5;
+            static constexpr T ADAPTIVE_LEARNING_RATE_DECAY = (T)1/(T)1.5;
+            static constexpr T ADAPTIVE_LEARNING_RATE_MIN = 1e-6;
+            static constexpr T ADAPTIVE_LEARNING_RATE_MAX = 1e-2;
+            static constexpr TI N_WARMUP_STEPS_CRITIC = 0;
+            static constexpr TI N_WARMUP_STEPS_ACTOR = 0;
+            static constexpr TI N_EPOCHS = 10;
+            static constexpr TI BATCH_SIZE = T_BATCH_SIZE;
+            static constexpr bool IGNORE_TERMINATION = false; // ignoring the termination flag is useful for training on environments with negative rewards, where the agent would try to terminate the episode as soon as possible otherwise
+            static constexpr bool SHUFFLE_EPOCH = true;
+            static constexpr bool STATEFUL_ACTOR_AND_CRITIC = false;
+            static constexpr bool TRUNCATE_ON_EACH_ITERATION = false;
+        };
+
+        template<
+                typename T_TYPE_POLICY,
+                typename T_TI,
+                typename T_ENVIRONMENT,
+                typename T_ACTOR_TYPE,
+                typename T_CRITIC_TYPE,
+                typename T_PARAMETERS
+        >
+        struct Specification {
+            using TYPE_POLICY = T_TYPE_POLICY;
+            using TI = T_TI;
+            using ENVIRONMENT = T_ENVIRONMENT;
+            using ACTOR_TYPE = T_ACTOR_TYPE;
+            using CRITIC_TYPE = T_CRITIC_TYPE;
+            using PARAMETERS = T_PARAMETERS;
+            static constexpr bool ASYMMETRIC_OBSERVATIONS = !rl_tools::utils::typing::is_same_v<typename ENVIRONMENT::Observation, typename ENVIRONMENT::ObservationPrivileged>;
+
+            using _OBS_SHAPE = typename ENVIRONMENT::Observation::SHAPE;
+            using _OBS_PRIV_SHAPE = typename ENVIRONMENT::ObservationPrivileged::SHAPE;
+#ifndef RL_TOOLS_DISABLE_VISUAL
+            static_assert(get_last(typename ACTOR_TYPE::INPUT_SHAPE{}) == get_last(_OBS_SHAPE{}));
+#endif
+            static_assert(get_last(typename CRITIC_TYPE::INPUT_SHAPE{}) == get_last(_OBS_PRIV_SHAPE{}));
+            static_assert(get_last(typename ACTOR_TYPE::OUTPUT_SHAPE{}) == ENVIRONMENT::ACTION_DIM);
+            static_assert(get_last(typename CRITIC_TYPE::OUTPUT_SHAPE{}) == 1);
+        };
+
+        template <typename T_SPEC, bool T_DYNAMIC_ALLOCATION=true>
+        struct BufferSpecification{
+            using SPEC = T_SPEC;
+            static constexpr bool DYNAMIC_ALLOCATION = T_DYNAMIC_ALLOCATION;
+        };
+        template <typename T_BUFFER_SPEC>
+        struct Buffers{
+            using BUFFER_SPEC = T_BUFFER_SPEC;
+            using SPEC = typename BUFFER_SPEC::SPEC;
+            using T = typename SPEC::TYPE_POLICY::template GET<numeric_types::categories::Accumulator>;
+            using TI = typename SPEC::TI;
+            static constexpr TI BATCH_SIZE = SPEC::PARAMETERS::BATCH_SIZE;
+            static constexpr TI ACTION_DIM = SPEC::ENVIRONMENT::ACTION_DIM;
+            static constexpr TI OBSERVATION_DIM = SPEC::ENVIRONMENT::Observation::DIM;
+            Matrix<matrix::Specification<T, TI, BATCH_SIZE, ACTION_DIM, BUFFER_SPEC::DYNAMIC_ALLOCATION>> current_batch_actions;
+            Matrix<matrix::Specification<T, TI, BATCH_SIZE, 1, BUFFER_SPEC::DYNAMIC_ALLOCATION>> d_critic_output;
+            Matrix<matrix::Specification<T, TI, BATCH_SIZE, ACTION_DIM, BUFFER_SPEC::DYNAMIC_ALLOCATION>> d_action_log_prob_d_action;
+            Matrix<matrix::Specification<T, TI, BATCH_SIZE, ACTION_DIM, BUFFER_SPEC::DYNAMIC_ALLOCATION>> d_action_log_prob_d_action_log_std;
+            Matrix<matrix::Specification<T, TI, 1, ACTION_DIM/SPEC::ENVIRONMENT::N_AGENTS, BUFFER_SPEC::DYNAMIC_ALLOCATION>> rollout_log_std;
+        };
+    }
+
+    template<typename T_SPEC>
+    struct PPO {
+        using SPEC = T_SPEC;
+        using TYPE_POLICY = typename SPEC::TYPE_POLICY;
+        using TI = typename SPEC::TI;
+
+        typename SPEC::ACTOR_TYPE actor;
+        typename SPEC::CRITIC_TYPE critic;
+#ifdef RL_TOOLS_DEBUG_RL_ALGORITHMS_PPO_CHECK_INIT
+        bool initialized = false;
+#endif
+
+    };
+}
+RL_TOOLS_NAMESPACE_WRAPPER_END
+
+#endif

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/algorithms/sac/loop/core/approximators_gru.h

@@ -0,0 +1,84 @@
+#include "../../../../../version.h"
+#if (defined(RL_TOOLS_DISABLE_INCLUDE_GUARDS) || !defined(RL_TOOLS_RL_ALGORITHMS_SAC_LOOP_CORE_APPROXIMATORS_GRU_H)) && (RL_TOOLS_USE_THIS_VERSION == 1)
+#pragma once
+#define RL_TOOLS_RL_ALGORITHMS_SAC_LOOP_CORE_APPROXIMATORS_GRU_H
+
+RL_TOOLS_NAMESPACE_WRAPPER_START
+namespace rl_tools::rl::algorithms::sac::loop::core{
+    template<typename TYPE_POLICY, typename TI, typename ENVIRONMENT, typename PARAMETERS, bool DYNAMIC_ALLOCATION=true>
+    struct ConfigApproximatorsGRU{
+        static constexpr bool USE_GRU = true;
+        using SAC_PARAMETERS = typename PARAMETERS::SAC_PARAMETERS;
+        template <typename CAPABILITY>
+        struct Actor{
+            using INPUT_SHAPE = tensor::Shape<TI, SAC_PARAMETERS::SEQUENCE_LENGTH, PARAMETERS::SAC_PARAMETERS::ACTOR_BATCH_SIZE, ENVIRONMENT::Observation::DIM>;
+            using INPUT_LAYER_CONFIG = nn::layers::dense::Configuration<TYPE_POLICY, TI, PARAMETERS::ACTOR_HIDDEN_DIM, PARAMETERS::CRITIC_ACTIVATION_FUNCTION, nn::layers::dense::DefaultInitializer<TYPE_POLICY, TI>, nn::parameters::groups::Input>;
+            using INPUT_LAYER = nn::layers::dense::BindConfiguration<INPUT_LAYER_CONFIG>;
+            using GRU_SPEC = nn::layers::gru::Configuration<TYPE_POLICY, TI, PARAMETERS::ACTOR_HIDDEN_DIM, nn::parameters::groups::Normal, true>;
+            using GRU = nn::layers::gru::BindConfiguration<GRU_SPEC>;
+            using GRU2_SPEC = nn::layers::gru::Configuration<TYPE_POLICY, TI, PARAMETERS::ACTOR_HIDDEN_DIM, nn::parameters::groups::Normal, true>;
+            using GRU2 = nn::layers::gru::BindConfiguration<GRU2_SPEC>;
+            using DENSE_LAYER_CONFIG = nn::layers::dense::Configuration<TYPE_POLICY, TI, PARAMETERS::ACTOR_HIDDEN_DIM, PARAMETERS::ACTOR_ACTIVATION_FUNCTION, nn::layers::dense::DefaultInitializer<TYPE_POLICY, TI>, nn::parameters::groups::Normal>;
+            using DENSE_LAYER = nn::layers::dense::BindConfiguration<DENSE_LAYER_CONFIG>;
+            using OUTPUT_LAYER_CONFIG = nn::layers::dense::Configuration<TYPE_POLICY, TI, 2*ENVIRONMENT::ACTION_DIM, nn::activation_functions::ActivationFunction::IDENTITY, nn::layers::dense::DefaultInitializer<TYPE_POLICY, TI>, nn::parameters::groups::Output>;
+            using OUTPUT_LAYER = nn::layers::dense::BindConfiguration<OUTPUT_LAYER_CONFIG>;
+            struct SAMPLE_AND_SQUASH_LAYER_PARAMETERS{
+                using T = typename TYPE_POLICY::DEFAULT;
+                static constexpr T LOG_STD_LOWER_BOUND = SAC_PARAMETERS::LOG_STD_LOWER_BOUND;
+                static constexpr T LOG_STD_UPPER_BOUND = SAC_PARAMETERS::LOG_STD_UPPER_BOUND;
+                static constexpr T LOG_PROBABILITY_EPSILON = SAC_PARAMETERS::LOG_PROBABILITY_EPSILON;
+                static constexpr bool ADAPTIVE_ALPHA = SAC_PARAMETERS::ADAPTIVE_ALPHA;
+                static constexpr bool UPDATE_ALPHA_WITH_ACTOR = false;
+                static constexpr T ALPHA = SAC_PARAMETERS::ALPHA;
+                static constexpr T TARGET_ENTROPY = SAC_PARAMETERS::TARGET_ENTROPY;
+            };
+            using SAMPLE_AND_SQUASH_LAYER_SPEC = nn::layers::sample_and_squash::Configuration<TYPE_POLICY, TI, SAMPLE_AND_SQUASH_LAYER_PARAMETERS>;
+            using SAMPLE_AND_SQUASH_LAYER = nn::layers::sample_and_squash::BindConfiguration<SAMPLE_AND_SQUASH_LAYER_SPEC>;
+            using SAMPLE_AND_SQUASH_MODULE = nn_models::sequential::Module<SAMPLE_AND_SQUASH_LAYER>;
+            using MODULE_GRU = nn_models::sequential::Module<INPUT_LAYER, nn_models::sequential::Module<GRU, nn_models::sequential::Module<OUTPUT_LAYER, SAMPLE_AND_SQUASH_MODULE>>>;
+            using MODULE_GRU_TWO_LAYER = nn_models::sequential::Module<INPUT_LAYER, nn_models::sequential::Module<GRU, nn_models::sequential::Module<GRU2, nn_models::sequential::Module<OUTPUT_LAYER, SAMPLE_AND_SQUASH_MODULE>>>>;
+            using MODULE_GRU_THREE_LAYER = nn_models::sequential::Module<INPUT_LAYER, nn_models::sequential::Module<GRU, nn_models::sequential::Module<GRU2, nn_models::sequential::Module<DENSE_LAYER, nn_models::sequential::Module<OUTPUT_LAYER, SAMPLE_AND_SQUASH_MODULE>>>>>;
+            using SELECTED_MODULE = rl_tools::utils::typing::conditional_t<PARAMETERS::CRITIC_NUM_LAYERS == 3, MODULE_GRU, rl_tools::utils::typing::conditional_t<PARAMETERS::CRITIC_NUM_LAYERS == 4, MODULE_GRU_TWO_LAYER, MODULE_GRU_THREE_LAYER>>;
+            static_assert(PARAMETERS::CRITIC_NUM_LAYERS == 3 || PARAMETERS::CRITIC_NUM_LAYERS == 4 || PARAMETERS::CRITIC_NUM_LAYERS == 5, "Only 3/4/5 layers (1 input + 1/2 GRU + 1/2 Output) are supported right now");
+            using MODEL = nn_models::sequential::Build<CAPABILITY, SELECTED_MODULE, INPUT_SHAPE>;
+        };
+        template <typename CAPABILITY>
+        struct Critic{
+            static constexpr TI INPUT_DIM = ENVIRONMENT::ObservationPrivileged::DIM+ENVIRONMENT::ACTION_DIM;
+            using INPUT_SHAPE = tensor::Shape<TI, SAC_PARAMETERS::SEQUENCE_LENGTH, PARAMETERS::SAC_PARAMETERS::CRITIC_BATCH_SIZE, INPUT_DIM>;
+            using INPUT_LAYER_CONFIG = nn::layers::dense::Configuration<TYPE_POLICY, TI, PARAMETERS::CRITIC_HIDDEN_DIM, PARAMETERS::CRITIC_ACTIVATION_FUNCTION, nn::layers::dense::DefaultInitializer<TYPE_POLICY, TI>, nn::parameters::groups::Input>;
+            using INPUT_LAYER = nn::layers::dense::BindConfiguration<INPUT_LAYER_CONFIG>;
+            using GRU_SPEC = nn::layers::gru::Configuration<TYPE_POLICY, TI, PARAMETERS::CRITIC_HIDDEN_DIM, nn::parameters::groups::Normal, true>;
+            using GRU = nn::layers::gru::BindConfiguration<GRU_SPEC>;
+            using GRU2_SPEC = nn::layers::gru::Configuration<TYPE_POLICY, TI, PARAMETERS::CRITIC_HIDDEN_DIM, nn::parameters::groups::Normal, true>;
+            using GRU2 = nn::layers::gru::BindConfiguration<GRU2_SPEC>;
+            using DENSE_LAYER_CONFIG = nn::layers::dense::Configuration<TYPE_POLICY, TI, PARAMETERS::CRITIC_HIDDEN_DIM, PARAMETERS::CRITIC_ACTIVATION_FUNCTION, nn::layers::dense::DefaultInitializer<TYPE_POLICY, TI>, nn::parameters::groups::Normal>;
+            using DENSE_LAYER = nn::layers::dense::BindConfiguration<DENSE_LAYER_CONFIG>;
+            using OUTPUT_LAYER_CONFIG = nn::layers::dense::Configuration<TYPE_POLICY, TI, 1, nn::activation_functions::ActivationFunction::IDENTITY, nn::layers::dense::DefaultInitializer<TYPE_POLICY, TI>, nn::parameters::groups::Output>;
+            using OUTPUT_LAYER = nn::layers::dense::BindConfiguration<OUTPUT_LAYER_CONFIG>;
+            using MODULE_GRU = nn_models::sequential::Module<INPUT_LAYER, nn_models::sequential::Module<GRU, nn_models::sequential::Module<OUTPUT_LAYER>>>;
+            using MODULE_GRU_TWO_LAYER = nn_models::sequential::Module<INPUT_LAYER, nn_models::sequential::Module<GRU, nn_models::sequential::Module<GRU2, nn_models::sequential::Module<OUTPUT_LAYER>>>>;
+            using MODULE_GRU_THREE_LAYER = nn_models::sequential::Module<INPUT_LAYER, nn_models::sequential::Module<GRU, nn_models::sequential::Module<GRU2, nn_models::sequential::Module<DENSE_LAYER, nn_models::sequential::Module<OUTPUT_LAYER>>>>>;
+            using SELECTED_MODULE = rl_tools::utils::typing::conditional_t<PARAMETERS::CRITIC_NUM_LAYERS == 3, MODULE_GRU, rl_tools::utils::typing::conditional_t<PARAMETERS::CRITIC_NUM_LAYERS == 4, MODULE_GRU_TWO_LAYER, MODULE_GRU_THREE_LAYER>>;
+            static_assert(PARAMETERS::CRITIC_NUM_LAYERS == 3 || PARAMETERS::CRITIC_NUM_LAYERS == 4 || PARAMETERS::CRITIC_NUM_LAYERS == 5, "Only 3/4/5 layers (1 input + 1/2 GRU + 1/2 Output) are supported right now");
+            using MODEL = nn_models::sequential::Build<CAPABILITY, SELECTED_MODULE, INPUT_SHAPE>;
+        };
+
+        using CAPABILITY_ACTOR = nn::capability::Gradient<nn::parameters::Adam, DYNAMIC_ALLOCATION>;
+        using CAPABILITY_CRITIC = nn::capability::Gradient<nn::parameters::Adam, DYNAMIC_ALLOCATION>;
+        using ACTOR_TYPE = typename Actor<CAPABILITY_ACTOR>::MODEL;
+        using CRITIC_TYPE = typename Critic<CAPABILITY_CRITIC>::MODEL;
+        using CRITIC_TARGET_TYPE = typename Critic<nn::capability::Forward<DYNAMIC_ALLOCATION>>::MODEL;
+        using ACTOR_OPTIMIZER_SPEC = nn::optimizers::adam::Specification<TYPE_POLICY, TI, typename PARAMETERS::ACTOR_OPTIMIZER_PARAMETERS, DYNAMIC_ALLOCATION>;
+        using CRITIC_OPTIMIZER_SPEC = nn::optimizers::adam::Specification<TYPE_POLICY, TI, typename PARAMETERS::CRITIC_OPTIMIZER_PARAMETERS, DYNAMIC_ALLOCATION>;
+        using ALPHA_OPTIMIZER_SPEC = nn::optimizers::adam::Specification<TYPE_POLICY, TI, typename PARAMETERS::ALPHA_OPTIMIZER_PARAMETERS, DYNAMIC_ALLOCATION>;
+        using ACTOR_OPTIMIZER = nn::optimizers::Adam<ACTOR_OPTIMIZER_SPEC>;
+        using CRITIC_OPTIMIZER = nn::optimizers::Adam<CRITIC_OPTIMIZER_SPEC>;
+        using ALPHA_OPTIMIZER = nn::optimizers::Adam<ALPHA_OPTIMIZER_SPEC>;
+
+    };
+}
+RL_TOOLS_NAMESPACE_WRAPPER_END
+
+#endif
+

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/algorithms/sac/loop/core/approximators_mlp.h

@@ -0,0 +1,61 @@
+#include "../../../../../version.h"
+#if (defined(RL_TOOLS_DISABLE_INCLUDE_GUARDS) || !defined(RL_TOOLS_RL_ALGORITHMS_SAC_LOOP_CORE_APPROXIMATORS_MLP_H)) && (RL_TOOLS_USE_THIS_VERSION == 1)
+#pragma once
+#define RL_TOOLS_RL_ALGORITHMS_SAC_LOOP_CORE_APPROXIMATORS_MLP_H
+
+RL_TOOLS_NAMESPACE_WRAPPER_START
+namespace rl_tools::rl::algorithms::sac::loop::core{
+    // The approximator config sets up any types that support the usual rl_tools::forward and rl_tools::backward operations (can be custom as well)
+    // We provide approximators based on the sequential and mlp models. The latter (mlp) allows for a variable number of layers, but is restricted to a uniform hidden layer size while the former allows for arbitrary layers to be combined in a sequential manner. Both support compile-time autodiff
+    template<typename TYPE_POLICY, typename TI, typename ENVIRONMENT, typename PARAMETERS, bool DYNAMIC_ALLOCATION=true>
+    struct ConfigApproximatorsMLP{
+        using SAC_PARAMETERS = typename PARAMETERS::SAC_PARAMETERS;
+        template <typename CAPABILITY>
+        struct Actor{
+            using INPUT_SHAPE = tensor::Shape<TI, SAC_PARAMETERS::SEQUENCE_LENGTH, SAC_PARAMETERS::ACTOR_BATCH_SIZE, ENVIRONMENT::Observation::DIM>;
+            using MLP_CONFIG = nn_models::mlp::Configuration<TYPE_POLICY, TI, 2*ENVIRONMENT::ACTION_DIM, PARAMETERS::ACTOR_NUM_LAYERS, PARAMETERS::ACTOR_HIDDEN_DIM, PARAMETERS::ACTOR_ACTIVATION_FUNCTION,  nn::activation_functions::IDENTITY, typename PARAMETERS::INITIALIZER>;
+            using MLP = nn_models::mlp::BindConfiguration<MLP_CONFIG>;
+            struct SAMPLE_AND_SQUASH_LAYER_PARAMETERS{
+                using T = typename TYPE_POLICY::DEFAULT;
+                static constexpr T LOG_STD_LOWER_BOUND = SAC_PARAMETERS::LOG_STD_LOWER_BOUND;
+                static constexpr T LOG_STD_UPPER_BOUND = SAC_PARAMETERS::LOG_STD_UPPER_BOUND;
+                static constexpr T LOG_PROBABILITY_EPSILON = SAC_PARAMETERS::LOG_PROBABILITY_EPSILON;
+                static constexpr bool ADAPTIVE_ALPHA = SAC_PARAMETERS::ADAPTIVE_ALPHA;
+                static constexpr bool UPDATE_ALPHA_WITH_ACTOR = false;
+                static constexpr T ALPHA = SAC_PARAMETERS::ALPHA;
+                static constexpr T TARGET_ENTROPY = SAC_PARAMETERS::TARGET_ENTROPY;
+            };
+            using SAMPLE_AND_SQUASH_CONFIG = nn::layers::sample_and_squash::Configuration<TYPE_POLICY, TI, SAMPLE_AND_SQUASH_LAYER_PARAMETERS>;
+            using SAMPLE_AND_SQUASH = nn::layers::sample_and_squash::BindConfiguration<SAMPLE_AND_SQUASH_CONFIG>;
+            using MODULE_CHAIN = nn_models::sequential::Module<MLP, SAMPLE_AND_SQUASH>;
+
+            using MODEL = nn_models::sequential::Build<CAPABILITY, MODULE_CHAIN, INPUT_SHAPE>;
+
+        };
+        template <typename CAPABILITY>
+        struct Critic{
+            static constexpr TI INPUT_DIM = ENVIRONMENT::ObservationPrivileged::DIM+ENVIRONMENT::ACTION_DIM;
+            using INPUT_SHAPE = tensor::Shape<TI, SAC_PARAMETERS::SEQUENCE_LENGTH, SAC_PARAMETERS::CRITIC_BATCH_SIZE, INPUT_DIM>;
+            using MLP_CONFIG = nn_models::mlp::Configuration<TYPE_POLICY, TI, 1, PARAMETERS::CRITIC_NUM_LAYERS, PARAMETERS::CRITIC_HIDDEN_DIM, PARAMETERS::CRITIC_ACTIVATION_FUNCTION, nn::activation_functions::IDENTITY, typename PARAMETERS::INITIALIZER>;
+            using MLP = nn_models::mlp::BindConfiguration<MLP_CONFIG>;
+            using MODULE_CHAIN = nn_models::sequential::Module<MLP>;
+
+            using MODEL = nn_models::sequential::Build<CAPABILITY, MODULE_CHAIN, INPUT_SHAPE>;
+        };
+
+        using ACTOR_OPTIMIZER_SPEC = nn::optimizers::adam::Specification<TYPE_POLICY, TI, typename PARAMETERS::ACTOR_OPTIMIZER_PARAMETERS, DYNAMIC_ALLOCATION>;
+        using CRITIC_OPTIMIZER_SPEC = nn::optimizers::adam::Specification<TYPE_POLICY, TI, typename PARAMETERS::CRITIC_OPTIMIZER_PARAMETERS, DYNAMIC_ALLOCATION>;
+        using ALPHA_OPTIMIZER_SPEC = nn::optimizers::adam::Specification<TYPE_POLICY, TI, typename PARAMETERS::ALPHA_OPTIMIZER_PARAMETERS, DYNAMIC_ALLOCATION>;
+        using ACTOR_OPTIMIZER = nn::optimizers::Adam<ACTOR_OPTIMIZER_SPEC>;
+        using CRITIC_OPTIMIZER = nn::optimizers::Adam<CRITIC_OPTIMIZER_SPEC>;
+        using ALPHA_OPTIMIZER = nn::optimizers::Adam<ALPHA_OPTIMIZER_SPEC>;
+        using CAPABILITY_ACTOR = nn::capability::Gradient<nn::parameters::Adam, DYNAMIC_ALLOCATION>;
+        using CAPABILITY_CRITIC = nn::capability::Gradient<nn::parameters::Adam, DYNAMIC_ALLOCATION>;
+        using ACTOR_TYPE = typename Actor<CAPABILITY_ACTOR>::MODEL;
+        using CRITIC_TYPE = typename Critic<CAPABILITY_CRITIC>::MODEL;
+        using CRITIC_TARGET_TYPE = typename Critic<nn::capability::Forward<DYNAMIC_ALLOCATION>>::MODEL;
+    };
+}
+RL_TOOLS_NAMESPACE_WRAPPER_END
+#endif
+

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/algorithms/sac/loop/core/config.h

@@ -0,0 +1,106 @@
+#include "../../../../../version.h"
+#if (defined(RL_TOOLS_DISABLE_INCLUDE_GUARDS) || !defined(RL_TOOLS_RL_ALGORITHMS_SAC_LOOP_CORE_CONFIG_H)) && (RL_TOOLS_USE_THIS_VERSION == 1)
+#pragma once
+#define RL_TOOLS_RL_ALGORITHMS_SAC_LOOP_CORE_CONFIG_H
+
+#include "../../../../../nn_models/sequential/model.h"
+#include "../../../../../nn_models/mlp/network.h"
+#include "../../../../../nn_models/random_uniform/model.h"
+#include "../../../../../rl/algorithms/sac/sac.h"
+#include "../../../../../nn/optimizers/adam/adam.h"
+#include "state.h"
+#include "approximators_mlp.h"
+#include "approximators_gru.h"
+
+RL_TOOLS_NAMESPACE_WRAPPER_START
+namespace rl_tools::rl::algorithms::sac::loop::core{
+    // Config State (Init/Step)
+    template<typename TYPE_POLICY, typename TI, typename ENVIRONMENT>
+    struct DefaultParameters{
+        using SAC_PARAMETERS = rl::algorithms::sac::DefaultParameters<TYPE_POLICY, TI, ENVIRONMENT::ACTION_DIM>;
+        static constexpr TI N_ENVIRONMENTS = 1;
+        static constexpr TI N_WARMUP_STEPS = 100; // Exploration executed with a uniform random policy for N_WARMUP_STEPS steps
+        static constexpr TI N_WARMUP_STEPS_CRITIC = 100; // Number of steps before critic training starts
+        static constexpr TI N_WARMUP_STEPS_ACTOR = 100; // Number of steps before actor training starts
+        static_assert(N_WARMUP_STEPS >= SAC_PARAMETERS::ACTOR_BATCH_SIZE);
+        static constexpr TI STEP_LIMIT = 10000;
+        static constexpr TI REPLAY_BUFFER_CAP = STEP_LIMIT; // Note: when inheriting from this class for overwriting the default STEP_LIMIT you need to set the REPLAY_BUFFER_CAP as well otherwise it will be the default step limit
+        static constexpr TI EPISODE_STEP_LIMIT = ENVIRONMENT::EPISODE_STEP_LIMIT;
+
+        static constexpr TI ACTOR_HIDDEN_DIM = 64;
+        static constexpr TI ACTOR_NUM_LAYERS = 3;
+        static constexpr auto ACTOR_ACTIVATION_FUNCTION = nn::activation_functions::ActivationFunction::RELU;
+        static constexpr TI CRITIC_HIDDEN_DIM = 64;
+        static constexpr TI CRITIC_NUM_LAYERS = 3;
+        static constexpr auto CRITIC_ACTIVATION_FUNCTION = nn::activation_functions::ActivationFunction::RELU;
+
+        static constexpr bool COLLECT_EPISODE_STATS = true;
+        static constexpr TI EPISODE_STATS_BUFFER_SIZE = 1000;
+
+        static constexpr bool SHARED_BATCH = true;
+        static constexpr bool SAMPLE_ENVIRONMENT_PARAMETERS = true;
+
+        using BATCH_SAMPLING_PARAMETERS = rl::components::off_policy_runner::SequentialBatchParameters<TYPE_POLICY, TI, SAC_PARAMETERS::SEQUENCE_LENGTH, rl::components::off_policy_runner::SequentialBatchParametersDefault>;
+
+        using INITIALIZER = nn::layers::dense::DefaultInitializer<TYPE_POLICY, TI>;
+
+        using ACTOR_OPTIMIZER_PARAMETERS = nn::optimizers::adam::DEFAULT_PARAMETERS_TENSORFLOW<TYPE_POLICY>;
+        using CRITIC_OPTIMIZER_PARAMETERS = nn::optimizers::adam::DEFAULT_PARAMETERS_TENSORFLOW<TYPE_POLICY>;
+        using ALPHA_OPTIMIZER_PARAMETERS = nn::optimizers::adam::DEFAULT_PARAMETERS_TENSORFLOW<TYPE_POLICY>;
+    };
+
+
+    template<typename T_TYPE_POLICY, typename T_TI, typename T_RNG, typename T_ENVIRONMENT, typename T_PARAMETERS = DefaultParameters<T_TYPE_POLICY, T_TI, T_ENVIRONMENT>, template<typename, typename, typename, typename, bool> class APPROXIMATOR_CONFIG=ConfigApproximatorsMLP, bool T_DYNAMIC_ALLOCATION=true>
+    struct Config{
+        using TYPE_POLICY = T_TYPE_POLICY;
+        using TI = T_TI;
+        using RNG = T_RNG;
+        using ENVIRONMENT = T_ENVIRONMENT;
+        using ENVIRONMENT_EVALUATION = T_ENVIRONMENT;
+        using CORE_PARAMETERS = T_PARAMETERS;
+        static constexpr bool DYNAMIC_ALLOCATION = T_DYNAMIC_ALLOCATION;
+
+        static constexpr TI ENVIRONMENT_STEPS_PER_LOOP_STEP = CORE_PARAMETERS::N_ENVIRONMENTS;
+
+        using NN = APPROXIMATOR_CONFIG<TYPE_POLICY, TI, T_ENVIRONMENT, CORE_PARAMETERS, DYNAMIC_ALLOCATION>;
+
+        using EXPLORATION_POLICY_SPEC = nn_models::random_uniform::Specification<TYPE_POLICY, TI, ENVIRONMENT::Observation::DIM, ENVIRONMENT::ACTION_DIM, nn_models::random_uniform::Range::MINUS_ONE_TO_ONE>;
+        using EXPLORATION_POLICY = nn_models::RandomUniform<EXPLORATION_POLICY_SPEC>;
+
+        using ALPHA_PARAMETER_TYPE = nn::parameters::Adam;
+//        using ALPHA_OPTIMIZER = nn::optimizers::Adam<typename NN::ALPHA_OPTIMIZER_SPEC>;
+
+        using ACTOR_CRITIC_SPEC = rl::algorithms::sac::Specification<TYPE_POLICY, TI, ENVIRONMENT, typename NN::ACTOR_TYPE, typename NN::CRITIC_TYPE, typename NN::CRITIC_TARGET_TYPE, ALPHA_PARAMETER_TYPE, typename NN::ACTOR_OPTIMIZER, typename NN::CRITIC_OPTIMIZER, typename NN::ALPHA_OPTIMIZER, typename CORE_PARAMETERS::SAC_PARAMETERS, CORE_PARAMETERS::BATCH_SAMPLING_PARAMETERS::INCLUDE_FIRST_STEP_IN_TARGETS>;
+        using ACTOR_CRITIC_TYPE = rl::algorithms::sac::ActorCritic<ACTOR_CRITIC_SPEC>;
+        static constexpr TI NUM_NNS = 3;
+        using EVAL_ACTOR_TYPE = typename NN::ACTOR_TYPE::template CHANGE_BATCH_SIZE<TI, CORE_PARAMETERS::N_ENVIRONMENTS>;
+
+        struct OFF_POLICY_RUNNER_PARAMETERS{
+            static constexpr TI N_ENVIRONMENTS = CORE_PARAMETERS::N_ENVIRONMENTS;
+            static constexpr bool ASYMMETRIC_OBSERVATIONS = !rl_tools::utils::typing::is_same_v<typename ENVIRONMENT::Observation, typename ENVIRONMENT::ObservationPrivileged>;
+            static constexpr TI REPLAY_BUFFER_CAPACITY = CORE_PARAMETERS::REPLAY_BUFFER_CAP;
+            static constexpr TI EPISODE_STEP_LIMIT = CORE_PARAMETERS::EPISODE_STEP_LIMIT;
+            static constexpr bool COLLECT_EPISODE_STATS = CORE_PARAMETERS::COLLECT_EPISODE_STATS;
+            static constexpr TI EPISODE_STATS_BUFFER_SIZE = CORE_PARAMETERS::EPISODE_STATS_BUFFER_SIZE;
+            static constexpr bool SAMPLE_PARAMETERS = CORE_PARAMETERS::SAMPLE_ENVIRONMENT_PARAMETERS;
+        };
+        using POLICIES = rl_tools::utils::Tuple<TI, EXPLORATION_POLICY, EVAL_ACTOR_TYPE>;
+
+        using OFF_POLICY_RUNNER_SPEC = rl::components::off_policy_runner::Specification<TYPE_POLICY, TI, ENVIRONMENT, POLICIES, OFF_POLICY_RUNNER_PARAMETERS, DYNAMIC_ALLOCATION>;
+        static_assert(ACTOR_CRITIC_TYPE::SPEC::PARAMETERS::ACTOR_BATCH_SIZE == ACTOR_CRITIC_TYPE::SPEC::PARAMETERS::CRITIC_BATCH_SIZE);
+        static constexpr TI TARGET_SEQUENCE_LENGTH = CORE_PARAMETERS::SAC_PARAMETERS::SEQUENCE_LENGTH + (CORE_PARAMETERS::BATCH_SAMPLING_PARAMETERS::INCLUDE_FIRST_STEP_IN_TARGETS ? 1 : 0);
+
+        static constexpr TI DEFAULT_SEQUENCE_LENGTH = DefaultParameters<TYPE_POLICY, TI, ENVIRONMENT>::SAC_PARAMETERS::SEQUENCE_LENGTH;
+        static constexpr bool USING_DEFAULT_BATCH_SAMPLING_PARAMETERS = rl_tools::utils::typing::is_base_of_v<components::off_policy_runner::SequentialBatchParametersDefault, typename CORE_PARAMETERS::BATCH_SAMPLING_PARAMETERS>;
+        static_assert(CORE_PARAMETERS::SAC_PARAMETERS::SEQUENCE_LENGTH == DEFAULT_SEQUENCE_LENGTH || !USING_DEFAULT_BATCH_SAMPLING_PARAMETERS, "When setting a custom sequence length, please study and set custom batch sampling parameters.");
+
+        using CRITIC_BATCH_SPEC = rl::components::off_policy_runner::SequentialBatchSpecification<OFF_POLICY_RUNNER_SPEC, ACTOR_CRITIC_TYPE::SPEC::PARAMETERS::SEQUENCE_LENGTH, ACTOR_CRITIC_TYPE::SPEC::PARAMETERS::CRITIC_BATCH_SIZE, typename CORE_PARAMETERS::BATCH_SAMPLING_PARAMETERS, DYNAMIC_ALLOCATION>;
+        using ACTOR_BATCH_SPEC = rl::components::off_policy_runner::SequentialBatchSpecification<OFF_POLICY_RUNNER_SPEC, ACTOR_CRITIC_TYPE::SPEC::PARAMETERS::SEQUENCE_LENGTH, ACTOR_CRITIC_TYPE::SPEC::PARAMETERS::ACTOR_BATCH_SIZE, typename CORE_PARAMETERS::BATCH_SAMPLING_PARAMETERS, DYNAMIC_ALLOCATION>;
+        template <typename CONFIG>
+        using State = State<CONFIG>;
+    };
+}
+RL_TOOLS_NAMESPACE_WRAPPER_END
+
+#endif
+