hyperverse 0.0.1__py3-none-any.whl

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/algorithms/sac/sac.h

@@ -0,0 +1,151 @@
+#include "../../../version.h"
+#if (defined(RL_TOOLS_DISABLE_INCLUDE_GUARDS) || !defined(RL_TOOLS_RL_ALGORITHMS_SAC_SAC_H)) && (RL_TOOLS_USE_THIS_VERSION == 1)
+#pragma once
+#define RL_TOOLS_RL_ALGORITHMS_SAC_SAC_H
+//#include "../../../nn_models/output_view/model.h"
+
+
+RL_TOOLS_NAMESPACE_WRAPPER_START
+namespace rl_tools::rl::algorithms::sac {
+    template<typename TYPE_POLICY, typename TI, TI ACTION_DIM=1>
+    struct DefaultParameters {
+        using T = typename TYPE_POLICY::DEFAULT;
+        static constexpr T GAMMA = 0.99;
+        static constexpr TI ACTOR_BATCH_SIZE = 32;
+        static constexpr TI CRITIC_BATCH_SIZE = 32;
+        static constexpr TI CRITIC_TRAINING_INTERVAL = 1;
+        static constexpr TI ACTOR_TRAINING_INTERVAL = 1;
+        static constexpr TI CRITIC_TARGET_UPDATE_INTERVAL = 1;
+        static constexpr T ACTOR_POLYAK = 1.0 - 0.005;
+        static constexpr T CRITIC_POLYAK = 1.0 - 0.005;
+        static constexpr bool IGNORE_TERMINATION = false; // ignoring the termination flag is useful for training on environments with negative rewards, where the agent would try to terminate the episode as soon as possible otherwise
+        static constexpr TI SEQUENCE_LENGTH = 1; // note that this implementation does only show next_observation sequences to the target actor and critic. Hence they have one step (the initial one in the sequence) less information. This makes the sequence length deterministic (otherwise it would depend on the number of resets in the batch). For most environments and for larger sequences the information gain should be negligible but for some (mostly artifiical) environments the first state matters (e.g. the FlagMemory environment). A possible mitigations is repeating the initial observation in the environment
+        static constexpr bool ENTROPY_BONUS = true;
+        static constexpr bool ENTROPY_BONUS_NEXT_STEP = true;
+        static constexpr bool MASK_NON_TERMINAL = true;
+
+        static constexpr T TARGET_ENTROPY = -((T)ACTION_DIM);
+        static constexpr T ALPHA = 0.5;
+        static constexpr bool ADAPTIVE_ALPHA = true;
+        static constexpr T LOG_STD_LOWER_BOUND = -20;
+        static constexpr T LOG_STD_UPPER_BOUND = 2;
+        static constexpr T LOG_PROBABILITY_EPSILON = 1e-6;
+    };
+
+    template<
+        typename T_TYPE_POLICY,
+        typename T_TI,
+        typename T_ENVIRONMENT,
+        typename T_ACTOR_NETWORK_TYPE,
+        typename T_CRITIC_NETWORK_TYPE,
+        typename T_CRITIC_TARGET_NETWORK_TYPE,
+        typename T_ALPHA_PARAMETER_TYPE,
+        typename T_ACTOR_OPTIMIZER,
+        typename T_CRITIC_OPTIMIZER,
+        typename T_ALPHA_OPTIMIZER,
+        typename T_PARAMETERS,
+        bool T_INCLUDE_FIRST_STEP_IN_TARGETS
+    >
+    struct Specification{
+        using TYPE_POLICY = T_TYPE_POLICY;
+        using TI = T_TI;
+        using ENVIRONMENT = T_ENVIRONMENT;
+        using ACTOR_NETWORK_TYPE = T_ACTOR_NETWORK_TYPE;
+        using CRITIC_NETWORK_TYPE = T_CRITIC_NETWORK_TYPE;
+        using CRITIC_TARGET_NETWORK_TYPE = T_CRITIC_TARGET_NETWORK_TYPE;
+        using ALPHA_PARAMETER_TYPE = T_ALPHA_PARAMETER_TYPE;
+        using ACTOR_OPTIMIZER = T_ACTOR_OPTIMIZER;
+        using CRITIC_OPTIMIZER = T_CRITIC_OPTIMIZER;
+        using ALPHA_OPTIMIZER = T_ALPHA_OPTIMIZER;
+        using PARAMETERS = T_PARAMETERS;
+        static constexpr bool INCLUDE_FIRST_STEP_IN_TARGETS = T_INCLUDE_FIRST_STEP_IN_TARGETS;
+    };
+
+    template <typename T_SPEC, bool T_DYNAMIC_ALLOCATION>
+    struct ActorTrainingBuffersSpecification{
+        using SPEC = T_SPEC;
+        static constexpr bool DYNAMIC_ALLOCATION = T_DYNAMIC_ALLOCATION;
+    };
+
+    template<typename T_SPEC>
+    struct ActorTrainingBuffers{
+        using SPEC = typename T_SPEC::SPEC;
+        using TYPE_POLICY = typename SPEC::TYPE_POLICY;
+        using T = typename TYPE_POLICY::template GET<numeric_types::categories::Buffer>;
+        using TI = typename SPEC::TI;
+        static constexpr bool DYNAMIC_ALLOCATION = T_SPEC::DYNAMIC_ALLOCATION;
+        static constexpr TI SEQUENCE_LENGTH = SPEC::PARAMETERS::SEQUENCE_LENGTH;
+        static constexpr TI BATCH_SIZE = SPEC::PARAMETERS::ACTOR_BATCH_SIZE;
+        static constexpr TI ACTOR_INPUT_DIM = get_last(typename SPEC::ACTOR_NETWORK_TYPE::INPUT_SHAPE{});
+        static constexpr TI ACTION_DIM = SPEC::ENVIRONMENT::ACTION_DIM;
+        static constexpr TI CRITIC_OBSERVATION_DIM = get_last(typename SPEC::CRITIC_NETWORK_TYPE::INPUT_SHAPE{}) - SPEC::ENVIRONMENT::ACTION_DIM;
+
+        Tensor<tensor::Specification<T, TI, tensor::Shape<TI, SEQUENCE_LENGTH, BATCH_SIZE, CRITIC_OBSERVATION_DIM + ACTION_DIM>, DYNAMIC_ALLOCATION>> state_action_value_input;
+        template<typename SPEC::TI DIM>
+        using STATE_ACTION_VALUE_VIEW = typename decltype(state_action_value_input)::template VIEW_RANGE<tensor::ViewSpec<2, DIM>>;
+        STATE_ACTION_VALUE_VIEW<CRITIC_OBSERVATION_DIM> observations;
+        STATE_ACTION_VALUE_VIEW<ACTION_DIM> actions;
+        Tensor<tensor::Specification<T, TI, tensor::Shape<TI, SEQUENCE_LENGTH, BATCH_SIZE, 1>, DYNAMIC_ALLOCATION>> d_output;
+        Tensor<tensor::Specification<T, TI, tensor::Shape<TI, SEQUENCE_LENGTH, BATCH_SIZE, CRITIC_OBSERVATION_DIM + ACTION_DIM>, DYNAMIC_ALLOCATION>> d_critic_1_input, d_critic_2_input;
+        Tensor<tensor::Specification<T, TI, tensor::Shape<TI, SEQUENCE_LENGTH, BATCH_SIZE, ACTION_DIM>, DYNAMIC_ALLOCATION>> d_critic_action_input;
+        Tensor<tensor::Specification<T, TI, tensor::Shape<TI, SEQUENCE_LENGTH, BATCH_SIZE, ACTION_DIM>, DYNAMIC_ALLOCATION>> action_sample, action_noise;
+        Tensor<tensor::Specification<T, TI, tensor::Shape<TI, SEQUENCE_LENGTH, BATCH_SIZE, ACTION_DIM>, DYNAMIC_ALLOCATION>> d_actor_output_squashing;
+        Tensor<tensor::Specification<T, TI, tensor::Shape<TI, SEQUENCE_LENGTH, BATCH_SIZE, ACTION_DIM * 2>, DYNAMIC_ALLOCATION>> d_squashing_input;
+        Tensor<tensor::Specification<T, TI, tensor::Shape<TI, SEQUENCE_LENGTH, BATCH_SIZE, ACTION_DIM * 2>, DYNAMIC_ALLOCATION>> d_actor_output;
+        Tensor<tensor::Specification<T, TI, tensor::Shape<TI, SEQUENCE_LENGTH, BATCH_SIZE, ACTOR_INPUT_DIM>, DYNAMIC_ALLOCATION>> d_actor_input;
+        Tensor<tensor::Specification<T, TI, tensor::Shape<TI, 1>, DYNAMIC_ALLOCATION>> loss_weight;
+    };
+    template <typename T_SPEC, bool T_DYNAMIC_ALLOCATION>
+    struct CriticTrainingBuffersSpecification{
+        using SPEC = T_SPEC;
+        static constexpr bool DYNAMIC_ALLOCATION = T_DYNAMIC_ALLOCATION;
+    };
+    template<typename T_SPEC>
+    struct CriticTrainingBuffers{
+        using SPEC = typename T_SPEC::SPEC;
+        using TYPE_POLICY = typename SPEC::TYPE_POLICY;
+        using T_BUFFER = typename TYPE_POLICY::template GET<numeric_types::categories::Buffer>;
+        using TI = typename SPEC::TI;
+        static constexpr bool DYNAMIC_ALLOCATION = T_SPEC::DYNAMIC_ALLOCATION;
+        static constexpr TI SEQUENCE_LENGTH = SPEC::PARAMETERS::SEQUENCE_LENGTH;
+        static constexpr TI NEXT_SEQUENCE_LENGTH = SPEC::INCLUDE_FIRST_STEP_IN_TARGETS ? SEQUENCE_LENGTH + 1 : SEQUENCE_LENGTH;
+        static constexpr TI BATCH_SIZE = SPEC::PARAMETERS::CRITIC_BATCH_SIZE;
+        static constexpr TI ACTION_DIM = SPEC::ENVIRONMENT::ACTION_DIM;
+        static constexpr TI CRITIC_OBSERVATION_DIM = get_last(typename SPEC::CRITIC_NETWORK_TYPE::INPUT_SHAPE{}) - SPEC::ENVIRONMENT::ACTION_DIM;
+
+
+        Tensor<tensor::Specification<T_BUFFER, TI, tensor::Shape<TI, NEXT_SEQUENCE_LENGTH, BATCH_SIZE, CRITIC_OBSERVATION_DIM + ACTION_DIM>, DYNAMIC_ALLOCATION>> next_state_action_value_input;
+        template<typename SPEC::TI DIM>
+        using NEXT_STATE_ACTION_VALUE_VIEW = typename decltype(next_state_action_value_input)::template VIEW_RANGE<tensor::ViewSpec<2, DIM>>;
+        NEXT_STATE_ACTION_VALUE_VIEW<CRITIC_OBSERVATION_DIM> next_observations;
+        NEXT_STATE_ACTION_VALUE_VIEW<ACTION_DIM> next_actions;
+        Tensor<tensor::Specification<T_BUFFER, TI, tensor::Shape<TI, SEQUENCE_LENGTH, BATCH_SIZE, 1>, DYNAMIC_ALLOCATION>> action_value;
+        Tensor<tensor::Specification<T_BUFFER, TI, tensor::Shape<TI, SEQUENCE_LENGTH, BATCH_SIZE, 1>, DYNAMIC_ALLOCATION>> target_action_value;
+        Tensor<tensor::Specification<T_BUFFER, TI, tensor::Shape<TI, NEXT_SEQUENCE_LENGTH, BATCH_SIZE, 1>, DYNAMIC_ALLOCATION>> next_state_action_value_critic_1;
+        Tensor<tensor::Specification<T_BUFFER, TI, tensor::Shape<TI, NEXT_SEQUENCE_LENGTH, BATCH_SIZE, 1>, DYNAMIC_ALLOCATION>> next_state_action_value_critic_2;
+        Tensor<tensor::Specification<T_BUFFER, TI, tensor::Shape<TI, SEQUENCE_LENGTH, BATCH_SIZE, CRITIC_OBSERVATION_DIM + ACTION_DIM>, DYNAMIC_ALLOCATION>> d_input;
+        Tensor<tensor::Specification<T_BUFFER, TI, tensor::Shape<TI, SEQUENCE_LENGTH, BATCH_SIZE, 1>, DYNAMIC_ALLOCATION>> d_output;
+        Tensor<tensor::Specification<T_BUFFER, TI, tensor::Shape<TI, 1>, DYNAMIC_ALLOCATION>> loss_weight;
+        Tensor<tensor::Specification<T_BUFFER, TI, tensor::Shape<TI, NEXT_SEQUENCE_LENGTH * BATCH_SIZE>, DYNAMIC_ALLOCATION>> next_action_log_probs;
+    };
+
+    template<typename T_SPEC>
+    struct ActorCritic {
+        using SPEC = T_SPEC;
+        // using T = typename SPEC::T;
+        using TI = typename SPEC::TI;
+
+        typename SPEC::ACTOR_NETWORK_TYPE actor;
+        typename SPEC::CRITIC_NETWORK_TYPE critics[2];
+        typename SPEC::CRITIC_TARGET_NETWORK_TYPE critics_target[2];
+
+        typename SPEC::ACTOR_OPTIMIZER actor_optimizer;
+        typename SPEC::CRITIC_OPTIMIZER critic_optimizers[2];
+        typename SPEC::ALPHA_OPTIMIZER alpha_optimizer;
+    };
+}
+RL_TOOLS_NAMESPACE_WRAPPER_END
+
+
+
+#endif

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/algorithms/td3/loop/core/approximators_gru.h

@@ -0,0 +1,72 @@
+#include "../../../../../version.h"
+#if (defined(RL_TOOLS_DISABLE_INCLUDE_GUARDS) || !defined(RL_TOOLS_RL_ALGORITHMS_TD3_LOOP_CORE_APPROXIMATORS_GRU_H)) && (RL_TOOLS_USE_THIS_VERSION == 1)
+#pragma once
+#define RL_TOOLS_RL_ALGORITHMS_TD3_LOOP_CORE_APPROXIMATORS_GRU_H
+
+RL_TOOLS_NAMESPACE_WRAPPER_START
+namespace rl_tools::rl::algorithms::td3::loop::core{
+    template<typename T, typename TI, typename ENVIRONMENT, typename PARAMETERS, bool DYNAMIC_ALLOCATION>
+    struct ConfigApproximatorsGRU{
+    //    static constexpr bool USE_GRU = true;
+        using TD3_PARAMETERS = typename PARAMETERS::TD3_PARAMETERS;
+        template <typename CAPABILITY>
+        struct Actor{
+            using INPUT_SHAPE = tensor::Shape<TI, TD3_PARAMETERS::SEQUENCE_LENGTH, TD3_PARAMETERS::ACTOR_BATCH_SIZE, ENVIRONMENT::Observation::DIM>;
+            using INPUT_LAYER_CONFIG = nn::layers::dense::Configuration<T, TI, PARAMETERS::ACTOR_HIDDEN_DIM, PARAMETERS::CRITIC_ACTIVATION_FUNCTION, nn::layers::dense::DefaultInitializer<T, TI>, nn::parameters::groups::Input>;
+            using INPUT_LAYER = nn::layers::dense::BindConfiguration<INPUT_LAYER_CONFIG>;
+            using GRU_CONFIG = nn::layers::gru::Configuration<T, TI, PARAMETERS::ACTOR_HIDDEN_DIM, nn::parameters::groups::Normal, true>;
+            using GRU = nn::layers::gru::BindConfiguration<GRU_CONFIG>;
+            using GRU2_CONFIG = nn::layers::gru::Configuration<T, TI, PARAMETERS::ACTOR_HIDDEN_DIM, nn::parameters::groups::Normal, true>;
+            using GRU2 = nn::layers::gru::BindConfiguration<GRU2_CONFIG>;
+            using DENSE_LAYER_CONFIG = nn::layers::dense::Configuration<T, TI, PARAMETERS::ACTOR_HIDDEN_DIM, PARAMETERS::ACTOR_ACTIVATION_FUNCTION, nn::layers::dense::DefaultInitializer<T, TI>, nn::parameters::groups::Normal>;
+            using DENSE_LAYER = nn::layers::dense::BindConfiguration<DENSE_LAYER_CONFIG>;
+            using OUTPUT_CONFIG = nn::layers::dense::Configuration<T, TI, ENVIRONMENT::ACTION_DIM, nn::activation_functions::ActivationFunction::IDENTITY, nn::layers::dense::DefaultInitializer<T, TI>, nn::parameters::groups::Output>;
+            using OUTPUT = nn::layers::dense::BindConfiguration<OUTPUT_CONFIG>;
+
+            using MODULE_GRU = nn_models::sequential::Module<GRU, nn_models::sequential::Module<OUTPUT>>;
+            using MODULE_GRU_TWO_LAYER = nn_models::sequential::Module<GRU, nn_models::sequential::Module<GRU2, nn_models::sequential::Module<OUTPUT>>>;
+            using MODULE_GRU_THREE_LAYER = nn_models::sequential::Module<GRU, nn_models::sequential::Module<GRU2, nn_models::sequential::Module<DENSE_LAYER, nn_models::sequential::Module<OUTPUT>>>>;
+
+            using SELECTED_MODULE = rl_tools::utils::typing::conditional_t<PARAMETERS::CRITIC_NUM_LAYERS == 3, MODULE_GRU, rl_tools::utils::typing::conditional_t<PARAMETERS::CRITIC_NUM_LAYERS == 4, MODULE_GRU_TWO_LAYER, MODULE_GRU_THREE_LAYER>>;
+            static_assert(PARAMETERS::CRITIC_NUM_LAYERS == 3 || PARAMETERS::CRITIC_NUM_LAYERS == 4 || PARAMETERS::CRITIC_NUM_LAYERS == 5, "Only 3/4/5 layers (1 input + 1/2 GRU + 1/2 Output) are supported right now");
+            using MODEL = nn_models::sequential::Build<CAPABILITY, SELECTED_MODULE, INPUT_SHAPE>;
+        };
+        template <typename CAPABILITY>
+        struct Critic{
+            using INPUT_SHAPE = tensor::Shape<TI, TD3_PARAMETERS::SEQUENCE_LENGTH, TD3_PARAMETERS::CRITIC_BATCH_SIZE, ENVIRONMENT::ObservationPrivileged::DIM + ENVIRONMENT::ACTION_DIM>;
+            using INPUT_LAYER_CONFIG = nn::layers::dense::Configuration<T, TI, PARAMETERS::CRITIC_HIDDEN_DIM, PARAMETERS::CRITIC_ACTIVATION_FUNCTION, nn::layers::dense::DefaultInitializer<T, TI>, nn::parameters::groups::Input>;
+            using INPUT_LAYER = nn::layers::dense::BindConfiguration<INPUT_LAYER_CONFIG>;
+            using GRU_CONFIG = nn::layers::gru::Configuration<T, TI, PARAMETERS::CRITIC_HIDDEN_DIM, nn::parameters::groups::Normal, true>;
+            using GRU = nn::layers::gru::BindConfiguration<GRU_CONFIG>;
+            using GRU2_CONFIG = nn::layers::gru::Configuration<T, TI, PARAMETERS::CRITIC_HIDDEN_DIM, nn::parameters::groups::Normal, true>;
+            using GRU2 = nn::layers::gru::BindConfiguration<GRU2_CONFIG>;
+            using DENSE_LAYER_CONFIG = nn::layers::dense::Configuration<T, TI, PARAMETERS::CRITIC_HIDDEN_DIM, PARAMETERS::CRITIC_ACTIVATION_FUNCTION, nn::layers::dense::DefaultInitializer<T, TI>, nn::parameters::groups::Normal>;
+            using DENSE_LAYER = nn::layers::dense::BindConfiguration<DENSE_LAYER_CONFIG>;
+            using OUTPUT_CONFIG = nn::layers::dense::Configuration<T, TI, 1, nn::activation_functions::ActivationFunction::IDENTITY, nn::layers::dense::DefaultInitializer<T, TI>, nn::parameters::groups::Output>;
+            using OUTPUT = nn::layers::dense::BindConfiguration<OUTPUT_CONFIG>;
+            static constexpr TI INPUT_DIM = ENVIRONMENT::ObservationPrivileged::DIM+ENVIRONMENT::ACTION_DIM;
+
+            using MODULE_GRU = nn_models::sequential::Module<GRU, nn_models::sequential::Module<OUTPUT>>;
+            using MODULE_GRU_TWO_LAYER = nn_models::sequential::Module<GRU, nn_models::sequential::Module<GRU2, nn_models::sequential::Module<OUTPUT>>>;
+            using MODULE_GRU_THREE_LAYER = nn_models::sequential::Module<GRU, nn_models::sequential::Module<GRU2, nn_models::sequential::Module<DENSE_LAYER, nn_models::sequential::Module<OUTPUT>>>>;
+
+            using SELECTED_MODULE = rl_tools::utils::typing::conditional_t<PARAMETERS::CRITIC_NUM_LAYERS == 3, MODULE_GRU, rl_tools::utils::typing::conditional_t<PARAMETERS::CRITIC_NUM_LAYERS == 4, MODULE_GRU_TWO_LAYER, MODULE_GRU_THREE_LAYER>>;
+            static_assert(PARAMETERS::CRITIC_NUM_LAYERS == 3 || PARAMETERS::CRITIC_NUM_LAYERS == 4 || PARAMETERS::CRITIC_NUM_LAYERS == 5, "Only 3/4/5 layers (1 input + 1/2 GRU + 1/2 Output) are supported right now");
+            using MODEL = nn_models::sequential::Build<CAPABILITY, SELECTED_MODULE, INPUT_SHAPE>;
+        };
+
+        using CAPABILITY_ACTOR = nn::capability::Gradient<nn::parameters::Adam, DYNAMIC_ALLOCATION>;
+        using CAPABILITY_CRITIC = nn::capability::Gradient<nn::parameters::Adam, DYNAMIC_ALLOCATION>;
+        using ACTOR_TYPE = typename Actor<CAPABILITY_ACTOR>::MODEL;
+        using CRITIC_TYPE = typename Critic<CAPABILITY_CRITIC>::MODEL;
+        using CRITIC_TARGET_TYPE = typename Critic<nn::capability::Forward<>>::MODEL;
+        using ACTOR_TARGET_TYPE = typename Actor<nn::capability::Forward<>>::MODEL;
+        using OPTIMIZER_SPEC = nn::optimizers::adam::Specification<T, TI, typename PARAMETERS::OPTIMIZER_PARAMETERS, DYNAMIC_ALLOCATION>;
+        using OPTIMIZER = nn::optimizers::Adam<OPTIMIZER_SPEC>;
+
+    };
+}
+RL_TOOLS_NAMESPACE_WRAPPER_END
+#endif
+
+

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/algorithms/td3/loop/core/approximators_mlp.h

@@ -0,0 +1,54 @@
+#include "../../../../../version.h"
+#if (defined(RL_TOOLS_DISABLE_INCLUDE_GUARDS) || !defined(RL_TOOLS_RL_ALGORITHMS_TD3_LOOP_CORE_APPROXIMATORS_MLP_H)) && (RL_TOOLS_USE_THIS_VERSION == 1)
+#pragma once
+#define RL_TOOLS_RL_ALGORITHMS_TD3_LOOP_CORE_APPROXIMATORS_MLP_H
+
+RL_TOOLS_NAMESPACE_WRAPPER_START
+namespace rl_tools::rl::algorithms::td3::loop::core{
+    // The approximator config sets up any types that support the usual rl_tools::forward and rl_tools::backward operations (can be custom as well)
+    // We provide approximators based on the sequential and mlp models. The latter (mlp) allows for a variable number of layers, but is restricted to a uniform hidden layer size while the former allows for arbitrary layers to be combined in a sequential manner. Both support compile-time autodiff
+    template<typename TYPE_POLICY, typename TI, typename ENVIRONMENT, typename PARAMETERS, bool DYNAMIC_ALLOCATION>
+    struct ConfigApproximatorsMLP{
+        using TD3_PARAMETERS = typename PARAMETERS::TD3_PARAMETERS;
+        template <typename CAPABILITY>
+        struct ACTOR{
+            using INPUT_SHAPE = tensor::Shape<TI, TD3_PARAMETERS::SEQUENCE_LENGTH, TD3_PARAMETERS::ACTOR_BATCH_SIZE, ENVIRONMENT::Observation::DIM>;
+            using MLP_CONFIG = nn_models::mlp::Configuration<TYPE_POLICY, TI, ENVIRONMENT::ACTION_DIM, PARAMETERS::ACTOR_NUM_LAYERS, PARAMETERS::ACTOR_HIDDEN_DIM, PARAMETERS::ACTOR_ACTIVATION_FUNCTION, nn::activation_functions::ActivationFunction::TANH>;
+            using MLP = nn_models::mlp::BindConfiguration<MLP_CONFIG>;
+            struct SAMPLING_PARAMETERS: nn::layers::td3_sampling::DefaultParameters<TYPE_POLICY>{
+                static constexpr typename TYPE_POLICY::DEFAULT STD = PARAMETERS::EXPLORATION_NOISE;
+            };
+            using SAMPLING_CONFIG = nn::layers::td3_sampling::Configuration<TYPE_POLICY, TI, SAMPLING_PARAMETERS>;
+            using SAMPLING = nn::layers::td3_sampling::BindConfiguration<SAMPLING_CONFIG>;
+
+            using MODULE_CHAIN = nn_models::sequential::Module<MLP, nn_models::sequential::Module<SAMPLING>>;
+            using MODEL = nn_models::sequential::Build<CAPABILITY, MODULE_CHAIN, INPUT_SHAPE>;
+        };
+
+        template <typename CAPABILITY>
+        struct CRITIC{
+            static constexpr TI HIDDEN_DIM = PARAMETERS::CRITIC_HIDDEN_DIM;
+            static constexpr auto ACTIVATION_FUNCTION = PARAMETERS::CRITIC_ACTIVATION_FUNCTION;
+
+            using INPUT_SHAPE = tensor::Shape<TI, TD3_PARAMETERS::SEQUENCE_LENGTH, TD3_PARAMETERS::CRITIC_BATCH_SIZE, ENVIRONMENT::ObservationPrivileged::DIM + ENVIRONMENT::ACTION_DIM>;
+            using MLP_CONFIG = nn_models::mlp::Configuration<TYPE_POLICY, TI, 1, PARAMETERS::ACTOR_NUM_LAYERS, PARAMETERS::ACTOR_HIDDEN_DIM, PARAMETERS::ACTOR_ACTIVATION_FUNCTION, nn::activation_functions::ActivationFunction::IDENTITY>;
+            using MLP = nn_models::mlp::BindConfiguration<MLP_CONFIG>;
+
+            using MODULE_CHAIN = nn_models::sequential::Module<MLP>;
+            using MODEL = nn_models::sequential::Build<CAPABILITY, MODULE_CHAIN, INPUT_SHAPE>;
+        };
+
+        using OPTIMIZER_SPEC = nn::optimizers::adam::Specification<TYPE_POLICY, TI, typename PARAMETERS::OPTIMIZER_PARAMETERS, DYNAMIC_ALLOCATION>;
+
+        using OPTIMIZER = nn::optimizers::Adam<OPTIMIZER_SPEC>;
+
+        using ACTOR_TYPE = typename ACTOR<nn::capability::Gradient<nn::parameters::Adam, DYNAMIC_ALLOCATION>>::MODEL;
+        using ACTOR_TARGET_TYPE = typename ACTOR<nn::capability::Forward<DYNAMIC_ALLOCATION>>::MODEL;
+        using CRITIC_TYPE = typename CRITIC<nn::capability::Gradient<nn::parameters::Adam, DYNAMIC_ALLOCATION>>::MODEL;
+        using CRITIC_TARGET_TYPE = typename CRITIC<nn::capability::Forward<DYNAMIC_ALLOCATION>>::MODEL;
+    };
+}
+RL_TOOLS_NAMESPACE_WRAPPER_END
+
+
+#endif

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/algorithms/td3/loop/core/config.h

@@ -0,0 +1,107 @@
+#include "../../../../../version.h"
+#if (defined(RL_TOOLS_DISABLE_INCLUDE_GUARDS) || !defined(RL_TOOLS_RL_ALGORITHMS_TD3_LOOP_CORE_CONFIG_H)) && (RL_TOOLS_USE_THIS_VERSION == 1)
+#pragma once
+#define RL_TOOLS_RL_ALGORITHMS_TD3_LOOP_CORE_CONFIG_H
+
+#include "../../../../../nn/layers/td3_sampling/layer.h"
+#include "../../../../../nn_models/mlp/network.h"
+#include "../../../../../nn_models/random_uniform/model.h"
+#include "../../../../../nn_models/sequential/model.h"
+#include "../../../../../rl/algorithms/td3/td3.h"
+#include "../../../../../nn/optimizers/adam/adam.h"
+#include "state.h"
+#include "approximators_mlp.h"
+#include "approximators_gru.h"
+
+RL_TOOLS_NAMESPACE_WRAPPER_START
+namespace rl_tools::rl::algorithms::td3::loop::core{
+    // Config State (Init/Step)
+
+    template<typename TYPE_POLICY, typename TI, typename ENVIRONMENT>
+    struct DefaultParameters{
+        using T = typename TYPE_POLICY::DEFAULT;
+        using TD3_PARAMETERS = rl::algorithms::td3::DefaultParameters<TYPE_POLICY, TI>;
+        static constexpr TI N_ENVIRONMENTS = 1;
+        static constexpr TI N_WARMUP_STEPS = 100; // Exploration executed with a uniform random policy for N_WARMUP_STEPS steps
+        static constexpr TI N_WARMUP_STEPS_CRITIC = 100; // Number of steps before critic training starts
+        static constexpr TI N_WARMUP_STEPS_ACTOR = 100; // Number of steps before actor training starts
+        static constexpr TI STEP_LIMIT = 10000;
+        static constexpr TI REPLAY_BUFFER_CAP = STEP_LIMIT; // Note: when inheriting from this class for overwriting the default STEP_LIMIT you need to set the REPLAY_BUFFER_CAP as well otherwise it will be the default step limit
+        static constexpr TI EPISODE_STEP_LIMIT = ENVIRONMENT::EPISODE_STEP_LIMIT;
+
+        static constexpr TI ACTOR_HIDDEN_DIM = 64;
+        static constexpr TI ACTOR_NUM_LAYERS = 3;
+        static constexpr auto ACTOR_ACTIVATION_FUNCTION = nn::activation_functions::ActivationFunction::RELU;
+        static constexpr TI CRITIC_HIDDEN_DIM = 64;
+        static constexpr TI CRITIC_NUM_LAYERS = 3;
+        static constexpr auto CRITIC_ACTIVATION_FUNCTION = nn::activation_functions::ActivationFunction::RELU;
+
+        static constexpr bool COLLECT_EPISODE_STATS = true;
+        static constexpr TI EPISODE_STATS_BUFFER_SIZE = 1000;
+
+        static constexpr T EXPLORATION_NOISE = 0.1;
+        static constexpr bool SHARED_BATCH = true;
+
+        using BATCH_SAMPLING_PARAMETERS = rl::components::off_policy_runner::SequentialBatchParameters<TYPE_POLICY, TI, TD3_PARAMETERS::SEQUENCE_LENGTH, rl::components::off_policy_runner::SequentialBatchParametersDefault>;
+
+        using OPTIMIZER_PARAMETERS = nn::optimizers::adam::DEFAULT_PARAMETERS_TENSORFLOW<TYPE_POLICY>;
+    };
+
+
+    template<typename T_TYPE_POLICY, typename T_TI, typename T_RNG, typename T_ENVIRONMENT, typename T_PARAMETERS = DefaultParameters<T_TYPE_POLICY, T_TI, T_ENVIRONMENT>, template<typename, typename, typename, typename, bool> class APPROXIMATOR_CONFIG=ConfigApproximatorsMLP, bool T_DYNAMIC_ALLOCATION=true>
+    struct Config{
+        using TYPE_POLICY = T_TYPE_POLICY;
+        using T_CONFIG = typename TYPE_POLICY::DEFAULT;
+        using TI = T_TI;
+        using RNG = T_RNG;
+        using ENVIRONMENT = T_ENVIRONMENT;
+        using ENVIRONMENT_EVALUATION = T_ENVIRONMENT;
+        static constexpr bool DYNAMIC_ALLOCATION = T_DYNAMIC_ALLOCATION;
+
+        using NN = APPROXIMATOR_CONFIG<TYPE_POLICY, TI, T_ENVIRONMENT, T_PARAMETERS, DYNAMIC_ALLOCATION>;
+//        using NN = ConfigApproximatorsMLP<T, TI, T_ENVIRONMENT, T_PARAMETERS>;
+
+
+        using CORE_PARAMETERS = T_PARAMETERS;
+
+        static constexpr TI ENVIRONMENT_STEPS_PER_LOOP_STEP = CORE_PARAMETERS::N_ENVIRONMENTS;
+
+
+        using EXPLORATION_POLICY_SPEC = nn_models::random_uniform::Specification<TYPE_POLICY, TI, ENVIRONMENT::Observation::DIM, ENVIRONMENT::ACTION_DIM, nn_models::random_uniform::Range::MINUS_ONE_TO_ONE>;
+        using EXPLORATION_POLICY = nn_models::RandomUniform<EXPLORATION_POLICY_SPEC>;
+
+        using ACTOR_CRITIC_SPEC = rl::algorithms::td3::Specification<TYPE_POLICY, TI, ENVIRONMENT, typename NN::ACTOR_TYPE, typename NN::ACTOR_TARGET_TYPE, typename NN::CRITIC_TYPE, typename NN::CRITIC_TARGET_TYPE, typename NN::OPTIMIZER, typename CORE_PARAMETERS::TD3_PARAMETERS, CORE_PARAMETERS::BATCH_SAMPLING_PARAMETERS::INCLUDE_FIRST_STEP_IN_TARGETS>;
+        using ACTOR_CRITIC_TYPE = rl::algorithms::td3::ActorCritic<ACTOR_CRITIC_SPEC>;
+        static constexpr TI NUM_NNS = 3;
+        using EVAL_ACTOR_TYPE = typename NN::ACTOR_TYPE::template CHANGE_BATCH_SIZE<TI, CORE_PARAMETERS::N_ENVIRONMENTS>;
+
+        struct OFF_POLICY_RUNNER_PARAMETERS{
+            static constexpr TI N_ENVIRONMENTS = CORE_PARAMETERS::N_ENVIRONMENTS;
+            static constexpr bool ASYMMETRIC_OBSERVATIONS = !rl_tools::utils::typing::is_same_v<typename ENVIRONMENT::Observation, typename ENVIRONMENT::ObservationPrivileged>;
+            static constexpr TI REPLAY_BUFFER_CAPACITY = CORE_PARAMETERS::REPLAY_BUFFER_CAP;
+            static constexpr TI EPISODE_STEP_LIMIT = CORE_PARAMETERS::EPISODE_STEP_LIMIT;
+            static constexpr bool COLLECT_EPISODE_STATS = CORE_PARAMETERS::COLLECT_EPISODE_STATS;
+            static constexpr TI EPISODE_STATS_BUFFER_SIZE = CORE_PARAMETERS::EPISODE_STATS_BUFFER_SIZE;
+            static constexpr T_CONFIG EXPLORATION_NOISE = CORE_PARAMETERS::EXPLORATION_NOISE;
+            static constexpr bool SAMPLE_PARAMETERS = true;
+        };
+        using POLICIES = rl_tools::utils::Tuple<TI, EXPLORATION_POLICY, EVAL_ACTOR_TYPE>;
+
+        using OFF_POLICY_RUNNER_SPEC = rl::components::off_policy_runner::Specification<TYPE_POLICY, TI, ENVIRONMENT, POLICIES, OFF_POLICY_RUNNER_PARAMETERS, T_DYNAMIC_ALLOCATION>;
+        static_assert(ACTOR_CRITIC_TYPE::SPEC::PARAMETERS::ACTOR_BATCH_SIZE == ACTOR_CRITIC_TYPE::SPEC::PARAMETERS::CRITIC_BATCH_SIZE);
+        static constexpr TI TARGET_SEQUENCE_LENGTH = CORE_PARAMETERS::TD3_PARAMETERS::SEQUENCE_LENGTH + (CORE_PARAMETERS::BATCH_SAMPLING_PARAMETERS::INCLUDE_FIRST_STEP_IN_TARGETS ? 1 : 0);
+
+        static constexpr TI DEFAULT_SEQUENCE_LENGTH = DefaultParameters<TYPE_POLICY, TI, ENVIRONMENT>::TD3_PARAMETERS::SEQUENCE_LENGTH;
+        static constexpr bool USING_DEFAULT_BATCH_SAMPLING_PARAMETERS = rl_tools::utils::typing::is_base_of_v<components::off_policy_runner::SequentialBatchParametersDefault, typename CORE_PARAMETERS::BATCH_SAMPLING_PARAMETERS>;
+        static_assert(CORE_PARAMETERS::TD3_PARAMETERS::SEQUENCE_LENGTH == DEFAULT_SEQUENCE_LENGTH || !USING_DEFAULT_BATCH_SAMPLING_PARAMETERS, "When setting a custom sequence length, please study and set custom batch sampling parameters.");
+
+        using CRITIC_BATCH_SPEC = rl::components::off_policy_runner::SequentialBatchSpecification<OFF_POLICY_RUNNER_SPEC, ACTOR_CRITIC_TYPE::SPEC::PARAMETERS::SEQUENCE_LENGTH, ACTOR_CRITIC_TYPE::SPEC::PARAMETERS::CRITIC_BATCH_SIZE, typename CORE_PARAMETERS::BATCH_SAMPLING_PARAMETERS, DYNAMIC_ALLOCATION>;
+        using ACTOR_BATCH_SPEC = rl::components::off_policy_runner::SequentialBatchSpecification<OFF_POLICY_RUNNER_SPEC, ACTOR_CRITIC_TYPE::SPEC::PARAMETERS::SEQUENCE_LENGTH, ACTOR_CRITIC_TYPE::SPEC::PARAMETERS::ACTOR_BATCH_SIZE, typename CORE_PARAMETERS::BATCH_SAMPLING_PARAMETERS, DYNAMIC_ALLOCATION>;
+        template <typename CONFIG>
+        using State = State<CONFIG>;
+    };
+}
+RL_TOOLS_NAMESPACE_WRAPPER_END
+
+#endif
+

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/algorithms/td3/loop/core/operations_generic.h

@@ -0,0 +1,174 @@
+#include "../../../../../version.h"
+#if (defined(RL_TOOLS_DISABLE_INCLUDE_GUARDS) || !defined(RL_TOOLS_RL_ALGORITHMS_TD3_LOOP_CORE_OPERATIONS_GENERIC_H)) && (RL_TOOLS_USE_THIS_VERSION == 1)
+#pragma once
+#define RL_TOOLS_RL_ALGORITHMS_TD3_LOOP_CORE_OPERATIONS_GENERIC_H
+
+#include "../../../../../nn/optimizers/adam/instance/operations_generic.h"
+#include "../../../../../nn/layers/td3_sampling/operations_generic.h"
+#include "../../../../../nn_models/operations_generic.h"
+#include "../../../../../nn_models/sequential/operations_generic.h"
+#include "../../../../../rl/algorithms/td3/operations_generic.h"
+#include "../../../../../nn_models/random_uniform/operations_generic.h"
+#include "../../../../../rl/components/off_policy_runner/operations_generic.h"
+#include "../../../../../random/operations_generic_array.h"
+
+#include "config.h"
+
+RL_TOOLS_NAMESPACE_WRAPPER_START
+namespace rl_tools{
+    template <typename DEVICE, typename T_CONFIG>
+    RL_TOOLS_FUNCTION_PLACEMENT void malloc(DEVICE& device, rl::algorithms::td3::loop::core::State<T_CONFIG>& ts){
+        malloc(device, ts.rng);
+        malloc(device, ts.actor_critic);
+        malloc(device, ts.off_policy_runner);
+        malloc(device, ts.critic_batch);
+        malloc(device, ts.critic_training_buffers);
+        malloc(device, ts.critic_buffers[0]);
+        malloc(device, ts.critic_buffers[1]);
+        malloc(device, ts.critic_target_buffers[0]);
+        malloc(device, ts.critic_target_buffers[1]);
+        malloc(device, ts.actor_batch);
+        malloc(device, ts.actor_training_buffers);
+        malloc(device, ts.actor_buffers_eval);
+        malloc(device, ts.actor_buffers[0]);
+        malloc(device, ts.actor_buffers[1]);
+        malloc(device, ts.actor_target_buffers[0]);
+        malloc(device, ts.actor_target_buffers[1]);
+        for(auto& env: ts.envs){
+            rl_tools::malloc(device, env);
+        }
+    }
+    template <typename DEVICE, typename T_CONFIG>
+    RL_TOOLS_FUNCTION_PLACEMENT void free(DEVICE& device, rl::algorithms::td3::loop::core::State<T_CONFIG>& ts){
+        free(device, ts.rng);
+        free(device, ts.actor_critic);
+        free(device, ts.off_policy_runner);
+        free(device, ts.critic_batch);
+        free(device, ts.critic_training_buffers);
+        free(device, ts.critic_buffers[0]);
+        free(device, ts.critic_buffers[1]);
+        free(device, ts.critic_target_buffers[0]);
+        free(device, ts.critic_target_buffers[1]);
+        free(device, ts.actor_batch);
+        free(device, ts.actor_training_buffers);
+        free(device, ts.actor_buffers_eval);
+        free(device, ts.actor_buffers[0]);
+        free(device, ts.actor_buffers[1]);
+        free(device, ts.actor_target_buffers[0]);
+        free(device, ts.actor_target_buffers[1]);
+        for(auto& env: ts.envs){
+            rl_tools::free(device, env);
+        }
+    }
+    template <typename DEVICE, typename T_CONFIG>
+    RL_TOOLS_FUNCTION_PLACEMENT void init(DEVICE& device, rl::algorithms::td3::loop::core::State<T_CONFIG>& ts, typename T_CONFIG::TI seed = 0){
+        using CONFIG = T_CONFIG;
+        using TYPE_POLICY = typename CONFIG::TYPE_POLICY;
+        using TI = typename DEVICE::index_t;
+
+        init(device, ts.rng, seed);
+
+        init(device, ts.actor_critic, ts.rng);
+
+        for(TI env_i = 0; env_i < CONFIG::CORE_PARAMETERS::N_ENVIRONMENTS; env_i ++){
+            rl_tools::init(device, ts.envs[env_i]);
+        }
+        init(device, ts.off_policy_runner);
+
+        ts.step = 0;
+    }
+
+
+    template <typename DEVICE, typename T_CONFIG>
+    RL_TOOLS_FUNCTION_PLACEMENT bool step(DEVICE& device, rl::algorithms::td3::loop::core::State<T_CONFIG>& ts){
+        using CONFIG = T_CONFIG;
+        set_step(device, device.logger, ts.step);
+        bool finished = false;
+        if(ts.step >= CONFIG::CORE_PARAMETERS::N_WARMUP_STEPS){
+            step<1>(device, ts.off_policy_runner, get_actor(ts), ts.actor_buffers_eval, ts.rng);
+        }
+        else{
+            typename CONFIG::EXPLORATION_POLICY exploration_policy;
+            typename CONFIG::EXPLORATION_POLICY::template Buffer<> exploration_policy_buffer;
+            step<0>(device, ts.off_policy_runner, exploration_policy, exploration_policy_buffer, ts.rng);
+        }
+
+        bool train_critic_flag = ts.step >= CONFIG::CORE_PARAMETERS::N_WARMUP_STEPS_CRITIC && ts.step % CONFIG::CORE_PARAMETERS::TD3_PARAMETERS::CRITIC_TRAINING_INTERVAL == 0;
+        bool update_critic_targets_flag = ts.step >= CONFIG::CORE_PARAMETERS::N_WARMUP_STEPS_CRITIC && ts.step % CONFIG::CORE_PARAMETERS::TD3_PARAMETERS::CRITIC_TARGET_UPDATE_INTERVAL == 0;
+        bool train_actor_flag = ts.step >= CONFIG::CORE_PARAMETERS::N_WARMUP_STEPS_ACTOR && ts.step % CONFIG::CORE_PARAMETERS::TD3_PARAMETERS::ACTOR_TRAINING_INTERVAL == 0;
+        bool update_actor_targets_flag = ts.step >= CONFIG::CORE_PARAMETERS::N_WARMUP_STEPS_ACTOR && ts.step % CONFIG::CORE_PARAMETERS::TD3_PARAMETERS::ACTOR_TARGET_UPDATE_INTERVAL == 0;
+
+        if(CONFIG::CORE_PARAMETERS::SHARED_BATCH && (train_critic_flag || train_actor_flag)){
+            gather_batch(device, ts.off_policy_runner, ts.critic_batch, ts.rng);
+            auto action_noise_matrix_view = matrix_view(device, ts.critic_training_buffers.target_next_action_noise);
+            target_action_noise(device, ts.actor_critic, action_noise_matrix_view, ts.rng);
+        }
+        if(train_critic_flag){
+            for(int critic_i = 0; critic_i < 2; critic_i++){
+                if constexpr(!CONFIG::CORE_PARAMETERS::SHARED_BATCH) {
+                    gather_batch(device, ts.off_policy_runner, ts.critic_batch, ts.rng);
+                    auto action_noise_matrix_view = matrix_view(device, ts.critic_training_buffers.target_next_action_noise);
+                    target_action_noise(device, ts.actor_critic, action_noise_matrix_view, ts.rng);
+                }
+                train_critic(device, ts.actor_critic, ts.actor_critic.critics[critic_i], ts.critic_batch, ts.actor_critic.critic_optimizers[critic_i], ts.actor_buffers[critic_i], ts.actor_target_buffers[critic_i], ts.critic_buffers[critic_i], ts.critic_target_buffers[critic_i], ts.critic_training_buffers, ts.rng);
+            }
+        }
+        if(update_critic_targets_flag){
+            update_critic_targets(device, ts.actor_critic);
+        }
+        if(train_actor_flag){
+            if constexpr(CONFIG::CORE_PARAMETERS::SHARED_BATCH) {
+                train_actor(device, ts.actor_critic, ts.critic_batch, ts.actor_critic.actor_optimizer, ts.actor_buffers[0], ts.critic_buffers[0], ts.actor_training_buffers, ts.rng);
+            }
+            else{
+                gather_batch(device, ts.off_policy_runner, ts.actor_batch, ts.rng);
+                train_actor(device, ts.actor_critic, ts.actor_batch, ts.actor_critic.actor_optimizer, ts.actor_buffers[0], ts.critic_buffers[0], ts.actor_training_buffers, ts.rng);
+            }
+        }
+        if(update_actor_targets_flag){
+            update_actor_target(device, ts.actor_critic);
+        }
+        ts.step++;
+        if(ts.step > CONFIG::CORE_PARAMETERS::STEP_LIMIT){
+            return true;
+        }
+        else{
+            return finished;
+        }
+    }
+    // the following operations are for nn_analytics iterating the neural networks
+    template <auto INDEX, typename DEVICE, typename T_CONFIG>
+    RL_TOOLS_FUNCTION_PLACEMENT constexpr auto& get_nn(DEVICE& device, rl::algorithms::td3::loop::core::State<T_CONFIG>& ts){
+        static_assert(INDEX < T_CONFIG::NUM_NNS, "Index out of bounds, there are only 3 neural networks in the TD3");
+        if constexpr(INDEX == 0){
+            return ts.actor_critic.actor;
+        }
+        else{
+            if constexpr(INDEX == 1){
+                return ts.actor_critic.critics[0];
+            }
+            else{
+                return ts.actor_critic.critics[1];
+            }
+        }
+    }
+    template <auto INDEX, typename DEVICE, typename T_CONFIG>
+    RL_TOOLS_FUNCTION_PLACEMENT constexpr auto& get_nn_name(DEVICE& device, rl::algorithms::td3::loop::core::State<T_CONFIG>& ts){
+        static_assert(INDEX < T_CONFIG::NUM_NNS, "Index out of bounds, there are only 3 neural networks in the TD3");
+        if constexpr(INDEX == 0){
+            return "actor";
+        }
+        else{
+            if constexpr(INDEX == 1){
+                return "critic[0]";
+            }
+            else{
+                return "critic[1]";
+            }
+        }
+    }
+}
+RL_TOOLS_NAMESPACE_WRAPPER_END
+
+
+#endif

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/algorithms/td3/loop/core/state.h

@@ -0,0 +1,46 @@
+#include "../../../../../version.h"
+#if (defined(RL_TOOLS_DISABLE_INCLUDE_GUARDS) || !defined(RL_TOOLS_RL_ALGORITHMS_TD3_LOOP_CORE_STATE_H)) && (RL_TOOLS_USE_THIS_VERSION == 1)
+#pragma once
+#define RL_TOOLS_RL_ALGORITHMS_TD3_LOOP_CORE_STATE_H
+
+#include "../../../../../rl/algorithms/td3/td3.h"
+#include "../../../../../rl/components/off_policy_runner/off_policy_runner.h"
+
+RL_TOOLS_NAMESPACE_WRAPPER_START
+namespace rl_tools{
+    namespace rl::algorithms::td3::loop::core{
+        // Config State (Init/Step)
+        template<typename T_CONFIG>
+        struct State{
+            using CONFIG = T_CONFIG;
+            using TI = typename CONFIG::TI;
+            typename CONFIG::RNG rng;
+            rl::components::OffPolicyRunner<typename CONFIG::OFF_POLICY_RUNNER_SPEC> off_policy_runner;
+            typename CONFIG::ENVIRONMENT envs[decltype(off_policy_runner)::N_ENVIRONMENTS];
+            typename CONFIG::ENVIRONMENT::Parameters env_parameters[decltype(off_policy_runner)::N_ENVIRONMENTS];
+            typename CONFIG::ACTOR_CRITIC_TYPE actor_critic;
+            // typename CONFIG::NN::ACTOR_TYPE::template Buffer<CONFIG::DYNAMIC_ALLOCATION> actor_deterministic_evaluation_buffers;
+//            rl::components::off_policy_runner::Batch<rl::components::off_policy_runner::BatchSpecification<typename decltype(off_policy_runner)::SPEC, CONFIG::ACTOR_CRITIC_TYPE::SPEC::PARAMETERS::CRITIC_BATCH_SIZE>> critic_batch;
+            rl::components::off_policy_runner::SequentialBatch<typename CONFIG::CRITIC_BATCH_SPEC> critic_batch;
+            rl::algorithms::td3::CriticTrainingBuffers<rl::algorithms::td3::CriticTrainingBuffersSpecification<typename CONFIG::ACTOR_CRITIC_SPEC, CONFIG::DYNAMIC_ALLOCATION>> critic_training_buffers;
+            typename CONFIG::NN::CRITIC_TYPE::template Buffer<CONFIG::DYNAMIC_ALLOCATION> critic_buffers[2];
+            using TARGET_CRITIC = typename rl_tools::utils::typing::remove_reference_t<decltype(actor_critic.critics_target[0])>::template CHANGE_SEQUENCE_LENGTH<TI, CONFIG::TARGET_SEQUENCE_LENGTH>;
+            typename TARGET_CRITIC::template Buffer<CONFIG::DYNAMIC_ALLOCATION> critic_target_buffers[2];
+            rl::components::off_policy_runner::SequentialBatch<typename CONFIG::ACTOR_BATCH_SPEC> actor_batch;
+//            rl::components::off_policy_runner::Batch<rl::components::off_policy_runner::BatchSpecification<typename decltype(off_policy_runner)::SPEC, CONFIG::ACTOR_CRITIC_TYPE::SPEC::PARAMETERS::ACTOR_BATCH_SIZE>> actor_batch;
+            rl::algorithms::td3::ActorTrainingBuffers<rl::algorithms::td3::ActorTrainingBuffersSpecification<typename CONFIG::ACTOR_CRITIC_TYPE::SPEC, CONFIG::DYNAMIC_ALLOCATION>> actor_training_buffers;
+            typename CONFIG::NN::ACTOR_TYPE::template Buffer<CONFIG::DYNAMIC_ALLOCATION> actor_buffers[2];
+            using TARGET_ACTOR = typename decltype(actor_critic.actor)::template CHANGE_SEQUENCE_LENGTH<TI, CONFIG::TARGET_SEQUENCE_LENGTH>;
+            typename TARGET_ACTOR::template Buffer<CONFIG::DYNAMIC_ALLOCATION> actor_target_buffers[2];
+            typename CONFIG::EVAL_ACTOR_TYPE::template Buffer<CONFIG::DYNAMIC_ALLOCATION> actor_buffers_eval;
+            TI step;
+        };
+    }
+    template <typename T_CONFIG>
+    constexpr auto& get_actor(rl::algorithms::td3::loop::core::State<T_CONFIG>& ts){
+        return ts.actor_critic.actor;
+    }
+
+}
+RL_TOOLS_NAMESPACE_WRAPPER_END
+#endif

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/algorithms/td3/operations_cpu.h

@@ -0,0 +1,9 @@
+#include "../../../version.h"
+#if (defined(RL_TOOLS_DISABLE_INCLUDE_GUARDS) || !defined(RL_TOOLS_RL_ALGORITHMS_TD3_OPERATIONS_CPU_H)) && (RL_TOOLS_USE_THIS_VERSION == 1)
+#pragma once
+#define RL_TOOLS_RL_ALGORITHMS_TD3_OPERATIONS_CPU_H
+
+#include "operations_generic.h"
+#include "../../../rl/components/operations_cpu.h"
+
+#endif

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/algorithms/td3/operations_cpu_accelerate.h

@@ -0,0 +1,2 @@
+#include "operations_generic.h"
+#include "../../../rl/components/operations_cpu_accelerate.h"

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/algorithms/td3/operations_cpu_mkl.h

@@ -0,0 +1,2 @@
+#include "operations_generic.h"
+#include "../../../rl/components/operations_cpu_mkl.h"

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/algorithms/td3/operations_cpu_mux.h

@@ -0,0 +1,9 @@
+#if defined(RL_TOOLS_BACKEND_ENABLE_MKL) && !defined(RL_TOOLS_BACKEND_DISABLE_BLAS)
+#include "../../../rl/algorithms/td3/operations_cpu_mkl.h"
+#else
+#if defined(RL_TOOLS_BACKEND_ENABLE_ACCELERATE) && !defined(RL_TOOLS_BACKEND_DISABLE_BLAS)
+#include "../../../rl/algorithms/td3/operations_cpu_accelerate.h"
+#else
+#include "../../../rl/algorithms/td3/operations_cpu.h"
+#endif
+#endif