hyperray 0.0.1__py3-none-any.whl

hyperdrone/_vendor/rl-tools/include/rl_tools/rendering/types.h

@@ -0,0 +1,76 @@
+#include "../version.h"
+#if (defined(RL_TOOLS_DISABLE_INCLUDE_GUARDS) || !defined(RL_TOOLS_RENDERING_TYPES_H)) && (RL_TOOLS_USE_THIS_VERSION == 1)
+#pragma once
+#define RL_TOOLS_RENDERING_TYPES_H
+
+#include "../rl_tools.h"
+
+RL_TOOLS_NAMESPACE_WRAPPER_START
+namespace rl_tools {
+    // Byte copies keep the IEEE-754 miss sentinel defined under fast-math without aliasing float storage.
+    RL_TOOLS_FUNCTION_PLACEMENT inline void store_miss(float& value){
+        static_assert(sizeof(float) == 4 && sizeof(unsigned int) == 4, "rendering requires 32-bit float and unsigned int");
+        const unsigned int bits = 0x7f800000u;
+        const auto* source = reinterpret_cast<const unsigned char*>(&bits);
+        auto* destination = reinterpret_cast<unsigned char*>(&value);
+        for(unsigned int i = 0; i < sizeof(value); i++){
+            destination[i] = source[i];
+        }
+    }
+
+    RL_TOOLS_FUNCTION_PLACEMENT inline bool is_miss(const float& value){
+        unsigned int bits = 0;
+        const auto* source = reinterpret_cast<const unsigned char*>(&value);
+        auto* destination = reinterpret_cast<unsigned char*>(&bits);
+        for(unsigned int i = 0; i < sizeof(value); i++){
+            destination[i] = source[i];
+        }
+        return bits == 0x7f800000u;
+    }
+
+    namespace rendering{
+        struct SceneLight {
+            int type; // 0=directional, 1=point, 2=spot
+            float position[3];
+            float direction[3];
+            float color[3];
+            float attenuation_constant;
+            float attenuation_linear;
+            float attenuation_quadratic;
+            float cos_inner_cone;
+            float cos_outer_cone;
+        };
+
+        struct CollisionResult {
+            float distance;
+            int hit;
+        };
+
+        template <
+            bool T_LOAD_TEXTURES,
+            bool T_NORMAL_SHADING,
+            bool T_METALLIC_REFLECTIONS,
+            bool T_SRGB_OUTPUT,
+            bool T_CHECKER_BACKGROUND,
+            bool T_PBR_SHADING,
+            bool T_PUNCTUAL_LIGHT_SHADOWS = false
+        >
+        struct ShadingOptions{
+            static constexpr bool LOAD_TEXTURES = T_LOAD_TEXTURES;
+            static constexpr bool NORMAL_SHADING = T_NORMAL_SHADING;
+            static constexpr bool METALLIC_REFLECTIONS = T_METALLIC_REFLECTIONS;
+            static constexpr bool SRGB_OUTPUT = T_SRGB_OUTPUT;
+            static constexpr bool CHECKER_BACKGROUND = T_CHECKER_BACKGROUND;
+            static constexpr bool PBR_SHADING = T_PBR_SHADING;
+            static constexpr bool PUNCTUAL_LIGHT_SHADOWS = T_PUNCTUAL_LIGHT_SHADOWS;
+        };
+
+        using Medium = ShadingOptions<true, true, true, true, true, false>;
+        using High = ShadingOptions<true, true, true, true, false, true>;
+        using VeryHigh = ShadingOptions<true, true, true, true, false, true, true>;
+        using Low = ShadingOptions<false, false, false, true, false, false>;
+    }
+}
+RL_TOOLS_NAMESPACE_WRAPPER_END
+
+#endif

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/algorithms/algorithms.h

@@ -0,0 +1,6 @@
+#include "ppo/ppo.h"
+#include "ppo/loop/core/config.h"
+#include "sac/sac.h"
+#include "sac/loop/core/config.h"
+#include "td3/td3.h"
+#include "td3/loop/core/config.h"

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/algorithms/operations_cpu.h

@@ -0,0 +1 @@
+#include "td3/operations_cpu_mkl.h"

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/algorithms/operations_cpu_mkl.h

@@ -0,0 +1 @@
+#include "td3/operations_cpu_mkl.h

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/algorithms/operations_generic.h

@@ -0,0 +1 @@
+//#include "td3/operations_generic.h"

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/algorithms/ppo/loop/core/config.h

@@ -0,0 +1,243 @@
+#include "../../../../../version.h"
+#if (defined(RL_TOOLS_DISABLE_INCLUDE_GUARDS) || !defined(RL_TOOLS_RL_ALGORITHMS_PPO_LOOP_CORE_CONFIG_H)) && (RL_TOOLS_USE_THIS_VERSION == 1)
+#pragma once
+#define RL_TOOLS_RL_ALGORITHMS_PPO_LOOP_CORE_CONFIG_H
+
+#include "../../../../../rl/environments/observation.h"
+#include "../../../../../nn/layers/standardize/layer.h"
+#include "../../../../../nn/layers/gru/layer.h"
+#include "../../../../../nn_models/sequential/model.h"
+#include "../../../../../nn_models/mlp_unconditional_stddev/network.h"
+#include "../../../../../nn_models/multi_agent_wrapper/model.h"
+#include "../../../../../rl/algorithms/ppo/ppo.h"
+#include "../../../../../rl/components/on_policy_runner/collection.h"
+#include "../../../../../rl/environments/batch/environment.h"
+#include "../../../../../rl/components/on_policy_runner/on_policy_runner.h"
+#include "../../../../../nn/optimizers/adam/adam.h"
+#include "../../../../../rl/loop/loop.h"
+#include "state.h"
+
+RL_TOOLS_NAMESPACE_WRAPPER_START
+namespace rl_tools{
+    namespace rl::algorithms::ppo::loop::core{
+        // Config State (Init/Step)
+
+        struct ParametersTag{};
+        template<typename TYPE_POLICY, typename TI, typename ENVIRONMENT>
+        struct DefaultParameters{
+            using T = typename TYPE_POLICY::DEFAULT;
+            using TAG = ParametersTag;
+            static constexpr TI STEP_LIMIT = 100;
+
+            static constexpr TI ACTOR_HIDDEN_DIM = 64;
+            static constexpr TI ACTOR_NUM_LAYERS = 3;
+            static constexpr auto ACTOR_ACTIVATION_FUNCTION = nn::activation_functions::ActivationFunction::RELU;
+            static constexpr TI CRITIC_HIDDEN_DIM = 64;
+            static constexpr TI CRITIC_NUM_LAYERS = 3;
+            static constexpr auto CRITIC_ACTIVATION_FUNCTION = nn::activation_functions::ActivationFunction::RELU;
+
+            static constexpr TI EPISODE_STEP_LIMIT = ENVIRONMENT::EPISODE_STEP_LIMIT;
+            static constexpr TI N_ENVIRONMENTS = 64;
+            static constexpr TI ON_POLICY_RUNNER_STEPS_PER_ENV = 64;
+            static constexpr TI DATASET_SIZE = ON_POLICY_RUNNER_STEPS_PER_ENV * N_ENVIRONMENTS;
+            static constexpr TI BATCH_SIZE = 64 * 8;
+
+            static constexpr bool NORMALIZE_OBSERVATIONS = false;
+            static constexpr bool NORMALIZE_OBSERVATIONS_CONTINUOUSLY = false;
+
+            using ACTOR_OPTIMIZER_PARAMETERS = nn::optimizers::adam::DEFAULT_PARAMETERS_TENSORFLOW<TYPE_POLICY>;
+            using CRITIC_OPTIMIZER_PARAMETERS = ACTOR_OPTIMIZER_PARAMETERS;
+
+            using PPO_PARAMETERS = rl::algorithms::ppo::DefaultParameters<TYPE_POLICY, TI, BATCH_SIZE>;
+        };
+
+        template<typename TYPE_POLICY, typename TI, typename ENVIRONMENT, typename PARAMETERS, bool DYNAMIC_ALLOCATION=true>
+        struct ConfigApproximatorsSequential{
+            static constexpr TI STEPS = PARAMETERS::PPO_PARAMETERS::STATEFUL_ACTOR_AND_CRITIC ? PARAMETERS::ON_POLICY_RUNNER_STEPS_PER_ENV : 1;
+            static constexpr TI FORWARD_BATCH_SIZE = PARAMETERS::PPO_PARAMETERS::STATEFUL_ACTOR_AND_CRITIC ? PARAMETERS::N_ENVIRONMENTS : PARAMETERS::BATCH_SIZE;
+            template <typename CAPABILITY>
+            struct Actor{
+                using OBS_SHAPE = typename ENVIRONMENT::Observation::SHAPE;
+                using INPUT_SHAPE = tensor::Prepend<tensor::Prepend<OBS_SHAPE, FORWARD_BATCH_SIZE>, STEPS>;
+                using STANDARDIZATION_LAYER_CONFIG = nn::layers::standardize::Configuration<TYPE_POLICY, TI>;
+                using STANDARDIZATION_LAYER = nn::layers::standardize::BindConfiguration<STANDARDIZATION_LAYER_CONFIG>;
+                using CONFIG = nn_models::mlp::Configuration<TYPE_POLICY, TI, ENVIRONMENT::ACTION_DIM, PARAMETERS::ACTOR_NUM_LAYERS, PARAMETERS::ACTOR_HIDDEN_DIM, PARAMETERS::ACTOR_ACTIVATION_FUNCTION,  nn::activation_functions::IDENTITY>;
+                using TYPE = nn_models::mlp_unconditional_stddev::BindConfiguration<CONFIG>;
+
+                using MODULE_CHAIN = nn_models::sequential::Module<STANDARDIZATION_LAYER, nn_models::sequential::Module<TYPE>>;
+                using MODEL = nn_models::sequential::Build<CAPABILITY, MODULE_CHAIN, INPUT_SHAPE>;
+            };
+            template <typename CAPABILITY>
+            struct Critic{
+                using OBS_PRIV_SHAPE = typename ENVIRONMENT::ObservationPrivileged::SHAPE;
+                using INPUT_SHAPE = tensor::Prepend<tensor::Prepend<OBS_PRIV_SHAPE, FORWARD_BATCH_SIZE>, STEPS>;
+                using STANDARDIZATION_LAYER_CONFIG = nn::layers::standardize::Configuration<TYPE_POLICY, TI>;
+                using STANDARDIZATION_LAYER = nn::layers::standardize::BindConfiguration<STANDARDIZATION_LAYER_CONFIG>;
+                using CONFIG = nn_models::mlp::Configuration<TYPE_POLICY, TI, 1, PARAMETERS::CRITIC_NUM_LAYERS, PARAMETERS::CRITIC_HIDDEN_DIM, PARAMETERS::CRITIC_ACTIVATION_FUNCTION, nn::activation_functions::IDENTITY>;
+                using TYPE = nn_models::mlp_unconditional_stddev::BindConfiguration<CONFIG>;
+
+                using MODULE_CHAIN = nn_models::sequential::Module<STANDARDIZATION_LAYER, nn_models::sequential::Module<TYPE>>;
+                using MODEL = nn_models::sequential::Build<CAPABILITY, MODULE_CHAIN, INPUT_SHAPE>;
+            };
+
+            using ACTOR_OPTIMIZER_SPEC = nn::optimizers::adam::Specification<TYPE_POLICY, TI, typename PARAMETERS::ACTOR_OPTIMIZER_PARAMETERS, DYNAMIC_ALLOCATION>;
+            using CRITIC_OPTIMIZER_SPEC = nn::optimizers::adam::Specification<TYPE_POLICY, TI, typename PARAMETERS::CRITIC_OPTIMIZER_PARAMETERS, DYNAMIC_ALLOCATION>;
+            using ACTOR_OPTIMIZER = nn::optimizers::Adam<ACTOR_OPTIMIZER_SPEC>;
+            using CRITIC_OPTIMIZER = nn::optimizers::Adam<CRITIC_OPTIMIZER_SPEC>;
+            using CAPABILITY_ADAM = nn::capability::Gradient<nn::parameters::Adam, DYNAMIC_ALLOCATION>;
+            using ACTOR_TYPE = typename Actor<CAPABILITY_ADAM>::MODEL;
+            using CRITIC_TYPE = typename Critic<CAPABILITY_ADAM>::MODEL;
+        };
+        template <bool T_GRU_CRITIC = false>
+        struct ConfigApproximatorsGRU{
+            template<typename TYPE_POLICY, typename TI, typename ENVIRONMENT, typename PARAMETERS, bool DYNAMIC_ALLOCATION=true>
+            struct Approximators{
+                static constexpr bool USE_GRU = true;
+                using PPO_PARAMETERS = typename PARAMETERS::PPO_PARAMETERS;
+                static_assert(PPO_PARAMETERS::SHUFFLE_EPOCH == false, "When using sequence models for the actor and critic, SHUFFLE_EPOCH has to be disabled (in the PPO parameters) because it scrambles the episodes.");
+                static_assert(PPO_PARAMETERS::STATEFUL_ACTOR_AND_CRITIC == true, "When using sequence models for the actor and critic, STATEFUL_ACTOR_AND_CRITIC has to be enabled.");
+                template <typename CAPABILITY>
+                struct Actor{
+                    using OBS_SHAPE = typename ENVIRONMENT::Observation::SHAPE;
+                    using INPUT_SHAPE = tensor::Prepend<tensor::Prepend<OBS_SHAPE, PARAMETERS::N_ENVIRONMENTS>, PARAMETERS::ON_POLICY_RUNNER_STEPS_PER_ENV>;
+                    using STANDARDIZATION_LAYER_CONFIG = nn::layers::standardize::Configuration<TYPE_POLICY, TI>;
+                    using STANDARDIZATION_LAYER = nn::layers::standardize::BindConfiguration<STANDARDIZATION_LAYER_CONFIG>;
+                    using INPUT_LAYER_CONFIG = nn::layers::dense::Configuration<TYPE_POLICY, TI, PARAMETERS::ACTOR_HIDDEN_DIM, PARAMETERS::ACTOR_ACTIVATION_FUNCTION, nn::layers::dense::DefaultInitializer<TYPE_POLICY, TI>, nn::parameters::groups::Input>;
+                    using INPUT_LAYER = nn::layers::dense::BindConfiguration<INPUT_LAYER_CONFIG>;
+                    static constexpr bool FAST_TANH = false;
+                    using GRU_SPEC = nn::layers::gru::Configuration<TYPE_POLICY, TI, PARAMETERS::ACTOR_HIDDEN_DIM, nn::parameters::groups::Normal, FAST_TANH>;
+                    using GRU = nn::layers::gru::BindConfiguration<GRU_SPEC>;
+                    using DENSE_LAYER_CONFIG = nn::layers::dense::Configuration<TYPE_POLICY, TI, PARAMETERS::ACTOR_HIDDEN_DIM, PARAMETERS::ACTOR_ACTIVATION_FUNCTION, nn::layers::dense::DefaultInitializer<TYPE_POLICY, TI>, nn::parameters::groups::Normal>;
+                    using DENSE_LAYER = nn::layers::dense::BindConfiguration<DENSE_LAYER_CONFIG>;
+                    using OUTPUT_LAYER_CONFIG = nn::layers::dense::Configuration<TYPE_POLICY, TI, ENVIRONMENT::ACTION_DIM, nn::activation_functions::IDENTITY, nn::layers::dense::DefaultInitializer<TYPE_POLICY, TI>, nn::parameters::groups::Output>;
+                    using OUTPUT_LAYER = nn::layers::dense::BindConfiguration<OUTPUT_LAYER_CONFIG>;
+                    using CONFIG = nn_models::mlp::Configuration<TYPE_POLICY, TI, ENVIRONMENT::ACTION_DIM, PARAMETERS::ACTOR_NUM_LAYERS, PARAMETERS::ACTOR_HIDDEN_DIM, PARAMETERS::ACTOR_ACTIVATION_FUNCTION,  nn::activation_functions::IDENTITY>;
+                    using MLP = nn_models::mlp_unconditional_stddev::BindConfiguration<CONFIG>;
+
+                    using MODULE = nn_models::sequential::Module<STANDARDIZATION_LAYER, nn_models::sequential::Module<INPUT_LAYER, nn_models::sequential::Module<GRU, nn_models::sequential::Module<MLP>>>>;
+                    using MODEL = nn_models::sequential::Build<CAPABILITY, MODULE, INPUT_SHAPE>;
+                };
+                template <typename CAPABILITY>
+                struct Critic{
+                    using OBS_PRIV_SHAPE = typename ENVIRONMENT::ObservationPrivileged::SHAPE;
+                    using INPUT_SHAPE = tensor::Prepend<tensor::Prepend<OBS_PRIV_SHAPE, PARAMETERS::N_ENVIRONMENTS>, PARAMETERS::ON_POLICY_RUNNER_STEPS_PER_ENV>;
+                    using STANDARDIZATION_LAYER_CONFIG = nn::layers::standardize::Configuration<TYPE_POLICY, TI>;
+                    using STANDARDIZATION_LAYER = nn::layers::standardize::BindConfiguration<STANDARDIZATION_LAYER_CONFIG>;
+                    using INPUT_LAYER_CONFIG = nn::layers::dense::Configuration<TYPE_POLICY, TI, PARAMETERS::CRITIC_HIDDEN_DIM, PARAMETERS::CRITIC_ACTIVATION_FUNCTION, nn::layers::dense::DefaultInitializer<TYPE_POLICY, TI>, nn::parameters::groups::Input>;
+                    using INPUT_LAYER = nn::layers::dense::BindConfiguration<INPUT_LAYER_CONFIG>;
+                    static constexpr bool FAST_TANH = false;
+                    using GRU_SPEC = nn::layers::gru::Configuration<TYPE_POLICY, TI, PARAMETERS::CRITIC_HIDDEN_DIM, nn::parameters::groups::Normal, FAST_TANH>;
+                    using GRU = nn::layers::gru::BindConfiguration<GRU_SPEC>;
+                    using DENSE_LAYER_CONFIG = nn::layers::dense::Configuration<TYPE_POLICY, TI, PARAMETERS::CRITIC_HIDDEN_DIM, PARAMETERS::CRITIC_ACTIVATION_FUNCTION, nn::layers::dense::DefaultInitializer<TYPE_POLICY, TI>, nn::parameters::groups::Normal>;
+                    using DENSE_LAYER = nn::layers::dense::BindConfiguration<DENSE_LAYER_CONFIG>;
+                    using OUTPUT_LAYER_CONFIG = nn::layers::dense::Configuration<TYPE_POLICY, TI, 1, nn::activation_functions::IDENTITY, nn::layers::dense::DefaultInitializer<TYPE_POLICY, TI>, nn::parameters::groups::Output>;
+                    using OUTPUT_LAYER = nn::layers::dense::BindConfiguration<OUTPUT_LAYER_CONFIG>;
+                    using CONFIG = nn_models::mlp::Configuration<TYPE_POLICY, TI, 1, PARAMETERS::CRITIC_NUM_LAYERS, PARAMETERS::CRITIC_HIDDEN_DIM, PARAMETERS::CRITIC_ACTIVATION_FUNCTION, nn::activation_functions::IDENTITY>;
+                    using MLP = nn_models::mlp::BindConfiguration<CONFIG>;
+                    using MODULE_DENSE = nn_models::sequential::Module<STANDARDIZATION_LAYER, nn_models::sequential::Module<INPUT_LAYER, nn_models::sequential::Module<DENSE_LAYER, nn_models::sequential::Module<MLP>>>>;
+                    using MODULE_GRU = nn_models::sequential::Module<STANDARDIZATION_LAYER, nn_models::sequential::Module<INPUT_LAYER, nn_models::sequential::Module<GRU, nn_models::sequential::Module<MLP>>>>;
+                    using MODULE = rl_tools::utils::typing::conditional_t<T_GRU_CRITIC, MODULE_GRU, MODULE_DENSE>;
+                    using MODEL = nn_models::sequential::Build<CAPABILITY, MODULE, INPUT_SHAPE>;
+                };
+
+                using ACTOR_OPTIMIZER_SPEC = nn::optimizers::adam::Specification<TYPE_POLICY, TI, typename PARAMETERS::ACTOR_OPTIMIZER_PARAMETERS, DYNAMIC_ALLOCATION>;
+                using CRITIC_OPTIMIZER_SPEC = nn::optimizers::adam::Specification<TYPE_POLICY, TI, typename PARAMETERS::CRITIC_OPTIMIZER_PARAMETERS, DYNAMIC_ALLOCATION>;
+                using ACTOR_OPTIMIZER = nn::optimizers::Adam<ACTOR_OPTIMIZER_SPEC>;
+                using CRITIC_OPTIMIZER = nn::optimizers::Adam<CRITIC_OPTIMIZER_SPEC>;
+                using CAPABILITY_ADAM = nn::capability::Gradient<nn::parameters::Adam, DYNAMIC_ALLOCATION>;
+                using ACTOR_TYPE = typename Actor<CAPABILITY_ADAM>::MODEL;
+                using CRITIC_TYPE = typename Critic<CAPABILITY_ADAM>::MODEL;
+            };
+        };
+
+        template<typename TYPE_POLICY, typename TI, typename ENVIRONMENT, typename PARAMETERS, bool DYNAMIC_ALLOCATION=true>
+        struct ConfigApproximatorsSequentialMultiAgent{
+            template <typename CAPABILITY>
+            struct Actor{
+                static constexpr TI N_AGENTS = ENVIRONMENT::N_AGENTS;
+                static_assert(ENVIRONMENT::Observation::DIM % N_AGENTS == 0);
+                static_assert(ENVIRONMENT::ACTION_DIM % N_AGENTS == 0);
+                using OBS_SHAPE = typename ENVIRONMENT::Observation::SHAPE;
+                using INPUT_SHAPE = tensor::Prepend<tensor::Prepend<OBS_SHAPE, PARAMETERS::BATCH_SIZE>, 1>;
+                using STANDARDIZATION_LAYER_CONFIG = nn::layers::standardize::Configuration<TYPE_POLICY, TI>;
+                using STANDARDIZATION_LAYER = nn::layers::standardize::BindConfiguration<STANDARDIZATION_LAYER_CONFIG>;
+                using CONFIG = nn_models::mlp::Configuration<TYPE_POLICY, TI, ENVIRONMENT::ACTION_DIM/N_AGENTS, PARAMETERS::ACTOR_NUM_LAYERS, PARAMETERS::ACTOR_HIDDEN_DIM, PARAMETERS::ACTOR_ACTIVATION_FUNCTION,  nn::activation_functions::IDENTITY>;
+                using TYPE = nn_models::mlp_unconditional_stddev::BindConfiguration<CONFIG>;
+
+                using INNER_MODULE_CHAIN = nn_models::sequential::Module<STANDARDIZATION_LAYER, nn_models::sequential::Module<TYPE>>;
+                using WRAPPER_CONFIG = nn_models::multi_agent_wrapper::Configuration<TYPE_POLICY, TI, N_AGENTS, INNER_MODULE_CHAIN>;
+                using MODEL = nn_models::multi_agent_wrapper::Build<CAPABILITY, WRAPPER_CONFIG, INPUT_SHAPE>;
+            };
+            template <typename CAPABILITY>
+            struct Critic{
+                using OBS_PRIV_SHAPE = typename ENVIRONMENT::ObservationPrivileged::SHAPE;
+                using INPUT_SHAPE = tensor::Prepend<tensor::Prepend<OBS_PRIV_SHAPE, PARAMETERS::BATCH_SIZE>, 1>;
+                using CONFIG = nn_models::mlp::Configuration<TYPE_POLICY, TI, 1, PARAMETERS::CRITIC_NUM_LAYERS, PARAMETERS::CRITIC_HIDDEN_DIM, PARAMETERS::CRITIC_ACTIVATION_FUNCTION, nn::activation_functions::IDENTITY>;
+                using TYPE = nn_models::mlp_unconditional_stddev::BindConfiguration<CONFIG>;
+                using STANDARDIZATION_LAYER_SPEC = nn::layers::standardize::Configuration<TYPE_POLICY, TI>;
+                using STANDARDIZATION_LAYER = nn::layers::standardize::BindConfiguration<STANDARDIZATION_LAYER_SPEC>;
+                using MODULE_CHAIN = nn_models::sequential::Module<STANDARDIZATION_LAYER, nn_models::sequential::Module<TYPE>>;
+                using MODEL = nn_models::sequential::Build<CAPABILITY, MODULE_CHAIN, INPUT_SHAPE>;
+
+
+//                using WRAPPER_CONFIG = nn_models::multi_agent_wrapper::Configuration<T, TI, N_AGENTS, INNER_MODULE_CHAIN>;
+//                using WRAPPER = nn_models::multi_agent_wrapper::BindConfiguration<WRAPPER_CONFIG>;
+//                using MODULE_CHAIN = Module<WRAPPER>;
+//                using MODEL = nn_models::multi_agent_wrapper::Module<WRAPPER_CONFIG, CAPABILITY, INPUT_SHAPE>;
+            };
+
+            using ACTOR_OPTIMIZER_SPEC = nn::optimizers::adam::Specification<TYPE_POLICY, TI, typename PARAMETERS::ACTOR_OPTIMIZER_PARAMETERS>;
+            using CRITIC_OPTIMIZER_SPEC = nn::optimizers::adam::Specification<TYPE_POLICY, TI, typename PARAMETERS::CRITIC_OPTIMIZER_PARAMETERS>;
+            using ACTOR_OPTIMIZER = nn::optimizers::Adam<ACTOR_OPTIMIZER_SPEC>;
+            using CRITIC_OPTIMIZER = nn::optimizers::Adam<CRITIC_OPTIMIZER_SPEC>;
+            using CAPABILITY_ADAM = nn::capability::Gradient<nn::parameters::Adam, DYNAMIC_ALLOCATION>;
+            using ACTOR_TYPE = typename Actor<CAPABILITY_ADAM>::MODEL;
+            using CRITIC_TYPE = typename Critic<CAPABILITY_ADAM>::MODEL;
+        };
+
+        struct ConfigTag{};
+        template<typename T_TYPE_POLICY, typename T_TI, typename T_RNG, typename T_ENVIRONMENT, typename T_PARAMETERS = DefaultParameters<T_TYPE_POLICY, T_TI, T_ENVIRONMENT>, template<typename, typename, typename, typename, bool> class APPROXIMATOR_CONFIG=ConfigApproximatorsSequential, bool T_DYNAMIC_ALLOCATION=true>
+        struct Config: rl::loop::Config{
+            using TAG = ConfigTag;
+            using TYPE_POLICY = T_TYPE_POLICY;
+            using TI = T_TI;
+            using RNG = T_RNG;
+            using ENVIRONMENT = T_ENVIRONMENT;
+            using ENVIRONMENT_EVALUATION = T_ENVIRONMENT;
+            using CORE_PARAMETERS = T_PARAMETERS;
+            static constexpr bool DYNAMIC_ALLOCATION = T_DYNAMIC_ALLOCATION;
+
+            static constexpr TI ENVIRONMENT_STEPS_PER_LOOP_STEP = CORE_PARAMETERS::N_ENVIRONMENTS * CORE_PARAMETERS::ON_POLICY_RUNNER_STEPS_PER_ENV;
+
+            using NN = APPROXIMATOR_CONFIG<TYPE_POLICY, TI, ENVIRONMENT, CORE_PARAMETERS, DYNAMIC_ALLOCATION>;
+
+
+            using T = typename TYPE_POLICY::DEFAULT;
+            static constexpr T OBSERVATION_NORMALIZATION_WARMUP_STEPS = CORE_PARAMETERS::NORMALIZE_OBSERVATIONS ? 1 : 0;
+            using PPO_SPEC = rl::algorithms::ppo::Specification<TYPE_POLICY, TI, ENVIRONMENT, typename NN::ACTOR_TYPE, typename NN::CRITIC_TYPE, typename CORE_PARAMETERS::PPO_PARAMETERS>;
+            using PPO_TYPE = rl::algorithms::PPO<PPO_SPEC>;
+            using PPO_BUFFERS_TYPE = rl::algorithms::ppo::Buffers<rl::algorithms::ppo::BufferSpecification<PPO_SPEC, DYNAMIC_ALLOCATION>>;
+            static constexpr TI NUM_NNS = 2;
+
+            using BATCH_ENVIRONMENT_SPEC = rl::environments::batch::Specification<ENVIRONMENT, CORE_PARAMETERS::N_ENVIRONMENTS, DYNAMIC_ALLOCATION>;
+            using BATCH_ENVIRONMENT = rl::environments::batch::Independent<BATCH_ENVIRONMENT_SPEC>;
+            using ON_POLICY_RUNNER_SPEC = rl::components::on_policy_runner::Specification<TYPE_POLICY, BATCH_ENVIRONMENT, typename NN::ACTOR_TYPE::template State<DYNAMIC_ALLOCATION>, typename ENVIRONMENT::Observation, typename ENVIRONMENT::ObservationPrivileged, T, T, CORE_PARAMETERS::EPISODE_STEP_LIMIT, CORE_PARAMETERS::PPO_PARAMETERS::TRUNCATE_ON_EACH_ITERATION, DYNAMIC_ALLOCATION, CORE_PARAMETERS::PPO_PARAMETERS::BOOTSTRAP_TRUNCATIONS || CORE_PARAMETERS::PPO_PARAMETERS::IGNORE_TERMINATION>;
+            using ON_POLICY_RUNNER_TYPE = rl::components::OnPolicyRunner<ON_POLICY_RUNNER_SPEC>;
+            using ON_POLICY_RUNNER_BUFFER_TYPE = rl::components::on_policy_runner::Buffer<ON_POLICY_RUNNER_SPEC>;
+            using ON_POLICY_RUNNER_DATASET_SPEC = rl::components::on_policy_runner::DatasetSpecification<ON_POLICY_RUNNER_SPEC, CORE_PARAMETERS::ON_POLICY_RUNNER_STEPS_PER_ENV, DYNAMIC_ALLOCATION>;
+            using ON_POLICY_RUNNER_DATASET_TYPE = rl::components::on_policy_runner::Dataset<ON_POLICY_RUNNER_DATASET_SPEC>;
+
+
+            using ACTOR_EVAL_BUFFERS = typename NN::ACTOR_TYPE::template Buffer<DYNAMIC_ALLOCATION>;
+            using ACTOR_BUFFERS = typename NN::ACTOR_TYPE::template Buffer<DYNAMIC_ALLOCATION>;
+            using CRITIC_BUFFERS = typename NN::CRITIC_TYPE::template Buffer<DYNAMIC_ALLOCATION>;
+            using CRITIC_COLLECTION_STATE = rl::components::on_policy_runner::ValueState<typename NN::CRITIC_TYPE, ON_POLICY_RUNNER_DATASET_SPEC>;
+            using CRITIC_COLLECTION_BUFFER = rl::components::on_policy_runner::ValueBuffer<typename NN::CRITIC_TYPE, ON_POLICY_RUNNER_DATASET_SPEC>;
+            template <typename CONFIG>
+            using State = State<CONFIG>;
+        };
+    }
+}
+RL_TOOLS_NAMESPACE_WRAPPER_END
+
+#endif

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/algorithms/ppo/loop/core/operations_cuda.h

@@ -0,0 +1,9 @@
+#include "../../../../../version.h"
+#if (defined(RL_TOOLS_DISABLE_INCLUDE_GUARDS) || !defined(RL_TOOLS_RL_ALGORITHMS_PPO_LOOP_CORE_OPERATIONS_CUDA_H)) && (RL_TOOLS_USE_THIS_VERSION == 1)
+#pragma once
+#define RL_TOOLS_RL_ALGORITHMS_PPO_LOOP_CORE_OPERATIONS_CUDA_H
+
+#include "../../../../../rl/environments/batch/operations_cuda.h"
+#include "../../../../../rl/components/on_policy_runner/operations_cuda.h"
+#include "operations_generic.h"
+#endif

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/algorithms/ppo/loop/core/operations_generic.h

@@ -0,0 +1,205 @@
+#include "../../../../../version.h"
+#if (defined(RL_TOOLS_DISABLE_INCLUDE_GUARDS) || !defined(RL_TOOLS_RL_ALGORITHMS_PPO_LOOP_CORE_OPERATIONS_GENERIC_H)) && (RL_TOOLS_USE_THIS_VERSION == 1)
+#pragma once
+#define RL_TOOLS_RL_ALGORITHMS_PPO_LOOP_CORE_OPERATIONS_GENERIC_H
+
+#include "../../../../../nn/optimizers/adam/instance/operations_generic.h"
+#include "../../../../../nn/layers/standardize/operations_generic.h"
+#include "../../../../../nn_models/mlp_unconditional_stddev/operations_generic.h"
+#include "../../../../../nn_models/sequential/operations_generic.h"
+#include "../../../../../nn/optimizers/adam/operations_generic.h"
+#include "../../../../../rl/algorithms/ppo/operations_generic.h"
+#include "../../../../../rl/environments/batch/operations_generic.h"
+#include "../../../../../rl/components/on_policy_runner/operations_generic.h"
+#include "../../../../../random/operations_generic_array.h"
+
+#include "config.h"
+
+RL_TOOLS_NAMESPACE_WRAPPER_START
+namespace rl_tools{
+    template <typename DEVICE, typename T_CONFIG>
+    RL_TOOLS_FUNCTION_PLACEMENT void malloc(DEVICE& device, rl::algorithms::ppo::loop::core::State<T_CONFIG>& ts){
+        malloc(device, ts.rng);
+        malloc(device, ts.ppo);
+        malloc(device, ts.ppo_buffers);
+        malloc(device, ts.environment);
+        malloc(device, ts.on_policy_runner_dataset);
+        malloc(device, ts.on_policy_runner);
+        malloc(device, ts.on_policy_runner_buffer);
+        malloc(device, ts.actor_eval_buffers);
+        malloc(device, ts.actor_buffers);
+        malloc(device, ts.critic_buffers);
+        malloc(device, ts.critic_collection_state);
+        malloc(device, ts.critic_collection_buffer);
+        malloc(device, ts.actor_optimizer);
+        malloc(device, ts.critic_optimizer);
+    }
+    template <typename DEVICE, typename T_CONFIG>
+    RL_TOOLS_FUNCTION_PLACEMENT void free(DEVICE& device, rl::algorithms::ppo::loop::core::State<T_CONFIG>& ts){
+        free(device, ts.rng);
+        free(device, ts.ppo);
+        free(device, ts.ppo_buffers);
+        free(device, ts.environment);
+        free(device, ts.on_policy_runner_dataset);
+        free(device, ts.on_policy_runner);
+        free(device, ts.on_policy_runner_buffer);
+        free(device, ts.actor_eval_buffers);
+        free(device, ts.actor_buffers);
+        free(device, ts.critic_buffers);
+        free(device, ts.critic_collection_state);
+        free(device, ts.critic_collection_buffer);
+        free(device, ts.actor_optimizer);
+        free(device, ts.critic_optimizer);
+    }
+    template <typename DEVICE, typename T_CONFIG>
+    RL_TOOLS_FUNCTION_PLACEMENT void init(DEVICE& device, rl::algorithms::ppo::loop::core::State<T_CONFIG>& ts, typename T_CONFIG::TI seed = 0){
+        using CONFIG = T_CONFIG;
+        using TI = typename DEVICE::index_t;
+
+        init(device, ts.rng, seed);
+        init(device, ts.environment);
+        init(device, ts.ppo, ts.actor_optimizer, ts.critic_optimizer, ts.rng);
+        init(device, ts.on_policy_runner, ts.environment, ts.rng);
+
+        ts.step = 0;
+    }
+    template <typename DEVICE, typename T_CONFIG>
+    RL_TOOLS_FUNCTION_PLACEMENT bool step(DEVICE& device, rl::algorithms::ppo::loop::core::State<T_CONFIG>& ts){
+        using CONFIG = T_CONFIG;
+        using TI = typename DEVICE::index_t;
+        constexpr TI CADENCE_PRE = CONFIG::CORE_PARAMETERS::STEP_LIMIT / 1000;
+        constexpr TI CADENCE = CADENCE_PRE > 0 ? CADENCE_PRE : 1;
+        constexpr TI N_AGENTS = T_CONFIG::ENVIRONMENT::N_AGENTS;
+        constexpr TI STEPS_TOTAL = CONFIG::ON_POLICY_RUNNER_DATASET_SPEC::STEPS_TOTAL;
+        set_step(device, device.logger, ts.step * CONFIG::CORE_PARAMETERS::N_ENVIRONMENTS * CONFIG::CORE_PARAMETERS::ON_POLICY_RUNNER_STEPS_PER_ENV);
+        bool finished = false;
+
+        if constexpr(T_CONFIG::CORE_PARAMETERS::NORMALIZE_OBSERVATIONS){
+            constexpr TI BATCH_SIZE = CONFIG::CORE_PARAMETERS::BATCH_SIZE;
+            constexpr TI N_BATCHES = STEPS_TOTAL / BATCH_SIZE;
+            static_assert(N_BATCHES > 0);
+            static constexpr TI ACCUMULATE_STEPS = CONFIG::PPO_SPEC::PARAMETERS::STATEFUL_ACTOR_AND_CRITIC ? CONFIG::ON_POLICY_RUNNER_DATASET_SPEC::STEPS_PER_ENV : 1;
+            static constexpr TI ACCUMULATE_FORWARD_BATCH_SIZE = CONFIG::PPO_SPEC::PARAMETERS::STATEFUL_ACTOR_AND_CRITIC ? CONFIG::ON_POLICY_RUNNER_DATASET_SPEC::SPEC::N_ENVIRONMENTS : BATCH_SIZE;
+            using OBS_SHAPE = typename T_CONFIG::ENVIRONMENT::Observation::SHAPE;
+            using OBS_PRIV_SHAPE = typename T_CONFIG::ENVIRONMENT::ObservationPrivileged::SHAPE;
+            using ACTOR_INPUT_SHAPE = tensor::Prepend<tensor::Prepend<OBS_SHAPE, ACCUMULATE_FORWARD_BATCH_SIZE>, ACCUMULATE_STEPS>;
+            using CRITIC_INPUT_SHAPE = tensor::Prepend<tensor::Prepend<OBS_PRIV_SHAPE, ACCUMULATE_FORWARD_BATCH_SIZE>, ACCUMULATE_STEPS>;
+            Mode<nn::layers::standardize::AccumulateMode<>> accumulate_mode;
+            if(ts.step == 0){
+                for(TI observation_normalization_warmup_step_i = 0; observation_normalization_warmup_step_i < T_CONFIG::OBSERVATION_NORMALIZATION_WARMUP_STEPS; observation_normalization_warmup_step_i++) {
+                    collect(device, ts.on_policy_runner_dataset, ts.on_policy_runner, ts.on_policy_runner_buffer, ts.environment, ts.ppo.actor, ts.actor_eval_buffers, ts.rng);
+                    for(TI batch_i = 0; batch_i < N_BATCHES; batch_i++){
+                        auto batch_observations = view_range(device, ts.on_policy_runner_dataset.all_observations, batch_i * BATCH_SIZE, tensor::ViewSpec<0, BATCH_SIZE>{});
+                        auto batch_observations_reshaped = reshape_row_major(device, batch_observations, ACTOR_INPUT_SHAPE{});
+                        forward(device, ts.ppo.actor, batch_observations_reshaped, ts.actor_buffers, ts.rng, accumulate_mode);
+                        auto batch_observations_privileged = view_range(device, ts.on_policy_runner_dataset.all_observations_privileged, batch_i * BATCH_SIZE, tensor::ViewSpec<0, BATCH_SIZE>{});
+                        auto batch_observations_privileged_reshaped = reshape_row_major(device, batch_observations_privileged, CRITIC_INPUT_SHAPE{});
+                        forward(device, ts.ppo.critic, batch_observations_privileged_reshaped, ts.critic_buffers, ts.rng, accumulate_mode);
+                    }
+                }
+                init(device, ts.on_policy_runner, ts.environment, ts.rng);
+            }
+        }
+        collect(device, ts.on_policy_runner_dataset, ts.on_policy_runner, ts.on_policy_runner_buffer, ts.environment, ts.ppo.actor, ts.actor_eval_buffers, ts.ppo.critic, ts.critic_collection_state, ts.critic_collection_buffer, ts.rng, typename decltype(ts.ppo)::SPEC::COLLECTION_MODE{});
+        estimate_generalized_advantages(device, ts.on_policy_runner_dataset, ts.on_policy_runner_dataset.bootstrap_values, typename CONFIG::PPO_TYPE::SPEC::PARAMETERS{});
+        train(device, ts.ppo, ts.on_policy_runner_dataset, ts.actor_optimizer, ts.critic_optimizer, ts.ppo_buffers, ts.actor_buffers, ts.critic_buffers, ts.rng);
+        if constexpr(T_CONFIG::CORE_PARAMETERS::NORMALIZE_OBSERVATIONS && T_CONFIG::CORE_PARAMETERS::NORMALIZE_OBSERVATIONS_CONTINUOUSLY){
+            constexpr TI BATCH_SIZE = CONFIG::CORE_PARAMETERS::BATCH_SIZE;
+            constexpr TI N_BATCHES = STEPS_TOTAL / BATCH_SIZE;
+            static constexpr TI ACCUMULATE_STEPS = CONFIG::PPO_SPEC::PARAMETERS::STATEFUL_ACTOR_AND_CRITIC ? CONFIG::ON_POLICY_RUNNER_DATASET_SPEC::STEPS_PER_ENV : 1;
+            static constexpr TI ACCUMULATE_FORWARD_BATCH_SIZE = CONFIG::PPO_SPEC::PARAMETERS::STATEFUL_ACTOR_AND_CRITIC ? CONFIG::ON_POLICY_RUNNER_DATASET_SPEC::SPEC::N_ENVIRONMENTS : BATCH_SIZE;
+            using OBS_SHAPE = typename T_CONFIG::ENVIRONMENT::Observation::SHAPE;
+            using OBS_PRIV_SHAPE = typename T_CONFIG::ENVIRONMENT::ObservationPrivileged::SHAPE;
+            using ACTOR_INPUT_SHAPE = tensor::Prepend<tensor::Prepend<OBS_SHAPE, ACCUMULATE_FORWARD_BATCH_SIZE>, ACCUMULATE_STEPS>;
+            using CRITIC_INPUT_SHAPE = tensor::Prepend<tensor::Prepend<OBS_PRIV_SHAPE, ACCUMULATE_FORWARD_BATCH_SIZE>, ACCUMULATE_STEPS>;
+            Mode<nn::layers::standardize::AccumulateMode<>> accumulate_mode;
+            for(TI batch_i = 0; batch_i < N_BATCHES; batch_i++){
+                auto batch_observations = view_range(device, ts.on_policy_runner_dataset.all_observations, batch_i * BATCH_SIZE, tensor::ViewSpec<0, BATCH_SIZE>{});
+                auto batch_observations_reshaped = reshape_row_major(device, batch_observations, ACTOR_INPUT_SHAPE{});
+                forward(device, ts.ppo.actor, batch_observations_reshaped, ts.actor_buffers, ts.rng, accumulate_mode);
+                auto batch_observations_privileged = view_range(device, ts.on_policy_runner_dataset.all_observations_privileged, batch_i * BATCH_SIZE, tensor::ViewSpec<0, BATCH_SIZE>{});
+                auto batch_observations_privileged_reshaped = reshape_row_major(device, batch_observations_privileged, CRITIC_INPUT_SHAPE{});
+                forward(device, ts.ppo.critic, batch_observations_privileged_reshaped, ts.critic_buffers, ts.rng, accumulate_mode);
+            }
+        }
+
+#ifndef RL_TOOLS_DISABLE_TENSORBOARD
+        {
+            // logging actor std
+            auto& last_layer = get_last_layer(ts.ppo.actor);
+            using T = typename CONFIG::T;
+            constexpr TI PER_AGENT_ACTION_DIM = T_CONFIG::ENVIRONMENT::ACTION_DIM/N_AGENTS;
+            for(TI action_i = 0; action_i < PER_AGENT_ACTION_DIM; action_i++){
+                T current_action_log_std = get(device, last_layer.log_std.parameters, action_i % PER_AGENT_ACTION_DIM);
+                add_scalar(device, device.logger, "actor/log_std", current_action_log_std, CADENCE);
+            }
+        }
+#endif
+
+        add_scalar(device, device.logger, "ppo/step", ts.step, CADENCE);
+
+        ts.step++;
+        if(ts.step > CONFIG::CORE_PARAMETERS::STEP_LIMIT){
+            return true;
+        }
+        else{
+            return finished;
+        }
+    }
+    template <typename DEVICE, typename PARAMETERS, typename utils::typing::enable_if<utils::typing::is_same_v<typename PARAMETERS::TAG, rl::algorithms::ppo::loop::core::ParametersTag>>::type* = nullptr>
+    RL_TOOLS_FUNCTION_PLACEMENT void log(DEVICE& device, PARAMETERS){
+        log(device, device.logger, "STEP_LIMIT: ", PARAMETERS::STEP_LIMIT);
+        log(device, device.logger, "ACTOR_HIDDEN_DIM: ", PARAMETERS::ACTOR_HIDDEN_DIM);
+        log(device, device.logger, "ACTOR_NUM_LAYERS: ", PARAMETERS::ACTOR_NUM_LAYERS);
+        log(device, device.logger, "ACTOR_ACTIVATION_FUNCTION: ", PARAMETERS::ACTOR_ACTIVATION_FUNCTION);
+        log(device, device.logger, "CRITIC_HIDDEN_DIM: ", PARAMETERS::CRITIC_HIDDEN_DIM);
+        log(device, device.logger, "CRITIC_NUM_LAYERS: ", PARAMETERS::CRITIC_NUM_LAYERS);
+        log(device, device.logger, "CRITIC_ACTIVATION_FUNCTION: ", PARAMETERS::CRITIC_ACTIVATION_FUNCTION);
+        log(device, device.logger, "EPISODE_STEP_LIMIT: ", PARAMETERS::EPISODE_STEP_LIMIT);
+        log(device, device.logger, "N_ENVIRONMENTS: ", PARAMETERS::N_ENVIRONMENTS);
+        log(device, device.logger, "ON_POLICY_RUNNER_STEPS_PER_ENV: ", PARAMETERS::ON_POLICY_RUNNER_STEPS_PER_ENV);
+        log(device, device.logger, "BATCH_SIZE: ", PARAMETERS::BATCH_SIZE);
+    }
+    template <typename DEVICE, typename CONFIG, typename utils::typing::enable_if<utils::typing::is_same_v<typename CONFIG::TAG, rl::algorithms::ppo::loop::core::ConfigTag>>::type* = nullptr>
+    RL_TOOLS_FUNCTION_PLACEMENT void log(DEVICE& device, CONFIG){
+        log(device, typename CONFIG::CORE_PARAMETERS{});
+    }
+
+    template <auto INDEX, typename DEVICE, typename T_CONFIG>
+    RL_TOOLS_FUNCTION_PLACEMENT constexpr auto& get_nn(DEVICE& device, rl::algorithms::ppo::loop::core::State<T_CONFIG>& ts){
+        static_assert(INDEX < T_CONFIG::NUM_NNS, "Index out of bounds, there are only 2 neural networks in the PPO");
+        if constexpr(INDEX == 0){
+            return ts.ppo.actor;
+        }
+        else{
+            return ts.ppo.critic;
+        }
+    }
+    template <auto INDEX, typename DEVICE, typename T_CONFIG>
+    RL_TOOLS_FUNCTION_PLACEMENT constexpr auto& get_nn_name(DEVICE& device, rl::algorithms::ppo::loop::core::State<T_CONFIG>& ts){
+        static_assert(INDEX < T_CONFIG::NUM_NNS, "Index out of bounds, there are only 2 neural networks in the PPO");
+        if constexpr(INDEX == 0){
+            return "actor";
+        }
+        else{
+            return "critic";
+        }
+    }
+    template <typename DEVICE, typename CONFIG>
+    RL_TOOLS_FUNCTION_PLACEMENT typename CONFIG::T abs_diff(DEVICE& device, rl::algorithms::ppo::loop::core::State<CONFIG>& s1, rl::algorithms::ppo::loop::core::State<CONFIG>& s2){
+        using T = typename CONFIG::T;
+        T acc = 0;
+        acc += abs_diff(device, s1.actor_optimizer, s2.actor_optimizer);
+        acc += abs_diff(device, s1.critic_optimizer, s2.critic_optimizer);
+        acc += abs_diff(device, s1.ppo, s2.ppo);
+        acc += abs_diff(device, s1.on_policy_runner, s2.on_policy_runner);
+        acc += abs_diff(device, s1.on_policy_runner_dataset, s2.on_policy_runner_dataset);
+        acc += math::abs(device.math, (T)s1.step - (T)s2.step);
+        acc += math::abs(device.math, (T)s1.next_checkpoint_id - (T)s2.next_checkpoint_id);
+        acc += math::abs(device.math, (T)s1.next_evaluation_id - (T)s2.next_evaluation_id);
+        acc += abs_diff(device, s1.rng, s2.rng);
+        return acc;
+    }
+}
+RL_TOOLS_NAMESPACE_WRAPPER_END
+#endif