hyperdrone 0.1.0__py3-none-any.whl

hyperdrone/_vendor/rl-tools/include/rl_tools/rendering/raytracing/scene/scene.h

@@ -0,0 +1,31 @@
+#include "../../../version.h"
+#include "../../../rl_tools.h"
+#if (defined(RL_TOOLS_DISABLE_INCLUDE_GUARDS) || !defined(RL_TOOLS_RENDERING_RAYTRACING_SCENE_SCENE_H)) && (RL_TOOLS_USE_THIS_VERSION == 1)
+#pragma once
+#define RL_TOOLS_RENDERING_RAYTRACING_SCENE_SCENE_H
+
+RL_TOOLS_NAMESPACE_WRAPPER_START
+namespace rl_tools::rendering::raytracing::scene {
+    template <typename T>
+    struct IndoorPosition {
+        T position[3];
+        T yaw;
+        T score;
+    };
+
+    template <typename T>
+    struct SceneConfig {
+        T scene_center[3];
+        T scene_radius;
+    };
+
+    template <typename T_T, typename T_TI, T_TI T_MAX_INDOOR_POSITIONS = 256>
+    struct SceneSpecification {
+        using T = T_T;
+        using TI = T_TI;
+        static constexpr TI MAX_INDOOR_POSITIONS = T_MAX_INDOOR_POSITIONS;
+    };
+}
+RL_TOOLS_NAMESPACE_WRAPPER_END
+
+#endif

hyperdrone/_vendor/rl-tools/include/rl_tools/rendering/raytracing/scene.h

@@ -0,0 +1,107 @@
+#include "../../version.h"
+#if (defined(RL_TOOLS_DISABLE_INCLUDE_GUARDS) || !defined(RL_TOOLS_RENDERING_RAYTRACING_SCENE_H)) && (RL_TOOLS_USE_THIS_VERSION == 1)
+#pragma once
+#define RL_TOOLS_RENDERING_RAYTRACING_SCENE_H
+
+#include "types.h"
+
+#include <vector>
+#include <string>
+#include <cstdint>
+
+RL_TOOLS_NAMESPACE_WRAPPER_START
+namespace rl_tools {
+    namespace rendering::raytracing{
+        struct Texture{
+            std::vector<uint8_t> pixels; // RGBA8
+            int width = 0;
+            int height = 0;
+            bool present() const {
+                return !pixels.empty() && width > 0 && height > 0;
+            }
+        };
+
+        struct Mesh{
+            std::vector<float> vertices; // vec3f flat
+            std::vector<int> indices;    // vec3i flat
+            std::vector<float> tex_coords; // vec2f flat
+            std::vector<float> normals;
+            float color[3];
+            Texture texture;
+            Texture normal_map;
+            Texture metallic_roughness_map;
+            Texture emissive_map;
+            Texture occlusion_map;
+            float metallic = 0.0f;
+            float roughness = 1.0f;
+            float emissive[3] = {0, 0, 0};
+            float opacity = 1.0f;
+            int alpha_mode = 0;
+            float alpha_cutoff = 0.5f;
+        };
+
+        struct Object{
+            std::vector<Mesh> meshes;
+            std::vector<SceneLight> lights; // object-local frame; transformed by the instance placing the object
+            uint32_t segmentation_class = 0; // user-assigned (e.g. from Object::name via a user taxonomy); reported by SEMANTIC_SEGMENTATION specs
+            std::string name; // the glTF root node's name for assembly parts; empty otherwise
+        };
+
+        // Load product of a split GLB: one part per glTF scene-root node, each an Object in the
+        // root node's local frame (its origin is the articulation pivot) plus the root's placement
+        // within the assembly frame. Deliberately one grouping level, no trees: deeper glTF
+        // hierarchy carries no articulation semantics and the Scene stays a flat instance list.
+        struct ObjectAssembly{
+            struct Part{
+                size_t object;       // index into ObjectAssembly::objects
+                float transform[12]; // part frame relative to the assembly frame, 3x4 row-major [R|t]
+            };
+            std::vector<Object> objects;
+            std::vector<Part> parts;
+        };
+
+        struct Placement{
+            size_t first_instance; // parts are placed as consecutive instances
+            size_t num_instances;
+        };
+
+        struct OverlayIndex{
+            size_t index;
+        };
+
+        // returned by spawn: the contiguous slot run holding one instantiated asset
+        struct OverlayPlacement{
+            size_t first_slot;
+            size_t num_parts;
+            size_t first_part; // into the renderer's flattened per-part tables (for rigid moves)
+        };
+
+        // Assets available to dynamic overlays: registered before init (BLASes built once, frozen
+        // after), instantiated per step via spawn. Must outlive rendering, like Scene; a boundary
+        // that outlives its caller (e.g. language bindings) must own copies of both.
+        struct AssetPool{
+            std::vector<ObjectAssembly> assemblies;
+        };
+        struct AssetHandle{
+            size_t index;
+        };
+
+        struct Instance{
+            size_t object;        // index into Scene::objects
+            float transform[12];  // object→world, 3x4 row-major [R|t]
+            bool identity;
+        };
+
+        // Host-side truth about what is rendered; filled by load/add, consumed by init(device,
+        // renderer, scene). The scene must stay alive while the renderer uses it: the generic
+        // backend renders directly from the mesh memory owned here.
+        struct Scene{
+            std::vector<Object> objects;
+            std::vector<Instance> instances;
+            std::vector<SceneLight> lights;
+        };
+    }
+}
+RL_TOOLS_NAMESPACE_WRAPPER_END
+
+#endif

hyperdrone/_vendor/rl-tools/include/rl_tools/rendering/raytracing/transforms_generic.h

@@ -0,0 +1,159 @@
+#include "../../version.h"
+#if (defined(RL_TOOLS_DISABLE_INCLUDE_GUARDS) || !defined(RL_TOOLS_RENDERING_RAYTRACING_TRANSFORMS_GENERIC_H)) && (RL_TOOLS_USE_THIS_VERSION == 1)
+#pragma once
+#define RL_TOOLS_RENDERING_RAYTRACING_TRANSFORMS_GENERIC_H
+
+#include "../../rl_tools.h"
+
+#include <math.h>
+
+#ifndef RL_TOOLS_FUNCTION_PLACEMENT
+#define RL_TOOLS_FUNCTION_PLACEMENT
+#endif
+
+// freestanding 3x4 row-major [R|t] transform math shared between the host verbs and device
+// producers (CUDA kernels define RL_TOOLS_FUNCTION_PLACEMENT before including this header)
+RL_TOOLS_NAMESPACE_WRAPPER_START
+namespace rl_tools{
+    namespace rendering::raytracing::detail{
+        RL_TOOLS_FUNCTION_PLACEMENT inline void compose_transforms(const float a[12], const float b[12], float out[12]){
+            for(int row = 0; row < 3; row++){
+                for(int column = 0; column < 3; column++){
+                    out[row * 4 + column] = a[row * 4] * b[column] + a[row * 4 + 1] * b[4 + column] + a[row * 4 + 2] * b[8 + column];
+                }
+                out[row * 4 + 3] = a[row * 4] * b[3] + a[row * 4 + 1] * b[7] + a[row * 4 + 2] * b[11] + a[row * 4 + 3];
+            }
+        }
+
+        RL_TOOLS_FUNCTION_PLACEMENT inline void quaternion_from_transform(const float m[12], float q[4]){
+            const float trace = m[0] + m[5] + m[10];
+            if(trace > 0){
+                const float s = sqrtf(trace + 1.0f) * 2;
+                q[0] = 0.25f * s;
+                q[1] = (m[9] - m[6]) / s;
+                q[2] = (m[2] - m[8]) / s;
+                q[3] = (m[4] - m[1]) / s;
+            }
+            else if(m[0] > m[5] && m[0] > m[10]){
+                const float s = sqrtf(1.0f + m[0] - m[5] - m[10]) * 2;
+                q[0] = (m[9] - m[6]) / s;
+                q[1] = 0.25f * s;
+                q[2] = (m[1] + m[4]) / s;
+                q[3] = (m[2] + m[8]) / s;
+            }
+            else if(m[5] > m[10]){
+                const float s = sqrtf(1.0f + m[5] - m[0] - m[10]) * 2;
+                q[0] = (m[2] - m[8]) / s;
+                q[1] = (m[1] + m[4]) / s;
+                q[2] = 0.25f * s;
+                q[3] = (m[6] + m[9]) / s;
+            }
+            else{
+                const float s = sqrtf(1.0f + m[10] - m[0] - m[5]) * 2;
+                q[0] = (m[4] - m[1]) / s;
+                q[1] = (m[2] + m[8]) / s;
+                q[2] = (m[6] + m[9]) / s;
+                q[3] = 0.25f * s;
+            }
+        }
+
+        RL_TOOLS_FUNCTION_PLACEMENT inline void rotate_by_quaternion(const float q[4], const float v[3], float out[3]){
+            const float w = q[0], x = q[1], y = q[2], z = q[3];
+            const float ux = y*v[2] - z*v[1], uy = z*v[0] - x*v[2], uz = x*v[1] - y*v[0];
+            const float uux = y*uz - z*uy, uuy = z*ux - x*uz, uuz = x*uy - y*ux;
+            out[0] = v[0] + 2*(w*ux + uux);
+            out[1] = v[1] + 2*(w*uy + uuy);
+            out[2] = v[2] + 2*(w*uz + uuz);
+        }
+
+        // constant-twist (screw motion) interpolation between two rigid 3x4 transforms: the
+        // rotation follows the shortest-arc slerp path and the translation follows the screw of
+        // the relative motion, so a fixed rotation axis line stays fixed — a translation lerp
+        // instead would cut the chord and displace an off-origin pivot (e.g. a spinning prop hub)
+        // at mid-shutter samples. The single interpolant behind set_transform_pair and
+        // expand_motion_transforms, so every consumer derives identical per-sample matrices from
+        // the same shutter pair. Exact for constant-velocity rigid motion below 180 degrees per
+        // shutter interval — faster motion must supply per-sample transforms directly.
+        RL_TOOLS_FUNCTION_PLACEMENT inline void slerp_transform(const float a[12], const float b[12], float t, float out[12]){
+            float qa[4], qb[4];
+            quaternion_from_transform(a, qa);
+            quaternion_from_transform(b, qb);
+            float relative[4] = {
+                 qb[0]*qa[0] + qb[1]*qa[1] + qb[2]*qa[2] + qb[3]*qa[3],
+                -qb[0]*qa[1] + qb[1]*qa[0] - qb[2]*qa[3] + qb[3]*qa[2],
+                -qb[0]*qa[2] + qb[1]*qa[3] + qb[2]*qa[0] - qb[3]*qa[1],
+                -qb[0]*qa[3] - qb[1]*qa[2] + qb[2]*qa[1] + qb[3]*qa[0]
+            };
+            if(relative[0] < 0){
+                for(int i = 0; i < 4; i++) relative[i] = -relative[i];
+            }
+            {
+                const float norm = sqrtf(relative[0]*relative[0] + relative[1]*relative[1] + relative[2]*relative[2] + relative[3]*relative[3]);
+                for(int i = 0; i < 4; i++) relative[i] /= norm;
+            }
+            const float sin_half_angle = sqrtf(relative[1]*relative[1] + relative[2]*relative[2] + relative[3]*relative[3]);
+            const float translation_a[3] = {a[3], a[7], a[11]};
+            float translation_a_relative[3];
+            rotate_by_quaternion(relative, translation_a, translation_a_relative);
+            const float displacement[3] = {b[3] - translation_a_relative[0], b[7] - translation_a_relative[1], b[11] - translation_a_relative[2]};
+            float relative_t[4];
+            float translation_relative_t[3];
+            if(sin_half_angle > 1e-3f){
+                const float half_angle = atan2f(sin_half_angle, relative[0]);
+                const float axis[3] = {relative[1] / sin_half_angle, relative[2] / sin_half_angle, relative[3] / sin_half_angle};
+                const float sin_t_half = sinf(t * half_angle);
+                relative_t[0] = cosf(t * half_angle);
+                relative_t[1] = axis[0] * sin_t_half;
+                relative_t[2] = axis[1] * sin_t_half;
+                relative_t[3] = axis[2] * sin_t_half;
+                const float displacement_along_axis = displacement[0]*axis[0] + displacement[1]*axis[1] + displacement[2]*axis[2];
+                const float perpendicular[3] = {displacement[0] - displacement_along_axis*axis[0], displacement[1] - displacement_along_axis*axis[1], displacement[2] - displacement_along_axis*axis[2]};
+                const float axis_cross_perpendicular[3] = {
+                    axis[1]*perpendicular[2] - axis[2]*perpendicular[1],
+                    axis[2]*perpendicular[0] - axis[0]*perpendicular[2],
+                    axis[0]*perpendicular[1] - axis[1]*perpendicular[0]
+                };
+                const float cot_half_angle = relative[0] / sin_half_angle;
+                float center[3], center_rotated[3];
+                for(int i = 0; i < 3; i++){
+                    center[i] = 0.5f * (perpendicular[i] + cot_half_angle * axis_cross_perpendicular[i]);
+                }
+                rotate_by_quaternion(relative_t, center, center_rotated);
+                for(int i = 0; i < 3; i++){
+                    translation_relative_t[i] = center[i] - center_rotated[i] + t * displacement_along_axis * axis[i];
+                }
+            }
+            else{
+                relative_t[0] = 1 + t * (relative[0] - 1);
+                relative_t[1] = t * relative[1];
+                relative_t[2] = t * relative[2];
+                relative_t[3] = t * relative[3];
+                const float norm = sqrtf(relative_t[0]*relative_t[0] + relative_t[1]*relative_t[1] + relative_t[2]*relative_t[2] + relative_t[3]*relative_t[3]);
+                for(int i = 0; i < 4; i++) relative_t[i] /= norm;
+                for(int i = 0; i < 3; i++){
+                    translation_relative_t[i] = t * displacement[i];
+                }
+            }
+            float q[4] = {
+                relative_t[0]*qa[0] - relative_t[1]*qa[1] - relative_t[2]*qa[2] - relative_t[3]*qa[3],
+                relative_t[0]*qa[1] + relative_t[1]*qa[0] + relative_t[2]*qa[3] - relative_t[3]*qa[2],
+                relative_t[0]*qa[2] - relative_t[1]*qa[3] + relative_t[2]*qa[0] + relative_t[3]*qa[1],
+                relative_t[0]*qa[3] + relative_t[1]*qa[2] - relative_t[2]*qa[1] + relative_t[3]*qa[0]
+            };
+            const float norm = sqrtf(q[0]*q[0] + q[1]*q[1] + q[2]*q[2] + q[3]*q[3]);
+            for(int i = 0; i < 4; i++) q[i] /= norm;
+            const float w = q[0], x = q[1], y = q[2], z = q[3];
+            out[0] = 1 - 2*(y*y + z*z); out[1] = 2*(x*y - w*z);     out[2]  = 2*(x*z + w*y);
+            out[4] = 2*(x*y + w*z);     out[5] = 1 - 2*(x*x + z*z); out[6]  = 2*(y*z - w*x);
+            out[8] = 2*(x*z - w*y);     out[9] = 2*(y*z + w*x);     out[10] = 1 - 2*(x*x + y*y);
+            float translation_a_rotated[3];
+            rotate_by_quaternion(relative_t, translation_a, translation_a_rotated);
+            out[3]  = translation_a_rotated[0] + translation_relative_t[0];
+            out[7]  = translation_a_rotated[1] + translation_relative_t[1];
+            out[11] = translation_a_rotated[2] + translation_relative_t[2];
+        }
+    }
+}
+RL_TOOLS_NAMESPACE_WRAPPER_END
+
+#endif

hyperdrone/_vendor/rl-tools/include/rl_tools/rendering/raytracing/types.h

@@ -0,0 +1,39 @@
+#include "../../version.h"
+#include "../../rl_tools.h"
+#if (defined(RL_TOOLS_DISABLE_INCLUDE_GUARDS) || !defined(RL_TOOLS_RENDERING_RAYTRACING_TYPES_H)) && (RL_TOOLS_USE_THIS_VERSION == 1)
+#pragma once
+#define RL_TOOLS_RENDERING_RAYTRACING_TYPES_H
+
+RL_TOOLS_NAMESPACE_WRAPPER_START
+namespace rl_tools {
+    namespace rendering::raytracing{
+        struct CollisionResult {
+            float distance;
+            int hit;
+        };
+
+        struct SceneLight {
+            int type; // 0=directional, 1=point, 2=spot
+            float position[3];
+            float direction[3];
+            float color[3];
+            float attenuation_constant;
+            float attenuation_linear;
+            float attenuation_quadratic;
+            float cos_inner_cone;
+            float cos_outer_cone;
+        };
+
+        template <typename T_T>
+        struct Camera {
+            T_T pos[3];
+            T_T dir_00[3];
+            T_T dir_du[3];
+            T_T dir_dv[3];
+        };
+
+    }
+}
+RL_TOOLS_NAMESPACE_WRAPPER_END
+
+#endif

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/algorithms/algorithms.h

@@ -0,0 +1,6 @@
+#include "ppo/ppo.h"
+#include "ppo/loop/core/config.h"
+#include "sac/sac.h"
+#include "sac/loop/core/config.h"
+#include "td3/td3.h"
+#include "td3/loop/core/config.h"

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/algorithms/operations_cpu.h

@@ -0,0 +1 @@
+#include "td3/operations_cpu_mkl.h"

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/algorithms/operations_cpu_mkl.h

@@ -0,0 +1 @@
+#include "td3/operations_cpu_mkl.h

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/algorithms/operations_generic.h

@@ -0,0 +1 @@
+//#include "td3/operations_generic.h"

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/algorithms/ppo/loop/core/config.h

@@ -0,0 +1,240 @@
+#include "../../../../../version.h"
+#if (defined(RL_TOOLS_DISABLE_INCLUDE_GUARDS) || !defined(RL_TOOLS_RL_ALGORITHMS_PPO_LOOP_CORE_CONFIG_H)) && (RL_TOOLS_USE_THIS_VERSION == 1)
+#pragma once
+#define RL_TOOLS_RL_ALGORITHMS_PPO_LOOP_CORE_CONFIG_H
+
+#include "../../../../../rl/environments/observation.h"
+#include "../../../../../nn/layers/standardize/layer.h"
+#include "../../../../../nn/layers/gru/layer.h"
+#include "../../../../../nn_models/sequential/model.h"
+#include "../../../../../nn_models/mlp_unconditional_stddev/network.h"
+#include "../../../../../nn_models/multi_agent_wrapper/model.h"
+#include "../../../../../rl/algorithms/ppo/ppo.h"
+#include "../../../../../rl/components/on_policy_runner/on_policy_runner.h"
+#include "../../../../../nn/optimizers/adam/adam.h"
+#include "../../../../../rl/loop/loop.h"
+#include "state.h"
+
+RL_TOOLS_NAMESPACE_WRAPPER_START
+namespace rl_tools{
+    namespace rl::algorithms::ppo::loop::core{
+        // Config State (Init/Step)
+
+        struct ParametersTag{};
+        template<typename TYPE_POLICY, typename TI, typename ENVIRONMENT>
+        struct DefaultParameters{
+            using T = typename TYPE_POLICY::DEFAULT;
+            using TAG = ParametersTag;
+            static constexpr TI STEP_LIMIT = 100;
+
+            static constexpr TI ACTOR_HIDDEN_DIM = 64;
+            static constexpr TI ACTOR_NUM_LAYERS = 3;
+            static constexpr auto ACTOR_ACTIVATION_FUNCTION = nn::activation_functions::ActivationFunction::RELU;
+            static constexpr TI CRITIC_HIDDEN_DIM = 64;
+            static constexpr TI CRITIC_NUM_LAYERS = 3;
+            static constexpr auto CRITIC_ACTIVATION_FUNCTION = nn::activation_functions::ActivationFunction::RELU;
+
+            static constexpr TI EPISODE_STEP_LIMIT = ENVIRONMENT::EPISODE_STEP_LIMIT;
+            static constexpr TI N_ENVIRONMENTS = 64;
+            static constexpr TI ON_POLICY_RUNNER_STEPS_PER_ENV = 64;
+            static constexpr TI DATASET_SIZE = ON_POLICY_RUNNER_STEPS_PER_ENV * N_ENVIRONMENTS;
+            static constexpr TI BATCH_SIZE = 64 * 8;
+
+            static constexpr bool NORMALIZE_OBSERVATIONS = false;
+            static constexpr bool NORMALIZE_OBSERVATIONS_CONTINUOUSLY = false;
+
+            using ACTOR_OPTIMIZER_PARAMETERS = nn::optimizers::adam::DEFAULT_PARAMETERS_TENSORFLOW<TYPE_POLICY>;
+            using CRITIC_OPTIMIZER_PARAMETERS = ACTOR_OPTIMIZER_PARAMETERS;
+
+            using PPO_PARAMETERS = rl::algorithms::ppo::DefaultParameters<TYPE_POLICY, TI, BATCH_SIZE>;
+        };
+
+        template<typename TYPE_POLICY, typename TI, typename ENVIRONMENT, typename PARAMETERS, bool DYNAMIC_ALLOCATION=true>
+        struct ConfigApproximatorsSequential{
+            static constexpr TI STEPS = PARAMETERS::PPO_PARAMETERS::STATEFUL_ACTOR_AND_CRITIC ? PARAMETERS::ON_POLICY_RUNNER_STEPS_PER_ENV : 1;
+            static constexpr TI FORWARD_BATCH_SIZE = PARAMETERS::PPO_PARAMETERS::STATEFUL_ACTOR_AND_CRITIC ? PARAMETERS::N_ENVIRONMENTS : PARAMETERS::BATCH_SIZE;
+            template <typename CAPABILITY>
+            struct Actor{
+                using OBS_SHAPE = typename ENVIRONMENT::Observation::SHAPE;
+                using INPUT_SHAPE = tensor::Prepend<tensor::Prepend<OBS_SHAPE, FORWARD_BATCH_SIZE>, STEPS>;
+                using STANDARDIZATION_LAYER_CONFIG = nn::layers::standardize::Configuration<TYPE_POLICY, TI>;
+                using STANDARDIZATION_LAYER = nn::layers::standardize::BindConfiguration<STANDARDIZATION_LAYER_CONFIG>;
+                using CONFIG = nn_models::mlp::Configuration<TYPE_POLICY, TI, ENVIRONMENT::ACTION_DIM, PARAMETERS::ACTOR_NUM_LAYERS, PARAMETERS::ACTOR_HIDDEN_DIM, PARAMETERS::ACTOR_ACTIVATION_FUNCTION,  nn::activation_functions::IDENTITY>;
+                using TYPE = nn_models::mlp_unconditional_stddev::BindConfiguration<CONFIG>;
+
+                using MODULE_CHAIN = nn_models::sequential::Module<STANDARDIZATION_LAYER, nn_models::sequential::Module<TYPE>>;
+                using MODEL = nn_models::sequential::Build<CAPABILITY, MODULE_CHAIN, INPUT_SHAPE>;
+            };
+            template <typename CAPABILITY>
+            struct Critic{
+                using OBS_PRIV_SHAPE = typename ENVIRONMENT::ObservationPrivileged::SHAPE;
+                using INPUT_SHAPE = tensor::Prepend<tensor::Prepend<OBS_PRIV_SHAPE, FORWARD_BATCH_SIZE>, STEPS>;
+                using STANDARDIZATION_LAYER_CONFIG = nn::layers::standardize::Configuration<TYPE_POLICY, TI>;
+                using STANDARDIZATION_LAYER = nn::layers::standardize::BindConfiguration<STANDARDIZATION_LAYER_CONFIG>;
+                using CONFIG = nn_models::mlp::Configuration<TYPE_POLICY, TI, 1, PARAMETERS::CRITIC_NUM_LAYERS, PARAMETERS::CRITIC_HIDDEN_DIM, PARAMETERS::CRITIC_ACTIVATION_FUNCTION, nn::activation_functions::IDENTITY>;
+                using TYPE = nn_models::mlp_unconditional_stddev::BindConfiguration<CONFIG>;
+
+                using MODULE_CHAIN = nn_models::sequential::Module<STANDARDIZATION_LAYER, nn_models::sequential::Module<TYPE>>;
+                using MODEL = nn_models::sequential::Build<CAPABILITY, MODULE_CHAIN, INPUT_SHAPE>;
+            };
+
+            using ACTOR_OPTIMIZER_SPEC = nn::optimizers::adam::Specification<TYPE_POLICY, TI, typename PARAMETERS::ACTOR_OPTIMIZER_PARAMETERS, DYNAMIC_ALLOCATION>;
+            using CRITIC_OPTIMIZER_SPEC = nn::optimizers::adam::Specification<TYPE_POLICY, TI, typename PARAMETERS::CRITIC_OPTIMIZER_PARAMETERS, DYNAMIC_ALLOCATION>;
+            using ACTOR_OPTIMIZER = nn::optimizers::Adam<ACTOR_OPTIMIZER_SPEC>;
+            using CRITIC_OPTIMIZER = nn::optimizers::Adam<CRITIC_OPTIMIZER_SPEC>;
+            using CAPABILITY_ADAM = nn::capability::Gradient<nn::parameters::Adam, DYNAMIC_ALLOCATION>;
+            using ACTOR_TYPE = typename Actor<CAPABILITY_ADAM>::MODEL;
+            using CRITIC_TYPE = typename Critic<CAPABILITY_ADAM>::MODEL;
+        };
+        template <bool T_GRU_CRITIC = false>
+        struct ConfigApproximatorsGRU{
+            template<typename TYPE_POLICY, typename TI, typename ENVIRONMENT, typename PARAMETERS, bool DYNAMIC_ALLOCATION=true>
+            struct Approximators{
+                static constexpr bool USE_GRU = true;
+                using PPO_PARAMETERS = typename PARAMETERS::PPO_PARAMETERS;
+                static_assert(PPO_PARAMETERS::SHUFFLE_EPOCH == false, "When using sequence models for the actor and critic, SHUFFLE_EPOCH has to be disabled (in the PPO parameters) because it scrambles the episodes.");
+                static_assert(PPO_PARAMETERS::STATEFUL_ACTOR_AND_CRITIC == true, "When using sequence models for the actor and critic, STATEFUL_ACTOR_AND_CRITIC has to be enabled.");
+                template <typename CAPABILITY>
+                struct Actor{
+                    using OBS_SHAPE = typename ENVIRONMENT::Observation::SHAPE;
+                    using INPUT_SHAPE = tensor::Prepend<tensor::Prepend<OBS_SHAPE, PARAMETERS::N_ENVIRONMENTS>, PARAMETERS::ON_POLICY_RUNNER_STEPS_PER_ENV>;
+                    using STANDARDIZATION_LAYER_CONFIG = nn::layers::standardize::Configuration<TYPE_POLICY, TI>;
+                    using STANDARDIZATION_LAYER = nn::layers::standardize::BindConfiguration<STANDARDIZATION_LAYER_CONFIG>;
+                    using INPUT_LAYER_CONFIG = nn::layers::dense::Configuration<TYPE_POLICY, TI, PARAMETERS::ACTOR_HIDDEN_DIM, PARAMETERS::ACTOR_ACTIVATION_FUNCTION, nn::layers::dense::DefaultInitializer<TYPE_POLICY, TI>, nn::parameters::groups::Input>;
+                    using INPUT_LAYER = nn::layers::dense::BindConfiguration<INPUT_LAYER_CONFIG>;
+                    static constexpr bool FAST_TANH = false;
+                    using GRU_SPEC = nn::layers::gru::Configuration<TYPE_POLICY, TI, PARAMETERS::ACTOR_HIDDEN_DIM, nn::parameters::groups::Normal, FAST_TANH>;
+                    using GRU = nn::layers::gru::BindConfiguration<GRU_SPEC>;
+                    using DENSE_LAYER_CONFIG = nn::layers::dense::Configuration<TYPE_POLICY, TI, PARAMETERS::ACTOR_HIDDEN_DIM, PARAMETERS::ACTOR_ACTIVATION_FUNCTION, nn::layers::dense::DefaultInitializer<TYPE_POLICY, TI>, nn::parameters::groups::Normal>;
+                    using DENSE_LAYER = nn::layers::dense::BindConfiguration<DENSE_LAYER_CONFIG>;
+                    using OUTPUT_LAYER_CONFIG = nn::layers::dense::Configuration<TYPE_POLICY, TI, ENVIRONMENT::ACTION_DIM, nn::activation_functions::IDENTITY, nn::layers::dense::DefaultInitializer<TYPE_POLICY, TI>, nn::parameters::groups::Output>;
+                    using OUTPUT_LAYER = nn::layers::dense::BindConfiguration<OUTPUT_LAYER_CONFIG>;
+                    using CONFIG = nn_models::mlp::Configuration<TYPE_POLICY, TI, ENVIRONMENT::ACTION_DIM, PARAMETERS::ACTOR_NUM_LAYERS, PARAMETERS::ACTOR_HIDDEN_DIM, PARAMETERS::ACTOR_ACTIVATION_FUNCTION,  nn::activation_functions::IDENTITY>;
+                    using MLP = nn_models::mlp_unconditional_stddev::BindConfiguration<CONFIG>;
+
+                    using MODULE = nn_models::sequential::Module<STANDARDIZATION_LAYER, nn_models::sequential::Module<INPUT_LAYER, nn_models::sequential::Module<GRU, nn_models::sequential::Module<MLP>>>>;
+                    using MODEL = nn_models::sequential::Build<CAPABILITY, MODULE, INPUT_SHAPE>;
+                };
+                template <typename CAPABILITY>
+                struct Critic{
+                    using OBS_PRIV_SHAPE = typename ENVIRONMENT::ObservationPrivileged::SHAPE;
+                    using INPUT_SHAPE = tensor::Prepend<tensor::Prepend<OBS_PRIV_SHAPE, PARAMETERS::N_ENVIRONMENTS>, PARAMETERS::ON_POLICY_RUNNER_STEPS_PER_ENV>;
+                    using STANDARDIZATION_LAYER_CONFIG = nn::layers::standardize::Configuration<TYPE_POLICY, TI>;
+                    using STANDARDIZATION_LAYER = nn::layers::standardize::BindConfiguration<STANDARDIZATION_LAYER_CONFIG>;
+                    using INPUT_LAYER_CONFIG = nn::layers::dense::Configuration<TYPE_POLICY, TI, PARAMETERS::CRITIC_HIDDEN_DIM, PARAMETERS::CRITIC_ACTIVATION_FUNCTION, nn::layers::dense::DefaultInitializer<TYPE_POLICY, TI>, nn::parameters::groups::Input>;
+                    using INPUT_LAYER = nn::layers::dense::BindConfiguration<INPUT_LAYER_CONFIG>;
+                    static constexpr bool FAST_TANH = false;
+                    using GRU_SPEC = nn::layers::gru::Configuration<TYPE_POLICY, TI, PARAMETERS::CRITIC_HIDDEN_DIM, nn::parameters::groups::Normal, FAST_TANH>;
+                    using GRU = nn::layers::gru::BindConfiguration<GRU_SPEC>;
+                    using DENSE_LAYER_CONFIG = nn::layers::dense::Configuration<TYPE_POLICY, TI, PARAMETERS::CRITIC_HIDDEN_DIM, PARAMETERS::CRITIC_ACTIVATION_FUNCTION, nn::layers::dense::DefaultInitializer<TYPE_POLICY, TI>, nn::parameters::groups::Normal>;
+                    using DENSE_LAYER = nn::layers::dense::BindConfiguration<DENSE_LAYER_CONFIG>;
+                    using OUTPUT_LAYER_CONFIG = nn::layers::dense::Configuration<TYPE_POLICY, TI, 1, nn::activation_functions::IDENTITY, nn::layers::dense::DefaultInitializer<TYPE_POLICY, TI>, nn::parameters::groups::Output>;
+                    using OUTPUT_LAYER = nn::layers::dense::BindConfiguration<OUTPUT_LAYER_CONFIG>;
+                    using CONFIG = nn_models::mlp::Configuration<TYPE_POLICY, TI, 1, PARAMETERS::CRITIC_NUM_LAYERS, PARAMETERS::CRITIC_HIDDEN_DIM, PARAMETERS::CRITIC_ACTIVATION_FUNCTION, nn::activation_functions::IDENTITY>;
+                    using MLP = nn_models::mlp::BindConfiguration<CONFIG>;
+                    using MODULE_DENSE = nn_models::sequential::Module<STANDARDIZATION_LAYER, nn_models::sequential::Module<INPUT_LAYER, nn_models::sequential::Module<DENSE_LAYER, nn_models::sequential::Module<MLP>>>>;
+                    using MODULE_GRU = nn_models::sequential::Module<STANDARDIZATION_LAYER, nn_models::sequential::Module<INPUT_LAYER, nn_models::sequential::Module<GRU, nn_models::sequential::Module<MLP>>>>;
+                    using MODULE = rl_tools::utils::typing::conditional_t<T_GRU_CRITIC, MODULE_GRU, MODULE_DENSE>;
+                    using MODEL = nn_models::sequential::Build<CAPABILITY, MODULE, INPUT_SHAPE>;
+                };
+
+                using ACTOR_OPTIMIZER_SPEC = nn::optimizers::adam::Specification<TYPE_POLICY, TI, typename PARAMETERS::ACTOR_OPTIMIZER_PARAMETERS, DYNAMIC_ALLOCATION>;
+                using CRITIC_OPTIMIZER_SPEC = nn::optimizers::adam::Specification<TYPE_POLICY, TI, typename PARAMETERS::CRITIC_OPTIMIZER_PARAMETERS, DYNAMIC_ALLOCATION>;
+                using ACTOR_OPTIMIZER = nn::optimizers::Adam<ACTOR_OPTIMIZER_SPEC>;
+                using CRITIC_OPTIMIZER = nn::optimizers::Adam<CRITIC_OPTIMIZER_SPEC>;
+                using CAPABILITY_ADAM = nn::capability::Gradient<nn::parameters::Adam, DYNAMIC_ALLOCATION>;
+                using ACTOR_TYPE = typename Actor<CAPABILITY_ADAM>::MODEL;
+                using CRITIC_TYPE = typename Critic<CAPABILITY_ADAM>::MODEL;
+            };
+        };
+
+        template<typename TYPE_POLICY, typename TI, typename ENVIRONMENT, typename PARAMETERS, bool DYNAMIC_ALLOCATION=true>
+        struct ConfigApproximatorsSequentialMultiAgent{
+            template <typename CAPABILITY>
+            struct Actor{
+                static constexpr TI N_AGENTS = ENVIRONMENT::N_AGENTS;
+                static_assert(ENVIRONMENT::Observation::DIM % N_AGENTS == 0);
+                static_assert(ENVIRONMENT::ACTION_DIM % N_AGENTS == 0);
+                using OBS_SHAPE = typename ENVIRONMENT::Observation::SHAPE;
+                using INPUT_SHAPE = tensor::Prepend<tensor::Prepend<OBS_SHAPE, PARAMETERS::BATCH_SIZE>, 1>;
+                using STANDARDIZATION_LAYER_CONFIG = nn::layers::standardize::Configuration<TYPE_POLICY, TI>;
+                using STANDARDIZATION_LAYER = nn::layers::standardize::BindConfiguration<STANDARDIZATION_LAYER_CONFIG>;
+                using CONFIG = nn_models::mlp::Configuration<TYPE_POLICY, TI, ENVIRONMENT::ACTION_DIM/N_AGENTS, PARAMETERS::ACTOR_NUM_LAYERS, PARAMETERS::ACTOR_HIDDEN_DIM, PARAMETERS::ACTOR_ACTIVATION_FUNCTION,  nn::activation_functions::IDENTITY>;
+                using TYPE = nn_models::mlp_unconditional_stddev::BindConfiguration<CONFIG>;
+
+                using INNER_MODULE_CHAIN = nn_models::sequential::Module<STANDARDIZATION_LAYER, nn_models::sequential::Module<TYPE>>;
+                using WRAPPER_CONFIG = nn_models::multi_agent_wrapper::Configuration<TYPE_POLICY, TI, N_AGENTS, INNER_MODULE_CHAIN>;
+                using MODEL = nn_models::multi_agent_wrapper::Build<CAPABILITY, WRAPPER_CONFIG, INPUT_SHAPE>;
+            };
+            template <typename CAPABILITY>
+            struct Critic{
+                using OBS_PRIV_SHAPE = typename ENVIRONMENT::ObservationPrivileged::SHAPE;
+                using INPUT_SHAPE = tensor::Prepend<tensor::Prepend<OBS_PRIV_SHAPE, PARAMETERS::BATCH_SIZE>, 1>;
+                using CONFIG = nn_models::mlp::Configuration<TYPE_POLICY, TI, 1, PARAMETERS::CRITIC_NUM_LAYERS, PARAMETERS::CRITIC_HIDDEN_DIM, PARAMETERS::CRITIC_ACTIVATION_FUNCTION, nn::activation_functions::IDENTITY>;
+                using TYPE = nn_models::mlp_unconditional_stddev::BindConfiguration<CONFIG>;
+                using STANDARDIZATION_LAYER_SPEC = nn::layers::standardize::Configuration<TYPE_POLICY, TI>;
+                using STANDARDIZATION_LAYER = nn::layers::standardize::BindConfiguration<STANDARDIZATION_LAYER_SPEC>;
+                using MODULE_CHAIN = nn_models::sequential::Module<STANDARDIZATION_LAYER, nn_models::sequential::Module<TYPE>>;
+                using MODEL = nn_models::sequential::Build<CAPABILITY, MODULE_CHAIN, INPUT_SHAPE>;
+
+
+//                using WRAPPER_CONFIG = nn_models::multi_agent_wrapper::Configuration<T, TI, N_AGENTS, INNER_MODULE_CHAIN>;
+//                using WRAPPER = nn_models::multi_agent_wrapper::BindConfiguration<WRAPPER_CONFIG>;
+//                using MODULE_CHAIN = Module<WRAPPER>;
+//                using MODEL = nn_models::multi_agent_wrapper::Module<WRAPPER_CONFIG, CAPABILITY, INPUT_SHAPE>;
+            };
+
+            using ACTOR_OPTIMIZER_SPEC = nn::optimizers::adam::Specification<TYPE_POLICY, TI, typename PARAMETERS::ACTOR_OPTIMIZER_PARAMETERS>;
+            using CRITIC_OPTIMIZER_SPEC = nn::optimizers::adam::Specification<TYPE_POLICY, TI, typename PARAMETERS::CRITIC_OPTIMIZER_PARAMETERS>;
+            using ACTOR_OPTIMIZER = nn::optimizers::Adam<ACTOR_OPTIMIZER_SPEC>;
+            using CRITIC_OPTIMIZER = nn::optimizers::Adam<CRITIC_OPTIMIZER_SPEC>;
+            using CAPABILITY_ADAM = nn::capability::Gradient<nn::parameters::Adam, DYNAMIC_ALLOCATION>;
+            using ACTOR_TYPE = typename Actor<CAPABILITY_ADAM>::MODEL;
+            using CRITIC_TYPE = typename Critic<CAPABILITY_ADAM>::MODEL;
+        };
+
+        struct ConfigTag{};
+        template<typename T_TYPE_POLICY, typename T_TI, typename T_RNG, typename T_ENVIRONMENT, typename T_PARAMETERS = DefaultParameters<T_TYPE_POLICY, T_TI, T_ENVIRONMENT>, template<typename, typename, typename, typename, bool> class APPROXIMATOR_CONFIG=ConfigApproximatorsSequential, bool T_DYNAMIC_ALLOCATION=true>
+        struct Config: rl::loop::Config{
+            using TAG = ConfigTag;
+            using TYPE_POLICY = T_TYPE_POLICY;
+            using TI = T_TI;
+            using RNG = T_RNG;
+            using ENVIRONMENT = T_ENVIRONMENT;
+            using ENVIRONMENT_EVALUATION = T_ENVIRONMENT;
+            using CORE_PARAMETERS = T_PARAMETERS;
+            static constexpr bool DYNAMIC_ALLOCATION = T_DYNAMIC_ALLOCATION;
+
+            static constexpr TI ENVIRONMENT_STEPS_PER_LOOP_STEP = CORE_PARAMETERS::N_ENVIRONMENTS * CORE_PARAMETERS::ON_POLICY_RUNNER_STEPS_PER_ENV;
+
+            using NN = APPROXIMATOR_CONFIG<TYPE_POLICY, TI, ENVIRONMENT, CORE_PARAMETERS, DYNAMIC_ALLOCATION>;
+
+
+            using T = typename TYPE_POLICY::DEFAULT;
+            static constexpr T OBSERVATION_NORMALIZATION_WARMUP_STEPS = CORE_PARAMETERS::NORMALIZE_OBSERVATIONS ? 1 : 0;
+            using PPO_SPEC = rl::algorithms::ppo::Specification<TYPE_POLICY, TI, ENVIRONMENT, typename NN::ACTOR_TYPE, typename NN::CRITIC_TYPE, typename CORE_PARAMETERS::PPO_PARAMETERS>;
+            using PPO_TYPE = rl::algorithms::PPO<PPO_SPEC>;
+            using PPO_BUFFERS_TYPE = rl::algorithms::ppo::Buffers<rl::algorithms::ppo::BufferSpecification<PPO_SPEC, DYNAMIC_ALLOCATION>>;
+            static constexpr TI NUM_NNS = 2;
+
+            static constexpr TI AGENTS_PER_ENV = 1;
+            using ON_POLICY_RUNNER_SPEC = rl::components::on_policy_runner::Specification<TYPE_POLICY, TI, ENVIRONMENT, typename NN::ACTOR_TYPE::template State<DYNAMIC_ALLOCATION>, CORE_PARAMETERS::N_ENVIRONMENTS, CORE_PARAMETERS::EPISODE_STEP_LIMIT, AGENTS_PER_ENV, CORE_PARAMETERS::PPO_PARAMETERS::TRUNCATE_ON_EACH_ITERATION, DYNAMIC_ALLOCATION>;
+            using ON_POLICY_RUNNER_TYPE = rl::components::OnPolicyRunner<ON_POLICY_RUNNER_SPEC>;
+            using ON_POLICY_RUNNER_DATASET_SPEC = rl::components::on_policy_runner::DatasetSpecification<ON_POLICY_RUNNER_SPEC, CORE_PARAMETERS::ON_POLICY_RUNNER_STEPS_PER_ENV, DYNAMIC_ALLOCATION>;
+            using ON_POLICY_RUNNER_DATASET_TYPE = rl::components::on_policy_runner::Dataset<ON_POLICY_RUNNER_DATASET_SPEC>;
+
+
+            using ACTOR_EVAL_BUFFERS = typename NN::ACTOR_TYPE::template Buffer<DYNAMIC_ALLOCATION>;
+            using ACTOR_BUFFERS = typename NN::ACTOR_TYPE::template Buffer<DYNAMIC_ALLOCATION>;
+            using CRITIC_BUFFERS = typename NN::CRITIC_TYPE::template Buffer<DYNAMIC_ALLOCATION>;
+            using CRITIC_GAE = typename NN::CRITIC_TYPE::template CHANGE_BATCH_SIZE<TI, ON_POLICY_RUNNER_DATASET_SPEC::STEPS_TOTAL_ALL>;
+            using CRITIC_BUFFERS_GAE = typename CRITIC_GAE::template Buffer<DYNAMIC_ALLOCATION>;
+            template <typename CONFIG>
+            using State = State<CONFIG>;
+        };
+    }
+}
+RL_TOOLS_NAMESPACE_WRAPPER_END
+
+#endif
+