hyperray 0.0.1__py3-none-any.whl

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/loop/steps/timing/operations_cpu.h

@@ -0,0 +1,52 @@
+#include "../../../../version.h"
+#if (defined(RL_TOOLS_DISABLE_INCLUDE_GUARDS) || !defined(RL_TOOLS_RL_LOOP_STEPS_TIMING_OPERATIONS_GENERIC_H)) && (RL_TOOLS_USE_THIS_VERSION == 1)
+#pragma once
+#define RL_TOOLS_RL_LOOP_STEPS_TIMING_OPERATIONS_GENERIC_H
+
+#include "config.h"
+
+RL_TOOLS_NAMESPACE_WRAPPER_START
+namespace rl_tools{
+    template <typename DEVICE, typename T_CONFIG>
+    void init(DEVICE& device, rl::loop::steps::timing::State<T_CONFIG>& ts, typename T_CONFIG::TI seed = 0){
+        using STATE = rl::loop::steps::timing::State<T_CONFIG>;
+        init(device, static_cast<typename STATE::NEXT&>(ts), seed);
+        ts.start_time = std::chrono::steady_clock::now();
+        ts.last_steps_per_second_time = ts.start_time;
+        ts.last_steps_per_second_step = ts.step;
+    }
+
+    template <typename DEVICE, typename T_CONFIG>
+    void free(DEVICE& device, rl::loop::steps::timing::State<T_CONFIG>& ts){
+        using STATE = rl::loop::steps::timing::State<T_CONFIG>;
+        free(device, static_cast<typename STATE::NEXT&>(ts));
+    }
+
+    template <typename DEVICE, typename CONFIG>
+    bool step(DEVICE& device, rl::loop::steps::timing::State<CONFIG>& ts){
+        using T = typename CONFIG::TYPE_POLICY::DEFAULT;
+        using TI = typename CONFIG::TI;
+        using STATE = rl::loop::steps::timing::State<CONFIG>;
+        bool finished = step(device, static_cast<typename STATE::NEXT&>(ts));
+        if(ts.step % CONFIG::PARAMETERS::INTERVAL == 0){
+            auto now = std::chrono::steady_clock::now();
+            if(now - ts.last_steps_per_second_time > std::chrono::seconds(10)){
+                TI steps = ts.step - ts.last_steps_per_second_step;
+                T steps_per_second = (T)steps / std::chrono::duration_cast<std::chrono::microseconds>(now - ts.last_steps_per_second_time).count() * 1000000 * CONFIG::ENVIRONMENT_STEPS_PER_LOOP_STEP;
+                log(device, device.logger, "Loop step: ", ts.step, ", env step: ", ts.step * CONFIG::ENVIRONMENT_STEPS_PER_LOOP_STEP, ", SPS: ", steps_per_second, " (elapsed: ", std::chrono::duration_cast<std::chrono::milliseconds>(now - ts.start_time).count()/1000.0, " s)");
+                add_scalar(device, device.logger, "steps_per_second", steps_per_second);
+                ts.last_steps_per_second_time = now;
+                ts.last_steps_per_second_step = ts.step;
+            }
+        }
+        if(finished){
+            auto now = std::chrono::steady_clock::now();
+            log(device, device.logger, "Time: ", std::chrono::duration_cast<std::chrono::milliseconds>(now - ts.start_time).count()/1000.0, "s");
+        }
+        return finished;
+    }
+}
+RL_TOOLS_NAMESPACE_WRAPPER_END
+
+
+#endif

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/loop/steps/timing/persist.h

@@ -0,0 +1,20 @@
+#include "../../../../version.h"
+#if (defined(RL_TOOLS_DISABLE_INCLUDE_GUARDS) || !defined(RL_TOOLS_RL_LOOP_STEPS_TIMING_PERSIST_H)) && (RL_TOOLS_USE_THIS_VERSION == 1)
+#pragma once
+#define RL_TOOLS_RL_LOOP_STEPS_TIMING_PERSIST_H
+#include "state.h"
+RL_TOOLS_NAMESPACE_WRAPPER_START
+namespace rl_tools{
+    template <typename DEVICE, typename T_CONFIG, typename GROUP>
+    void save(DEVICE& device, rl::loop::steps::timing::State<T_CONFIG>& ts, GROUP& group){
+        using STATE = rl::loop::steps::timing::State<T_CONFIG>;
+        save(device, static_cast<typename STATE::NEXT&>(ts), group);
+    }
+    template <typename DEVICE, typename T_CONFIG, typename GROUP>
+    bool load(DEVICE& device, rl::loop::steps::timing::State<T_CONFIG>& ts, GROUP& group){
+        using STATE = rl::loop::steps::timing::State<T_CONFIG>;
+        return load(device, static_cast<typename STATE::NEXT&>(ts), group);
+    }
+}
+RL_TOOLS_NAMESPACE_WRAPPER_END
+#endif

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/loop/steps/timing/state.h

@@ -0,0 +1,25 @@
+#include "../../../../version.h"
+#if (defined(RL_TOOLS_DISABLE_INCLUDE_GUARDS) || !defined(RL_TOOLS_RL_LOOP_STEPS_TIMING_STATE_H)) && (RL_TOOLS_USE_THIS_VERSION == 1)
+#pragma once
+#define RL_TOOLS_RL_LOOP_STEPS_TIMING_STATE_H
+
+
+#include <chrono>
+
+RL_TOOLS_NAMESPACE_WRAPPER_START
+namespace rl_tools::rl::loop::steps::timing{
+    template<typename T_CONFIG, typename T_NEXT = typename T_CONFIG::NEXT::template State<typename T_CONFIG::NEXT>>
+    struct State: T_NEXT {
+        using CONFIG = T_CONFIG;
+        using NEXT = T_NEXT;
+        using TI = typename CONFIG::TI;
+        std::chrono::steady_clock::time_point start_time, last_steps_per_second_time;
+        TI last_steps_per_second_step;
+    };
+}
+RL_TOOLS_NAMESPACE_WRAPPER_END
+#endif
+
+
+
+

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/operations_generic.h

@@ -0,0 +1,3 @@
+//#include "algorithms/operations_generic.h"
+//#include "components/operations_generic.h"
+//#include "environments/operations_generic.h"

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/rl.h

@@ -0,0 +1,5 @@
+#include "algorithms/algorithms.h"
+#include "components/components.h"
+#include "environment_wrappers/wrappers.h"
+#include "environments/environments.h"
+#include "loop/loop.h"

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/utils/evaluation/evaluation.h

@@ -0,0 +1,106 @@
+#include "../../../version.h"
+#if (defined(RL_TOOLS_DISABLE_INCLUDE_GUARDS) || !defined(RL_TOOLS_RL_UTILS_EVALUATION_EVALUATION_H)) && (RL_TOOLS_USE_THIS_VERSION == 1)
+#pragma once
+#define RL_TOOLS_RL_UTILS_EVALUATION_EVALUATION_H
+/*
+ * This file relies on the environments methods hence it should be included after the operations of the environments that it will be used with
+ */
+#include "../../../utils/generic/typing.h"
+
+RL_TOOLS_NAMESPACE_WRAPPER_START
+namespace rl_tools::rl::utils::evaluation{
+    template <typename T, typename TI, typename ENV_STATE>
+    struct State{
+        T episode_return = 0;
+        TI episode_step = 0;
+
+        ENV_STATE state;
+    };
+    template <typename T_TYPE_POLICY, typename T_TI, typename T_ENVIRONMENT, T_TI T_N_EPISODES, T_TI T_STEP_LIMIT, bool T_DETERMINISTIC_INITIAL_STATE=false>
+    struct Specification{
+        using TYPE_POLICY = T_TYPE_POLICY;
+        using T = typename TYPE_POLICY::DEFAULT;
+        using TI = T_TI;
+        using ENVIRONMENT = T_ENVIRONMENT;
+        constexpr static TI N_EPISODES = T_N_EPISODES;
+        constexpr static TI STEP_LIMIT = T_STEP_LIMIT;
+        constexpr static bool DETERMINISTIC_INITIAL_STATE = T_DETERMINISTIC_INITIAL_STATE;
+    };
+    template <typename T_SPEC, bool T_DYNAMIC_ALLOCATION=true>
+    struct DataSpecification{
+        using SPEC = T_SPEC;
+        static constexpr bool DYNAMIC_ALLOCATION = T_DYNAMIC_ALLOCATION;
+    };
+    template <typename T_SPEC>
+    struct Data{
+        using SPEC = typename T_SPEC::SPEC;
+        using T = typename SPEC::TYPE_POLICY::DEFAULT;
+        using TI = typename SPEC::TI;
+        using ENVIRONMENT = typename SPEC::ENVIRONMENT;
+        static constexpr bool DYNAMIC_ALLOCATION = T_SPEC::DYNAMIC_ALLOCATION;
+        Tensor<tensor::Specification<typename ENVIRONMENT::Parameters, TI, tensor::Shape<TI, SPEC::N_EPISODES>, DYNAMIC_ALLOCATION>> parameters;
+        Tensor<tensor::Specification<bool, TI, tensor::Shape<TI, SPEC::N_EPISODES, SPEC::STEP_LIMIT>, DYNAMIC_ALLOCATION>> terminated;
+        Tensor<tensor::Specification<T, TI, tensor::Shape<TI, SPEC::N_EPISODES, SPEC::STEP_LIMIT>, DYNAMIC_ALLOCATION>> rewards;
+        Tensor<tensor::Specification<typename ENVIRONMENT::State, TI, tensor::Shape<TI, SPEC::N_EPISODES, SPEC::STEP_LIMIT>, DYNAMIC_ALLOCATION>> states;
+        Tensor<tensor::Specification<T, TI, tensor::Shape<TI, SPEC::N_EPISODES, SPEC::STEP_LIMIT, ENVIRONMENT::ACTION_DIM>, DYNAMIC_ALLOCATION>> actions;
+        Tensor<tensor::Specification<T, TI, tensor::Shape<TI, SPEC::N_EPISODES, SPEC::STEP_LIMIT>, DYNAMIC_ALLOCATION>> dt;
+    };
+    template <typename T_SPEC>
+    struct NoData{};
+    template <typename T_SPEC>
+    struct Result{
+        using SPEC = T_SPEC;
+        using T = typename SPEC::TYPE_POLICY::DEFAULT;
+        using TI = typename SPEC::TI;
+        using ENVIRONMENT = typename SPEC::ENVIRONMENT;
+        constexpr static auto N_EPISODES = SPEC::N_EPISODES;
+        constexpr static auto STEP_LIMIT = SPEC::STEP_LIMIT;
+        T returns[N_EPISODES];
+        T returns_mean;
+        T returns_std;
+        TI episode_length[N_EPISODES];
+        T episode_length_mean;
+        T episode_length_std;
+        TI num_terminated;
+        T share_terminated;
+    };
+    template <typename T_SPEC, bool T_DYNAMIC_ALLOCATION=true>
+    struct BufferSpecification{
+        using SPEC = T_SPEC;
+        static constexpr bool DYNAMIC_ALLOCATION=T_DYNAMIC_ALLOCATION;
+    };
+    template <typename T_SPEC>
+    struct Buffer{
+        using SPEC = typename T_SPEC::SPEC;
+        using T = typename SPEC::TYPE_POLICY::DEFAULT;
+        using TI = typename SPEC::TI;
+        using ENVIRONMENT = typename SPEC::ENVIRONMENT;
+        static constexpr bool DYNAMIC_ALLOCATION = T_SPEC::DYNAMIC_ALLOCATION;
+        Matrix<matrix::Specification<T, TI, SPEC::N_EPISODES, ENVIRONMENT::ACTION_DIM, DYNAMIC_ALLOCATION>> actions;
+        Matrix<matrix::Specification<T, TI, SPEC::N_EPISODES, ENVIRONMENT::Observation::DIM, DYNAMIC_ALLOCATION>> observations;
+        Tensor<tensor::Specification<ENVIRONMENT, TI, tensor::Shape<TI, SPEC::N_EPISODES>, DYNAMIC_ALLOCATION>> envs;
+        Tensor<tensor::Specification<typename ENVIRONMENT::State, TI, tensor::Shape<TI, SPEC::N_EPISODES>, DYNAMIC_ALLOCATION>> states;
+        Tensor<tensor::Specification<typename ENVIRONMENT::Parameters, TI, tensor::Shape<TI, SPEC::N_EPISODES>, DYNAMIC_ALLOCATION>> parameters;
+    };
+    template <typename T_SPEC, typename T_POLICY, bool T_DYNAMIC_ALLOCATION=true>
+    struct PolicyBufferSpecification{
+        using SPEC = T_SPEC;
+        using POLICY = T_POLICY;
+        static constexpr bool DYNAMIC_ALLOCATION=T_DYNAMIC_ALLOCATION;
+    };
+    template <typename T_SPEC>
+    struct PolicyBuffer{
+        using SPEC = typename T_SPEC::SPEC;
+        static constexpr bool DYNAMIC_ALLOCATION = T_SPEC::DYNAMIC_ALLOCATION;
+        using POLICY_ORIG = typename T_SPEC::POLICY;
+        using POLICY_BATCH_SIZE = typename POLICY_ORIG::template CHANGE_BATCH_SIZE<typename SPEC::TI, SPEC::N_EPISODES>;
+        using POLICY = typename POLICY_BATCH_SIZE::template CHANGE_CAPABILITY<nn::capability::Forward<DYNAMIC_ALLOCATION>>;
+        POLICY policy;
+        typename POLICY::template State<DYNAMIC_ALLOCATION> policy_state;
+        typename POLICY::template Buffer<DYNAMIC_ALLOCATION> policy_evaluation_buffers;
+        Buffer<BufferSpecification<SPEC, DYNAMIC_ALLOCATION>> buffer;
+    };
+
+}
+RL_TOOLS_NAMESPACE_WRAPPER_END
+#endif

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/utils/evaluation/operations_cpu.h

@@ -0,0 +1,48 @@
+#include "../../../version.h"
+#if (defined(RL_TOOLS_DISABLE_INCLUDE_GUARDS) || !defined(RL_TOOLS_RL_UTILS_EVALUATION_OPERATIONS_CPU_H)) && (RL_TOOLS_USE_THIS_VERSION == 1)
+#pragma once
+#define RL_TOOLS_RL_UTILS_EVALUATION_OPERATIONS_CPU_H
+/*
+ * This file relies on the environments methods hence it should be included after the operations of the environments that it will be used with
+ */
+
+#include "operations_generic.h"
+
+RL_TOOLS_NAMESPACE_WRAPPER_START
+namespace rl_tools{
+    template <typename DEVICE, typename SPEC>
+    std::string json(DEVICE& device, rl::utils::evaluation::Result<SPEC>& result, typename DEVICE::index_t step = 0) {
+        using TI = typename DEVICE::index_t;
+        std::string data;
+        data += "{";
+        data += "\"step\": " + std::to_string(step) + ", ";
+        data += "\"returns_mean\": " + std::to_string(result.returns_mean) + ", ";
+        data += "\"returns_std\": " + std::to_string(result.returns_std) + ", ";
+        data += "\"episode_length_mean\": " + std::to_string(result.episode_length_mean) + ", ";
+        data += "\"episode_length_std\": " + std::to_string(result.episode_length_std) + ", ";
+        data += "\"returns\": [";
+        for(TI episode_i = 0; episode_i < SPEC::N_EPISODES; episode_i++){
+            data += std::to_string(result.returns[episode_i]);
+            if(episode_i < SPEC::N_EPISODES - 1){
+                data += ", ";
+            }
+        }
+        data += "], ";
+        data += "\"episode_length\": [";
+        for(TI episode_i = 0; episode_i < SPEC::N_EPISODES; episode_i++){
+            data += std::to_string(result.episode_length[episode_i]);
+            if(episode_i < SPEC::N_EPISODES - 1){
+                data += ", ";
+            }
+        }
+        data += "], ";
+        data += "\"num_terminated\": " + std::to_string(result.num_terminated) + ", ";
+        data += "\"share_terminated\": " + std::to_string(result.share_terminated);
+        data += "}";
+        return data;
+    }
+}
+RL_TOOLS_NAMESPACE_WRAPPER_END
+
+
+#endif

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/utils/evaluation/operations_generic.h

@@ -0,0 +1,275 @@
+#include "../../../version.h"
+#if (defined(RL_TOOLS_DISABLE_INCLUDE_GUARDS) || !defined(RL_TOOLS_RL_UTILS_EVALUATION_OPERATIONS_GENERIC_H)) && (RL_TOOLS_USE_THIS_VERSION == 1)
+#pragma once
+#define RL_TOOLS_RL_UTILS_EVALUATION_OPERATIONS_GENERIC_H
+/*
+ * This file relies on the environments methods hence it should be included after the operations of the environments that it will be used with
+ */
+
+#include "../../../math/operations_generic.h"
+#include "../../environments/operations_generic.h"
+
+#include "evaluation.h"
+
+RL_TOOLS_NAMESPACE_WRAPPER_START
+namespace rl_tools{
+    template <typename DEVICE, typename SPEC>
+    RL_TOOLS_FUNCTION_PLACEMENT void malloc(DEVICE& device, rl::utils::evaluation::NoData<SPEC>& data){ }
+    template <typename DEVICE, typename SPEC>
+    RL_TOOLS_FUNCTION_PLACEMENT void free(DEVICE& device, rl::utils::evaluation::NoData<SPEC>& data){ }
+    template <typename DEVICE, typename SPEC>
+    RL_TOOLS_FUNCTION_PLACEMENT void malloc(DEVICE& device, rl::utils::evaluation::Data<SPEC>& data){
+        malloc(device, data.parameters);
+        malloc(device, data.terminated);
+        malloc(device, data.rewards);
+        malloc(device, data.states);
+        malloc(device, data.actions);
+        malloc(device, data.dt);
+    }
+    template <typename DEVICE, typename SPEC>
+    RL_TOOLS_FUNCTION_PLACEMENT void free(DEVICE& device, rl::utils::evaluation::Data<SPEC>& data){
+        free(device, data.parameters);
+        free(device, data.terminated);
+        free(device, data.rewards);
+        free(device, data.states);
+        free(device, data.actions);
+        free(device, data.dt);
+    }
+    template <typename DEVICE, typename SPEC>
+    RL_TOOLS_FUNCTION_PLACEMENT void malloc(DEVICE& device, rl::utils::evaluation::Buffer<SPEC>& buffer){
+        malloc(device, buffer.actions);
+        malloc(device, buffer.observations);
+        malloc(device, buffer.envs);
+        malloc(device, buffer.states);
+        malloc(device, buffer.parameters);
+    }
+    template <typename DEVICE, typename SPEC>
+    RL_TOOLS_FUNCTION_PLACEMENT void free(DEVICE& device, rl::utils::evaluation::Buffer<SPEC>& buffer){
+        free(device, buffer.actions);
+        free(device, buffer.observations);
+        free(device, buffer.envs);
+        free(device, buffer.states);
+        free(device, buffer.parameters);
+    }
+    template <typename DEVICE, typename SPEC>
+    RL_TOOLS_FUNCTION_PLACEMENT void malloc(DEVICE& device, rl::utils::evaluation::PolicyBuffer<SPEC>& buffer){
+        malloc(device, buffer.buffer);
+        malloc(device, buffer.policy);
+        malloc(device, buffer.policy_state);
+        malloc(device, buffer.policy_evaluation_buffers);
+    }
+    template <typename DEVICE, typename SPEC>
+    RL_TOOLS_FUNCTION_PLACEMENT void free(DEVICE& device, rl::utils::evaluation::PolicyBuffer<SPEC>& buffer){
+        free(device, buffer.buffer);
+        free(device, buffer.policy);
+        free(device, buffer.policy_state);
+        free(device, buffer.policy_evaluation_buffers);
+    }
+    namespace rl::utils::evaluation{
+        template<typename DEVICE, typename TI, typename SPEC>
+        RL_TOOLS_FUNCTION_PLACEMENT void set_state(DEVICE& device, rl::utils::evaluation::NoData<SPEC>& data, TI episode_i, TI step_i, const typename SPEC::SPEC::ENVIRONMENT::State& state){}
+        template<typename DEVICE, typename TI, typename SPEC>
+        RL_TOOLS_FUNCTION_PLACEMENT void set_state(DEVICE& device, rl::utils::evaluation::Data<SPEC>& data, TI episode_i, TI step_i, const typename SPEC::SPEC::ENVIRONMENT::State& state){
+            set(device, data.states, state, episode_i, step_i);
+        }
+        template<typename DEVICE, typename TI, typename SPEC>
+        RL_TOOLS_FUNCTION_PLACEMENT void set_parameters(DEVICE& device, rl::utils::evaluation::NoData<SPEC>& data, TI episode_i, const typename SPEC::SPEC::ENVIRONMENT::Parameters& parameters){ }
+        template<typename DEVICE, typename TI, typename SPEC>
+        RL_TOOLS_FUNCTION_PLACEMENT void set_parameters(DEVICE& device, rl::utils::evaluation::Data<SPEC>& data, TI episode_i, const typename SPEC::SPEC::ENVIRONMENT::Parameters& parameters){
+            set(device, data.parameters, parameters, episode_i);
+        }
+        template<typename DEVICE, typename TI, typename SPEC, typename ACTION_SPEC>
+        RL_TOOLS_FUNCTION_PLACEMENT void set_action(DEVICE& device, rl::utils::evaluation::NoData<SPEC>& data, TI step_i, const Matrix<ACTION_SPEC>& actions){}
+        template<typename DEVICE, typename TI, typename DATA_SPEC, typename ACTION_SPEC>
+        RL_TOOLS_FUNCTION_PLACEMENT void set_action(DEVICE& device, rl::utils::evaluation::Data<DATA_SPEC>& data, TI step_i, const Matrix<ACTION_SPEC>& actions){
+            using SPEC = typename DATA_SPEC::SPEC;
+            static_assert(ACTION_SPEC::ROWS == SPEC::N_EPISODES);
+            static_assert(ACTION_SPEC::COLS == SPEC::ENVIRONMENT::ACTION_DIM);
+            for (TI episode_i = 0; episode_i < SPEC::N_EPISODES; episode_i++){
+                for (TI action_i = 0; action_i < SPEC::ENVIRONMENT::ACTION_DIM; action_i++) {
+                    set(device, data.actions, get(actions, episode_i, action_i), episode_i, step_i, action_i);
+                }
+            }
+        }
+        template<typename DEVICE, typename TI, typename SPEC>
+        RL_TOOLS_FUNCTION_PLACEMENT void set_dt(DEVICE& device, rl::utils::evaluation::NoData<SPEC>& data, TI episode_i, TI step_i, typename SPEC::SPEC::T dt){ }
+        template<typename DEVICE, typename TI, typename SPEC>
+        RL_TOOLS_FUNCTION_PLACEMENT void set_dt(DEVICE& device, rl::utils::evaluation::Data<SPEC>& data, TI episode_i, TI step_i, typename SPEC::SPEC::T dt){
+            set(device, data.dt, dt, episode_i, step_i);
+        }
+        template<typename DEVICE, typename TI, typename SPEC>
+        RL_TOOLS_FUNCTION_PLACEMENT void set_reward(DEVICE& device, rl::utils::evaluation::NoData<SPEC>& data, TI episode_i, TI step_i, typename SPEC::SPEC::T reward){}
+        template<typename DEVICE, typename TI, typename SPEC>
+        RL_TOOLS_FUNCTION_PLACEMENT void set_reward(DEVICE& device, rl::utils::evaluation::Data<SPEC>& data, TI episode_i, TI step_i, typename SPEC::SPEC::T reward){
+            set(device, data.rewards, reward, episode_i, step_i);
+        }
+        template<typename DEVICE, typename TI, typename SPEC>
+        RL_TOOLS_FUNCTION_PLACEMENT void set_terminated(DEVICE& device, rl::utils::evaluation::NoData<SPEC>& data, TI episode_i, TI step_i, bool terminated){}
+        template<typename DEVICE, typename TI, typename SPEC>
+        RL_TOOLS_FUNCTION_PLACEMENT void set_terminated(DEVICE& device, rl::utils::evaluation::Data<SPEC>& data, TI episode_i, TI step_i, bool terminated){
+            set(device, data.terminated, terminated, episode_i, step_i);
+        }
+    }
+    template<typename DEVICE, typename ENVIRONMENT, typename UI, typename POLICY, typename POLICY_STATE, typename POLICY_EVALUATION_BUFFERS, typename BUFFER_SPEC, typename DATA_SPEC, typename RNG, typename SPEC, template <typename> typename DATA, typename MODE>
+    RL_TOOLS_FUNCTION_PLACEMENT void evaluate(DEVICE& device, ENVIRONMENT& env_init, UI& ui, const POLICY& policy, POLICY_STATE& policy_state, POLICY_EVALUATION_BUFFERS& policy_evaluation_buffers, rl::utils::evaluation::Buffer<BUFFER_SPEC>& evaluation_buffers, rl::utils::evaluation::Result<SPEC>& results, DATA<DATA_SPEC>& data, RNG &rng, const Mode<MODE>& mode){
+        static_assert(utils::typing::is_same_v<SPEC, typename BUFFER_SPEC::SPEC>);
+        static_assert(utils::typing::is_same_v<SPEC, typename DATA_SPEC::SPEC>);
+        using T = typename POLICY::TYPE_POLICY::DEFAULT;
+        using TI = typename DEVICE::index_t;
+        constexpr TI OUTPUT_DIM = get_last(typename POLICY::OUTPUT_SHAPE{});
+        static_assert(ENVIRONMENT::ACTION_DIM == OUTPUT_DIM || (2*ENVIRONMENT::ACTION_DIM == OUTPUT_DIM), "Action and policy output dimensions must match");
+        results.returns_mean = 0;
+        results.returns_std = 0;
+        results.episode_length_mean = 0;
+        results.episode_length_std = 0;
+        results.num_terminated = 0;
+
+        auto actions_buffer = view(device, evaluation_buffers.actions, matrix::ViewSpec<SPEC::N_EPISODES, ENVIRONMENT::ACTION_DIM>{});
+
+        bool terminated[SPEC::N_EPISODES];
+        // using ADJUSTED_POLICY = typename POLICY::template CHANGE_BATCH_SIZE<TI, SPEC::N_EPISODES>;
+        // typename ADJUSTED_POLICY::template State<true> policy_state;
+        auto& envs = evaluation_buffers.envs;
+        auto& states = evaluation_buffers.states;
+        auto& parameters = evaluation_buffers.parameters;
+
+        reset(device, policy, policy_state, rng);
+        for(TI env_i = 0; env_i < SPEC::N_EPISODES; env_i++){
+            auto& env = get_ref(device, envs, env_i);
+            env = env_init;
+            malloc(device, env);
+            results.returns[env_i] = 0;
+            results.episode_length[env_i] = 0;
+            terminated[env_i] = false;
+            auto& state = get_ref(device, states, env_i);
+            auto& current_parameters = get_ref(device, parameters, env_i);
+            if constexpr(SPEC::DETERMINISTIC_INITIAL_STATE) {
+                rl_tools::initial_parameters(device, env, current_parameters);
+                rl_tools::initial_state(device, env, current_parameters, state);
+            }
+            else{
+                sample_initial_parameters(device, env, current_parameters, rng);
+                sample_initial_state(device, env, current_parameters, state, rng);
+            }
+            rl::utils::evaluation::set_parameters(device, data, env_i, current_parameters);
+        }
+        for(TI step_i = 0; step_i < SPEC::STEP_LIMIT; step_i++){
+            for(TI env_i = 0; env_i < SPEC::N_EPISODES; env_i++){
+                auto observation = row(device, evaluation_buffers.observations, env_i);
+                auto& state = get_ref(device, states, env_i);
+                auto& env_parameters = get_ref(device, parameters, env_i);
+                rl::utils::evaluation::set_state(device, data, env_i, step_i, state);
+                auto& env = get_ref(device, envs, env_i);
+                observe(device, env, env_parameters, state, typename ENVIRONMENT::Observation{}, observation, rng);
+            }
+            auto observations_chunk = view(device, evaluation_buffers.observations, matrix::ViewSpec<SPEC::N_EPISODES, ENVIRONMENT::Observation::DIM>{}, 0, 0);
+            auto actions_buffer_chunk = view(device, evaluation_buffers.actions, matrix::ViewSpec<SPEC::N_EPISODES, ENVIRONMENT::ACTION_DIM>{}, 0, 0);
+            auto input_tensor_flat = to_tensor(device, observations_chunk);
+            using OBS_SHAPE = typename ENVIRONMENT::Observation::SHAPE;
+            using INPUT_SHAPE = tensor::Prepend<OBS_SHAPE, SPEC::N_EPISODES>;
+            auto input_tensor = view_memory<INPUT_SHAPE>(device, input_tensor_flat);
+            auto output_tensor = to_tensor(device, actions_buffer_chunk);
+
+            evaluate_step(device, policy, input_tensor, policy_state, output_tensor, policy_evaluation_buffers, rng, mode);
+            rl::utils::evaluation::set_action(device, data, step_i, actions_buffer);
+            for(TI env_i = 0; env_i < SPEC::N_EPISODES; env_i++) {
+                if(step_i > 0){
+                    if(terminated[env_i]){
+                        continue;
+                    }
+                }
+                auto& env = get_ref(device, envs, env_i);
+                typename ENVIRONMENT::State next_state;
+                auto& state = get_ref(device, states, env_i);
+                auto& env_parameters = get_ref(device, parameters, env_i);
+                auto action = row(device, actions_buffer, env_i);
+                T dt = step(device, env, env_parameters, state, action, next_state, rng);
+                if(env_i == 0 && !terminated[env_i]){ // only render the first environment
+                    set_state(device, env, env_parameters, ui, state, action);
+                    render(device, env, env_parameters, ui);
+                }
+                rl::utils::evaluation::set_dt(device, data, env_i, step_i, dt);
+                T r = reward(device, env, env_parameters, state, action, next_state, rng);
+                rl::utils::evaluation::set_reward(device, data, env_i, step_i, r);
+                bool terminated_flag = rl_tools::terminated(device, env, env_parameters, next_state, rng);
+                if(!terminated[env_i]){
+                    // count the final step as well (e.g. termination penalty)
+                    results.returns[env_i] += r;
+                    results.episode_length[env_i] += 1;
+                    results.num_terminated += terminated_flag ? 1 : 0;
+                }
+                terminated_flag = terminated_flag || terminated[env_i];
+                terminated[env_i] = terminated_flag;
+                rl::utils::evaluation::set_terminated(device, data, env_i, step_i, terminated_flag);
+                if(terminated_flag){
+                    set_truncated(device, env, env_parameters, ui, next_state); // this is to sed the terminated flag to the car env
+                    render(device, env, env_parameters, ui);
+                }
+                state = next_state;
+            }
+        }
+        for(TI env_i = 0; env_i < SPEC::N_EPISODES; env_i++) {
+            auto& env_parameters = get_ref(device, parameters, env_i);
+            auto& env = get_ref(device, envs, env_i);
+            auto& state = get_ref(device, states, env_i);
+            set_truncated(device, env, env_parameters, ui, state); // this is to sed the terminated flag to the car env
+            render(device, env, env_parameters, ui);
+        }
+        for(TI env_i = 0; env_i < SPEC::N_EPISODES; env_i++) {
+            auto &env = get_ref(device, envs, env_i);
+            free(device, env);
+        }
+        for(TI env_i = 0; env_i < SPEC::N_EPISODES; env_i++){
+            results.returns[env_i] = results.returns[env_i];
+            results.returns_mean += results.returns[env_i];
+            results.returns_std += results.returns[env_i]*results.returns[env_i];
+            results.episode_length[env_i] = results.episode_length[env_i];
+            results.episode_length_mean += results.episode_length[env_i];
+            results.episode_length_std += results.episode_length[env_i]*results.episode_length[env_i];
+        }
+        results.returns_mean /= SPEC::N_EPISODES;
+        results.returns_std = math::sqrt(device.math, math::max(device.math, (T)0, results.returns_std/SPEC::N_EPISODES - results.returns_mean*results.returns_mean));
+        results.episode_length_mean /= SPEC::N_EPISODES;
+        results.episode_length_std = math::sqrt(device.math, math::max(device.math, (T)0, results.episode_length_std/SPEC::N_EPISODES - results.episode_length_mean*results.episode_length_mean));
+        results.share_terminated = results.num_terminated / (T)SPEC::N_EPISODES;
+    }
+    template<typename DEVICE, typename ENVIRONMENT, typename UI, typename POLICY, typename POLICY_BUFFER_SPEC, typename RESULT_SPEC, template <typename> typename DATA, typename DATA_SPEC, typename RNG, typename MODE>
+    RL_TOOLS_FUNCTION_PLACEMENT void evaluate(DEVICE& device, ENVIRONMENT& env_init, UI& ui, const POLICY& policy, rl::utils::evaluation::PolicyBuffer<POLICY_BUFFER_SPEC>& buffer, rl::utils::evaluation::Result<RESULT_SPEC>& results, DATA<DATA_SPEC>& data, RNG &rng, const Mode<MODE>& mode){
+        evaluate(device, env_init, ui, policy, buffer.policy_state, buffer.policy_evaluation_buffers, buffer.buffer, results, data, rng, mode);
+    }
+    template<typename DEVICE, typename ENVIRONMENT, typename UI, typename POLICY, typename POLICY_BUFFER_SPEC, typename RESULT_SPEC, typename RNG, typename MODE>
+    RL_TOOLS_FUNCTION_PLACEMENT void evaluate(DEVICE& device, ENVIRONMENT& env_init, UI& ui, const POLICY& policy, rl::utils::evaluation::PolicyBuffer<POLICY_BUFFER_SPEC>& buffer, rl::utils::evaluation::Result<RESULT_SPEC>& results, RNG &rng, const Mode<MODE>& mode){
+        rl::utils::evaluation::NoData<rl::utils::evaluation::DataSpecification<RESULT_SPEC>> data;
+        evaluate(device, env_init, ui, policy, buffer, results, data, rng, mode);
+    }
+    template<typename DEVICE, typename ENVIRONMENT, typename UI, typename POLICY, typename RNG, typename SPEC, template <typename> typename DATA, typename DATA_SPEC, typename MODE>
+    RL_TOOLS_FUNCTION_PLACEMENT void evaluate(DEVICE& device, ENVIRONMENT& env_init, UI& ui, const POLICY& policy, rl::utils::evaluation::Result<SPEC>& results, DATA<DATA_SPEC>& data, RNG &rng, const Mode<MODE>& mode){
+        using TI = typename DEVICE::index_t;
+        static constexpr bool DYNAMIC_ALLOCATION = true;
+        using ADJUSTED_POLICY = typename POLICY::template CHANGE_BATCH_SIZE<TI, SPEC::N_EPISODES>;
+        typename ADJUSTED_POLICY::template State<DYNAMIC_ALLOCATION> policy_state;
+        typename ADJUSTED_POLICY::template Buffer<DYNAMIC_ALLOCATION> policy_evaluation_buffers;
+        rl::utils::evaluation::Buffer<rl::utils::evaluation::BufferSpecification<SPEC, DYNAMIC_ALLOCATION>> evaluation_buffers;
+        malloc(device, policy_state);
+        malloc(device, policy_evaluation_buffers);
+        malloc(device, evaluation_buffers);
+        evaluate(device, env_init, ui, policy, policy_state, policy_evaluation_buffers, evaluation_buffers, results, data, rng, mode);
+        free(device, policy_state);
+        free(device, policy_evaluation_buffers);
+        free(device, evaluation_buffers);
+    }
+    // proxies without data collection
+    template<typename DEVICE, typename ENVIRONMENT, typename UI, typename POLICY, typename POLICY_STATE, typename POLICY_EVALUATION_BUFFERS, typename BUFFER_SPEC, typename RNG, typename SPEC, typename MODE>
+    RL_TOOLS_FUNCTION_PLACEMENT void evaluate(DEVICE& device, ENVIRONMENT& env_init, UI& ui, const POLICY& policy, POLICY_STATE& policy_state, POLICY_EVALUATION_BUFFERS& policy_evaluation_buffers, rl::utils::evaluation::Buffer<BUFFER_SPEC>& evaluation_buffers, rl::utils::evaluation::Result<SPEC>& results, RNG &rng, const Mode<MODE>& mode){
+        rl::utils::evaluation::NoData<rl::utils::evaluation::DataSpecification<SPEC>> data;
+        evaluate(device, env_init, ui, policy, policy_state, policy_evaluation_buffers, evaluation_buffers, results, data, rng, mode);
+    }
+    template<typename DEVICE, typename ENVIRONMENT, typename UI, typename POLICY, typename RNG, typename SPEC, typename MODE>
+    RL_TOOLS_FUNCTION_PLACEMENT void evaluate(DEVICE& device, ENVIRONMENT& env_init, UI& ui, const POLICY& policy, rl::utils::evaluation::Result<SPEC>& results, RNG &rng, const Mode<MODE>& mode){
+        rl::utils::evaluation::NoData<rl::utils::evaluation::DataSpecification<SPEC>> data;
+        evaluate(device, env_init, ui, policy, results, data, rng, mode);
+    }
+}
+RL_TOOLS_NAMESPACE_WRAPPER_END
+
+#endif