hyperray 0.0.1__py3-none-any.whl

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/loop/steps/checkpoint/operations_cpu.h

@@ -0,0 +1,233 @@
+#include "../../../../version.h"
+#if (defined(RL_TOOLS_DISABLE_INCLUDE_GUARDS) || !defined(RL_TOOLS_RL_LOOP_STEPS_CHECKPOINT_OPERATIONS_GENERIC_H)) && (RL_TOOLS_USE_THIS_VERSION == 1)
+#pragma once
+#define RL_TOOLS_RL_LOOP_STEPS_CHECKPOINT_OPERATIONS_GENERIC_H
+
+#include "config.h"
+
+#ifdef RL_TOOLS_ENABLE_ZLIB
+#include <zlib.h>
+#include <cstring>
+#endif
+
+#if defined(RL_TOOLS_ENABLE_HDF5) && !defined(RL_TOOLS_DISABLE_HDF5)
+#include "../../../../persist/backends/hdf5/hdf5.h"
+#else
+#include "../../../../persist/backends/tar/operations_cpu.h"
+#endif
+
+#include "../../../../numeric_types/persist_code.h"
+#include "../../../../containers/matrix/persist_code.h"
+#include "../../../../containers/tensor/persist_code.h"
+#include "../../../../nn/optimizers/adam/instance/persist_code.h"
+#include "../../../../nn/parameters/persist_code.h"
+#include "../../../../nn/layers/dense/persist_code.h"
+#include "../../../../nn/layers/gru/persist_code.h"
+#include "../../../../nn/layers/sample_and_squash/persist_code.h"
+#include "../../../../nn_models/mlp/persist_code.h"
+#include "../../../../nn_models/sequential/persist_code.h"
+
+
+
+#include "../../../../utils/zlib/operations_cpu.h"
+#include "../../../../utils/extrack/operations_cpu.h"
+
+#include <filesystem>
+#include <iostream>
+#include <fstream>
+
+
+RL_TOOLS_NAMESPACE_WRAPPER_START
+namespace rl_tools{
+    template <typename DEVICE, typename T_CONFIG>
+    void malloc(DEVICE& device, rl::loop::steps::checkpoint::State<T_CONFIG>& ts){
+        using STATE = rl::loop::steps::checkpoint::State<T_CONFIG>;
+        malloc(device, static_cast<typename STATE::NEXT&>(ts));
+        malloc(device, ts.rng_checkpoint);
+    }
+
+    template <typename DEVICE, typename T_CONFIG>
+    void free(DEVICE& device, rl::loop::steps::checkpoint::State<T_CONFIG>& ts){
+        using STATE = rl::loop::steps::checkpoint::State<T_CONFIG>;
+        free(device, static_cast<typename STATE::NEXT&>(ts));
+        free(device, ts.rng_checkpoint);
+    }
+    template <typename DEVICE, typename T_CONFIG>
+    void init(DEVICE& device, rl::loop::steps::checkpoint::State<T_CONFIG>& ts, typename T_CONFIG::TI seed = 0){
+        using STATE = rl::loop::steps::checkpoint::State<T_CONFIG>;
+        init(device, static_cast<typename STATE::NEXT&>(ts), seed);
+        init(device, ts.rng_checkpoint, seed);
+        ts.checkpoint_this_step = false;
+    }
+
+    namespace rl::loop::steps::checkpoint{
+        template <bool DYNAMIC_ALLOCATION, typename ENVIRONMENT, typename DEVICE, typename ACTOR_TYPE, typename RNG, bool STORE_UNCOMPRESSED_ANYWAYS=true>
+        void save_code(DEVICE& device, const std::string step_folder, const std::filesystem::path& latest_folder, typename DEVICE::index_t step, ACTOR_TYPE& actor_forward, RNG& rng){
+            using T = typename ACTOR_TYPE::TYPE_POLICY::DEFAULT;
+            using TI = typename DEVICE::index_t;
+            std::filesystem::path step_folder_path = step_folder;
+            auto actor_weights = rl_tools::save_code(device, actor_forward, std::string("rl_tools::checkpoint::actor"), true);
+            std::stringstream output_ss;
+            output_ss << actor_weights;
+            { // note: this is duplicated for hdf5 (see futher down) and code
+                Tensor<tensor::Specification<T, TI, typename ACTOR_TYPE::INPUT_SHAPE, DYNAMIC_ALLOCATION>> input;
+                Tensor<tensor::Specification<T, TI, typename ACTOR_TYPE::OUTPUT_SHAPE, DYNAMIC_ALLOCATION>> output;
+                typename ACTOR_TYPE::template Buffer<DYNAMIC_ALLOCATION> actor_buffer;
+                malloc(device, input);
+                malloc(device, output);
+                malloc(device, actor_buffer);
+                randn(device, input, rng);
+                Mode<mode::Evaluation<>> mode;
+                evaluate(device, actor_forward, input, output, actor_buffer, rng, mode);
+                output_ss << "\n" << "namespace rl_tools::checkpoint::example::inputs{";
+                output_ss << "\n" << save_code(device, input, std::string("_0"), true);
+                output_ss << "\n" << "}";
+                output_ss << "\n" << "namespace rl_tools::checkpoint::example::outputs{";
+                output_ss << "\n" << save_code(device, output, std::string("_0"), true);
+                output_ss << "\n" << "}";
+                free(device, input);
+                free(device, output);
+                free(device, actor_buffer);
+            }
+            output_ss << "\n" << "namespace rl_tools::checkpoint::meta{";
+            output_ss << "\n" << "   " << "char name[] = \"" << step_folder << "\";";
+            output_ss << "\n" << "   " << "char commit_hash[] = \"" << RL_TOOLS_STRINGIFY(RL_TOOLS_COMMIT_HASH) << "\";";
+            output_ss << "\n" << "}";
+            { // Save environment
+                ENVIRONMENT env;
+                typename ENVIRONMENT::Parameters env_parameters;
+                malloc(device, env);
+                init(device, env);
+                initial_parameters(device, env, env_parameters);
+                auto env_info = save_code_env(device, env, env_parameters, "rl_tools::checkpoint::meta::environment");
+                free(device, env);
+                output_ss << "\n" << env_info;
+            }
+            std::string output_string = output_ss.str();
+            bool stored_compressed = false;
+#ifdef RL_TOOLS_ENABLE_ZLIB
+            {
+                std::filesystem::path checkpoint_code_path = step_folder_path / "checkpoint.h.gz";
+                std::vector<uint8_t> checkpoint_output;
+                if(!compress_zlib(output_string, checkpoint_output)){
+                    std::cerr << "Error while compressing trajectories." << std::endl;
+                    return;
+                }
+                std::ofstream actor_output_file(checkpoint_code_path, std::ios::binary);
+                actor_output_file.write(reinterpret_cast<const char*>(checkpoint_output.data()), checkpoint_output.size());
+                actor_output_file.close();
+                link_latest_artifact(device, latest_folder, checkpoint_code_path, step_folder_path, step);
+                stored_compressed = true;
+            };
+#endif
+            if(!stored_compressed || STORE_UNCOMPRESSED_ANYWAYS){
+                std::filesystem::create_directories(step_folder_path);
+                std::filesystem::path checkpoint_code_path = step_folder_path / "checkpoint.h";
+                std::cerr << "Checkpointing to: " << checkpoint_code_path << std::endl;
+                std::ofstream actor_output_file(checkpoint_code_path);
+                actor_output_file << output_string;
+                actor_output_file.close();
+                link_latest_artifact(device, latest_folder, checkpoint_code_path, step_folder_path, step);
+            }
+
+        }
+        template <bool DYNAMIC_ALLOCATION, typename ENVIRONMENT, typename DEVICE, typename ACTOR_TYPE, typename RNG, bool STORE_UNCOMPRESSED_ANYWAYS=true>
+        void save_code(DEVICE& device, const std::string step_folder, ACTOR_TYPE& actor_forward, RNG& rng){
+            save_code<DYNAMIC_ALLOCATION, ENVIRONMENT, DEVICE, ACTOR_TYPE, RNG, STORE_UNCOMPRESSED_ANYWAYS>(device, step_folder, std::filesystem::path{}, 0, actor_forward, rng);
+        }
+        template <bool DYNAMIC_ALLOCATION, typename ENVIRONMENT, typename CHECKPOINT_PARAMETERS, typename DEVICE, typename ACTOR, typename RNG>
+        void save(DEVICE& device, const std::string step_folder, const std::filesystem::path& latest_folder, typename DEVICE::index_t step, ACTOR& actor, RNG& rng){
+            using TI = typename DEVICE::index_t;
+            static constexpr TI BATCH_SIZE = CHECKPOINT_PARAMETERS::TEST_INPUT_BATCH_SIZE;
+            using INPUT_SHAPE = tensor::Replace<typename ACTOR::INPUT_SHAPE, BATCH_SIZE, 1>;
+            using EVALUATION_ACTOR_TYPE_BATCH_SIZE = typename ACTOR::template CHANGE_BATCH_SIZE<TI, BATCH_SIZE>;
+            using EVALUATION_ACTOR_TYPE = typename EVALUATION_ACTOR_TYPE_BATCH_SIZE::template CHANGE_CAPABILITY<nn::capability::Forward<DYNAMIC_ALLOCATION>>;
+            EVALUATION_ACTOR_TYPE evaluation_actor;
+            malloc(device, evaluation_actor);
+            copy(device, device, actor, evaluation_actor);
+            std::filesystem::path step_folder_path = step_folder;
+#if defined(RL_TOOLS_ENABLE_HDF5) && !defined(RL_TOOLS_DISABLE_HDF5)
+            std::filesystem::path checkpoint_path = step_folder_path / "checkpoint.h5";
+#else
+            std::filesystem::path checkpoint_path = step_folder_path / "checkpoint.tar";
+#endif
+            auto save_checkpoint_contents = [&](auto& root, auto& actor_group){
+                std::cerr << "Checkpointing to: " << checkpoint_path << std::endl;
+                set_attribute(device, actor_group, "checkpoint_name", step_folder.c_str());
+                ENVIRONMENT environment;
+                auto env_description = json(device, environment);
+                std::string meta = "{\"environment\": " + env_description + "}";
+                set_attribute(device, actor_group, "meta", meta.c_str());
+                rl_tools::save(device, evaluation_actor, actor_group);
+                using T = typename EVALUATION_ACTOR_TYPE::TYPE_POLICY::DEFAULT;
+                Tensor<tensor::Specification<T, TI, typename EVALUATION_ACTOR_TYPE::INPUT_SHAPE, DYNAMIC_ALLOCATION>> input;
+                Tensor<tensor::Specification<T, TI, typename EVALUATION_ACTOR_TYPE::OUTPUT_SHAPE, DYNAMIC_ALLOCATION>> output;
+                typename EVALUATION_ACTOR_TYPE::template Buffer<DYNAMIC_ALLOCATION> actor_buffer;
+                malloc(device, input);
+                malloc(device, output);
+                malloc(device, actor_buffer);
+                randn(device, input, rng);
+                Mode<mode::Evaluation<>> mode;
+                evaluate(device, actor, input, output, actor_buffer, rng, mode);
+                auto example_group = create_group(device, root, "example");
+                auto inputs_group = create_group(device, example_group, "inputs");
+                save(device, input, inputs_group, "0");
+                auto outputs_group = create_group(device, example_group, "outputs");
+                save(device, output, outputs_group, "0");
+                free(device, input);
+                free(device, output);
+                free(device, actor_buffer);
+            };
+            try{
+#if defined(RL_TOOLS_ENABLE_HDF5) && !defined(RL_TOOLS_DISABLE_HDF5)
+                {
+                    std::lock_guard<std::mutex> lock(persist::backends::hdf5::global_mutex());
+                    persist::backends::hdf5::File root_file(checkpoint_path.string(), persist::backends::hdf5::Mode::WRITE);
+                    auto actor_group = create_group(device, root_file, "actor");
+                    save_checkpoint_contents(root_file, actor_group);
+                }
+#else
+                persist::backends::tar::Writer writer;
+                persist::backends::tar::WriterGroup<persist::backends::tar::WriterGroupSpecification<TI, decltype(writer)>> root_group{"", &writer};
+                auto actor_group = create_group(device, root_group, "actor");
+                save_checkpoint_contents(root_group, actor_group);
+                persist::backends::tar::finalize(device, writer);
+                auto actor_file = std::ofstream(checkpoint_path, std::ios::binary);
+                actor_file.write(writer.buffer.data(), writer.buffer.size());
+                actor_file.close();
+#endif
+                link_latest_artifact(device, latest_folder, checkpoint_path, step_folder_path, step);
+            }
+            catch(std::exception& e){
+                std::cerr << "Error while saving actor at " + checkpoint_path.string() + ": " << e.what() << std::endl;
+            }
+            rl::loop::steps::checkpoint::save_code<DYNAMIC_ALLOCATION, ENVIRONMENT>(device, step_folder, latest_folder, step, evaluation_actor, rng);
+            free(device, evaluation_actor);
+
+        }
+        template <bool DYNAMIC_ALLOCATION, typename ENVIRONMENT, typename CHECKPOINT_PARAMETERS, typename DEVICE, typename ACTOR, typename RNG>
+        void save(DEVICE& device, const std::string step_folder, ACTOR& actor, RNG& rng){
+            save<DYNAMIC_ALLOCATION, ENVIRONMENT, CHECKPOINT_PARAMETERS>(device, step_folder, std::filesystem::path{}, 0, actor, rng);
+        }
+    }
+
+    template <typename DEVICE, typename CONFIG>
+    bool step(DEVICE& device, rl::loop::steps::checkpoint::State<CONFIG>& ts){
+        using TYPE_POLICY = typename CONFIG::TYPE_POLICY;
+        using TI = typename CONFIG::TI;
+        using STATE = rl::loop::steps::checkpoint::State<CONFIG>;
+        if(ts.step % CONFIG::CHECKPOINT_PARAMETERS::CHECKPOINT_INTERVAL == 0 || ts.checkpoint_this_step){
+            ts.checkpoint_this_step = false;
+            auto step_folder = get_step_folder(device, ts.extrack_config, ts.extrack_paths, ts.step);
+            auto latest_folder = get_latest_folder(device, ts.extrack_paths);
+            auto& actor = get_actor(ts);
+            rl::loop::steps::checkpoint::save<CONFIG::DYNAMIC_ALLOCATION, typename CONFIG::ENVIRONMENT, typename CONFIG::CHECKPOINT_PARAMETERS>(device, step_folder.string(), latest_folder, ts.step, actor, ts.rng_checkpoint);
+        }
+        bool finished = step(device, static_cast<typename STATE::NEXT&>(ts));
+        return finished;
+    }
+}
+RL_TOOLS_NAMESPACE_WRAPPER_END
+
+
+#endif

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/loop/steps/checkpoint/persist.h

@@ -0,0 +1,30 @@
+#include "../../../../version.h"
+#if (defined(RL_TOOLS_DISABLE_INCLUDE_GUARDS) || !defined(RL_TOOLS_RL_LOOP_STEPS_CHECKPOINT_PERSIST_H)) && (RL_TOOLS_USE_THIS_VERSION == 1)
+#pragma once
+#define RL_TOOLS_RL_LOOP_STEPS_CHECKPOINT_PERSIST_H
+#include "state.h"
+#include "../../../../random/persist.h"
+RL_TOOLS_NAMESPACE_WRAPPER_START
+namespace rl_tools{
+    template <typename DEVICE, typename T_CONFIG, typename GROUP>
+    void save(DEVICE& device, rl::loop::steps::checkpoint::State<T_CONFIG>& ts, GROUP& group){
+        using STATE = rl::loop::steps::checkpoint::State<T_CONFIG>;
+        save(device, static_cast<typename STATE::NEXT&>(ts), group);
+        auto checkpoint_group = create_group(device, group, "checkpoint");
+        auto rng_group = create_group(device, checkpoint_group, "rng_checkpoint");
+        save(device, ts.rng_checkpoint, rng_group);
+    }
+    template <typename DEVICE, typename T_CONFIG, typename GROUP>
+    bool load(DEVICE& device, rl::loop::steps::checkpoint::State<T_CONFIG>& ts, GROUP& group){
+        using STATE = rl::loop::steps::checkpoint::State<T_CONFIG>;
+        bool success = load(device, static_cast<typename STATE::NEXT&>(ts), group);
+        auto checkpoint_group = get_group(device, group, "checkpoint");
+        auto rng_group = get_group(device, checkpoint_group, "rng_checkpoint");
+        bool step_result = load(device, ts.rng_checkpoint, rng_group);
+        if(!step_result){ log(device, device.logger, "Checkpoint loop load failed: rng_checkpoint"); }
+        success &= step_result;
+        return success;
+    }
+}
+RL_TOOLS_NAMESPACE_WRAPPER_END
+#endif

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/loop/steps/checkpoint/state.h

@@ -0,0 +1,21 @@
+#include "../../../../version.h"
+#if (defined(RL_TOOLS_DISABLE_INCLUDE_GUARDS) || !defined(RL_TOOLS_RL_LOOP_STEPS_CHECKPOINT_STATE_H)) && (RL_TOOLS_USE_THIS_VERSION == 1)
+#pragma once
+#define RL_TOOLS_RL_LOOP_STEPS_CHECKPOINT_STATE_H
+
+RL_TOOLS_NAMESPACE_WRAPPER_START
+namespace rl_tools::rl::loop::steps::checkpoint{
+    template<typename T_CONFIG, typename T_NEXT = typename T_CONFIG::NEXT::template State<typename T_CONFIG::NEXT>>
+    struct State: T_NEXT {
+        using CONFIG = T_CONFIG;
+        using NEXT = T_NEXT;
+        typename CONFIG::RNG rng_checkpoint;
+        bool checkpoint_this_step = false;
+    };
+}
+RL_TOOLS_NAMESPACE_WRAPPER_END
+#endif
+
+
+
+

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/loop/steps/curriculum/config.h

@@ -0,0 +1,25 @@
+#include "../../../../version.h"
+#if (defined(RL_TOOLS_DISABLE_INCLUDE_GUARDS) || !defined(RL_TOOLS_RL_LOOP_STEPS_CURRICULUM_CONFIG_H)) && (RL_TOOLS_USE_THIS_VERSION == 1)
+#pragma once
+#define RL_TOOLS_RL_LOOP_STEPS_CURRICULUM_CONFIG_H
+
+#include "state.h"
+
+RL_TOOLS_NAMESPACE_WRAPPER_START
+namespace rl_tools::rl::loop::steps::curriculum{
+    struct DefaultTag{};
+    template<typename T_TI, T_TI T_INTERVAL=0>
+    struct Parameters{
+        static constexpr T_TI INTERVAL = T_INTERVAL; // 0 = use evaluation interval
+    };
+    template<typename T_NEXT, typename T_PARAMETERS = Parameters<typename T_NEXT::TI>, typename T_TAG = DefaultTag>
+    struct Config: T_NEXT {
+        using NEXT = T_NEXT;
+        using CURRICULUM_PARAMETERS = T_PARAMETERS;
+        using CURRICULUM_TAG = T_TAG;
+        template <typename CONFIG>
+        using State = State<CONFIG>;
+    };
+}
+RL_TOOLS_NAMESPACE_WRAPPER_END
+#endif

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/loop/steps/curriculum/operations_generic.h

@@ -0,0 +1,47 @@
+#include "../../../../version.h"
+#if (defined(RL_TOOLS_DISABLE_INCLUDE_GUARDS) || !defined(RL_TOOLS_RL_LOOP_STEPS_CURRICULUM_OPERATIONS_GENERIC_H)) && (RL_TOOLS_USE_THIS_VERSION == 1)
+#pragma once
+#define RL_TOOLS_RL_LOOP_STEPS_CURRICULUM_OPERATIONS_GENERIC_H
+
+#include "config.h"
+
+RL_TOOLS_NAMESPACE_WRAPPER_START
+namespace rl_tools{
+    template <typename DEVICE, typename T_CONFIG>
+    RL_TOOLS_FUNCTION_PLACEMENT void curriculum(DEVICE& device, rl::loop::steps::curriculum::State<T_CONFIG>& ts, rl::loop::steps::curriculum::DefaultTag){
+        // default: no-op
+    }
+    template <typename DEVICE, typename T_CONFIG>
+    RL_TOOLS_FUNCTION_PLACEMENT void malloc(DEVICE& device, rl::loop::steps::curriculum::State<T_CONFIG>& ts){
+        using STATE = rl::loop::steps::curriculum::State<T_CONFIG>;
+        malloc(device, static_cast<typename STATE::NEXT&>(ts));
+    }
+    template <typename DEVICE, typename T_CONFIG>
+    RL_TOOLS_FUNCTION_PLACEMENT void free(DEVICE& device, rl::loop::steps::curriculum::State<T_CONFIG>& ts){
+        using STATE = rl::loop::steps::curriculum::State<T_CONFIG>;
+        free(device, static_cast<typename STATE::NEXT&>(ts));
+    }
+    template <typename DEVICE, typename T_CONFIG>
+    RL_TOOLS_FUNCTION_PLACEMENT void init(DEVICE& device, rl::loop::steps::curriculum::State<T_CONFIG>& ts, typename T_CONFIG::TI seed = 0){
+        using STATE = rl::loop::steps::curriculum::State<T_CONFIG>;
+        init(device, static_cast<typename STATE::NEXT&>(ts), seed);
+        ts.curriculum_level = 0;
+    }
+    template <typename DEVICE, typename CONFIG>
+    RL_TOOLS_FUNCTION_PLACEMENT bool step(DEVICE& device, rl::loop::steps::curriculum::State<CONFIG>& ts){
+        using TI = typename CONFIG::TI;
+        using STATE = rl::loop::steps::curriculum::State<CONFIG>;
+        using PARAMS = typename CONFIG::CURRICULUM_PARAMETERS;
+        constexpr TI EVAL_INTERVAL = CONFIG::EVALUATION_PARAMETERS::EVALUATION_INTERVAL;
+        constexpr TI INTERVAL = PARAMS::INTERVAL > 0 ? PARAMS::INTERVAL : EVAL_INTERVAL;
+        TI step_before = ts.step;
+        bool finished = step(device, static_cast<typename STATE::NEXT&>(ts));
+        if(step_before % INTERVAL == 0){
+            curriculum(device, ts, typename CONFIG::CURRICULUM_TAG{});
+            add_scalar(device, device.logger, "curriculum/level", ts.curriculum_level);
+        }
+        return finished;
+    }
+}
+RL_TOOLS_NAMESPACE_WRAPPER_END
+#endif

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/loop/steps/curriculum/persist.h

@@ -0,0 +1,20 @@
+#include "../../../../version.h"
+#if (defined(RL_TOOLS_DISABLE_INCLUDE_GUARDS) || !defined(RL_TOOLS_RL_LOOP_STEPS_CURRICULUM_PERSIST_H)) && (RL_TOOLS_USE_THIS_VERSION == 1)
+#pragma once
+#define RL_TOOLS_RL_LOOP_STEPS_CURRICULUM_PERSIST_H
+#include "state.h"
+RL_TOOLS_NAMESPACE_WRAPPER_START
+namespace rl_tools{
+    template <typename DEVICE, typename T_CONFIG, typename GROUP>
+    void save(DEVICE& device, rl::loop::steps::curriculum::State<T_CONFIG>& ts, GROUP& group){
+        using STATE = rl::loop::steps::curriculum::State<T_CONFIG>;
+        save(device, static_cast<typename STATE::NEXT&>(ts), group);
+    }
+    template <typename DEVICE, typename T_CONFIG, typename GROUP>
+    bool load(DEVICE& device, rl::loop::steps::curriculum::State<T_CONFIG>& ts, GROUP& group){
+        using STATE = rl::loop::steps::curriculum::State<T_CONFIG>;
+        return load(device, static_cast<typename STATE::NEXT&>(ts), group);
+    }
+}
+RL_TOOLS_NAMESPACE_WRAPPER_END
+#endif

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/loop/steps/curriculum/state.h

@@ -0,0 +1,17 @@
+#include "../../../../version.h"
+#if (defined(RL_TOOLS_DISABLE_INCLUDE_GUARDS) || !defined(RL_TOOLS_RL_LOOP_STEPS_CURRICULUM_STATE_H)) && (RL_TOOLS_USE_THIS_VERSION == 1)
+#pragma once
+#define RL_TOOLS_RL_LOOP_STEPS_CURRICULUM_STATE_H
+
+RL_TOOLS_NAMESPACE_WRAPPER_START
+namespace rl_tools::rl::loop::steps::curriculum{
+    template<typename T_CONFIG, typename T_NEXT = typename T_CONFIG::NEXT::template State<typename T_CONFIG::NEXT>>
+    struct State: T_NEXT {
+        using CONFIG = T_CONFIG;
+        using NEXT = T_NEXT;
+        using T = typename CONFIG::TYPE_POLICY::DEFAULT;
+        T curriculum_level;
+    };
+}
+RL_TOOLS_NAMESPACE_WRAPPER_END
+#endif

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/loop/steps/evaluation/config.h

@@ -0,0 +1,47 @@
+#include "../../../../version.h"
+#if (defined(RL_TOOLS_DISABLE_INCLUDE_GUARDS) || !defined(RL_TOOLS_RL_LOOP_STEPS_EVALUATION_CONFIG_H)) && (RL_TOOLS_USE_THIS_VERSION == 1)
+#pragma once
+#define RL_TOOLS_RL_LOOP_STEPS_EVALUATION_CONFIG_H
+
+#include "../../../../rl/environments/environments.h"
+
+#include "state.h"
+
+RL_TOOLS_NAMESPACE_WRAPPER_START
+namespace rl_tools::rl::loop::steps::evaluation{
+    struct ParametersTag{};
+    template <typename TYPE_POLICY, typename TI, typename NEXT, TI T_NUM_EVALUATION_EPISODES = 10, TI T_EVALUATION_INTERVAL = 1000>
+    struct Parameters{
+        using TAG = ParametersTag;
+        static constexpr bool DETERMINISTIC_EVALUATION = true;
+        static constexpr TI EVALUATION_INTERVAL = T_EVALUATION_INTERVAL;
+        static constexpr TI NUM_EVALUATION_EPISODES = T_NUM_EVALUATION_EPISODES;
+        static constexpr TI N_EVALUATIONS = NEXT::CORE_PARAMETERS::STEP_LIMIT / EVALUATION_INTERVAL + 1;
+        static constexpr TI EPISODE_STEP_LIMIT = NEXT::CORE_PARAMETERS::EPISODE_STEP_LIMIT;
+        using EVALUATION_MODE = mode::Evaluation<>;
+        static constexpr bool DETERMINISTIC_INITIAL_STATE = false;
+        static constexpr bool SAMPLE_ENVIRONMENT_PARAMETERS = true;
+    };
+    struct ConfigTag{};
+    template<typename T_NEXT, typename T_PARAMETERS = Parameters<typename T_NEXT::TYPE_POLICY, typename T_NEXT::TI, T_NEXT>, typename T_UI = environments::DummyUI>
+    struct Config: T_NEXT {
+        using TAG = ConfigTag;
+        using TYPE_POLICY = typename T_NEXT::TYPE_POLICY;
+        using NEXT = T_NEXT;
+        using EVALUATION_PARAMETERS = T_PARAMETERS;
+        using UI = T_UI;
+        using TI = typename NEXT::TI;
+        using EVALUATION_SPEC = rl::utils::evaluation::Specification<TYPE_POLICY, TI, typename NEXT::ENVIRONMENT_EVALUATION, EVALUATION_PARAMETERS::NUM_EVALUATION_EPISODES, EVALUATION_PARAMETERS::EPISODE_STEP_LIMIT>;
+        using EVALUATION_RESULT_SPEC = rl::utils::evaluation::Specification<TYPE_POLICY, TI, typename NEXT::ENVIRONMENT_EVALUATION, EVALUATION_PARAMETERS::NUM_EVALUATION_EPISODES, EVALUATION_PARAMETERS::EPISODE_STEP_LIMIT, EVALUATION_PARAMETERS::DETERMINISTIC_INITIAL_STATE>;
+        static_assert(EVALUATION_PARAMETERS::N_EVALUATIONS > 0);
+        static_assert(EVALUATION_PARAMETERS::N_EVALUATIONS < 1000000);
+        template <typename CONFIG>
+        using State = State<CONFIG>;
+    };
+}
+RL_TOOLS_NAMESPACE_WRAPPER_END
+#endif
+
+
+
+

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/loop/steps/evaluation/operations_generic.h

@@ -0,0 +1,106 @@
+#include "../../../../version.h"
+#if (defined(RL_TOOLS_DISABLE_INCLUDE_GUARDS) || !defined(RL_TOOLS_RL_LOOP_STEPS_EVALUATION_OPERATIONS_GENERIC_H)) && (RL_TOOLS_USE_THIS_VERSION == 1)
+#pragma once
+#define RL_TOOLS_RL_LOOP_STEPS_EVALUATION_OPERATIONS_GENERIC_H
+
+#include "../../../../rl/algorithms/sac/operations_generic.h"
+#include "../../../../rl/components/off_policy_runner/operations_generic.h"
+
+#include "../../../../rl/environments/operations_generic.h"
+
+
+#include "../../../../rl/utils/evaluation/operations_generic.h"
+
+#include "config.h"
+
+RL_TOOLS_NAMESPACE_WRAPPER_START
+namespace rl_tools{
+    template <typename DEVICE, typename T_CONFIG>
+    RL_TOOLS_FUNCTION_PLACEMENT void malloc(DEVICE& device, rl::loop::steps::evaluation::State<T_CONFIG>& ts){
+        using STATE = rl::loop::steps::evaluation::State<T_CONFIG>;
+        malloc(device, ts.rng_eval);
+        malloc(device, ts.rng_eval_on_demand);
+        malloc(device, ts.env_eval);
+        malloc(device, ts.actor_deterministic_evaluation_state);
+        malloc(device, ts.actor_deterministic_evaluation_buffers);
+        malloc(device, ts.evaluation_buffer);
+        malloc(device, ts.evaluation_results);
+        malloc(device, static_cast<typename STATE::NEXT&>(ts));
+    }
+    template <typename DEVICE, typename T_CONFIG>
+    RL_TOOLS_FUNCTION_PLACEMENT void free(DEVICE& device, rl::loop::steps::evaluation::State<T_CONFIG>& ts){
+        using STATE = rl::loop::steps::evaluation::State<T_CONFIG>;
+        free(device, ts.rng_eval);
+        free(device, ts.rng_eval_on_demand);
+        free(device, ts.env_eval);
+        free(device, ts.actor_deterministic_evaluation_buffers);
+        free(device, ts.evaluation_buffer);
+        free(device, ts.evaluation_results);
+        free(device, static_cast<typename STATE::NEXT&>(ts));
+    }
+    template <typename DEVICE, typename T_CONFIG>
+    RL_TOOLS_FUNCTION_PLACEMENT void init(DEVICE& device, rl::loop::steps::evaluation::State<T_CONFIG>& ts, typename T_CONFIG::TI seed = 0){
+        using STATE = rl::loop::steps::evaluation::State<T_CONFIG>;
+        init(device, static_cast<typename STATE::NEXT&>(ts), seed);
+        init(device, ts.env_eval);
+        initial_parameters(device, ts.env_eval, ts.env_eval_parameters);
+        init(device, ts.env_eval, ts.env_eval_parameters, ts.ui);
+        initial_parameters(device, ts.env_eval, ts.env_eval_parameters);
+        init(device, ts.rng_eval, seed);
+        init(device, ts.rng_eval_on_demand, seed);
+        ts.evaluate_this_step = false;
+    }
+
+
+    template <typename DEVICE, typename CONFIG>
+    RL_TOOLS_FUNCTION_PLACEMENT bool step(DEVICE& device, rl::loop::steps::evaluation::State<CONFIG>& ts){
+        using TS = rl::loop::steps::evaluation::State<CONFIG>;
+        using TI = typename CONFIG::TI;
+        using PARAMETERS = typename CONFIG::EVALUATION_PARAMETERS;
+        using STATE = rl::loop::steps::evaluation::State<CONFIG>;
+        if constexpr(PARAMETERS::DETERMINISTIC_EVALUATION == true){
+            TI evaluation_index = ts.step / PARAMETERS::EVALUATION_INTERVAL;
+            bool evaluate_scheduled = ts.step % PARAMETERS::EVALUATION_INTERVAL == 0 && evaluation_index < PARAMETERS::N_EVALUATIONS;
+            // if evaulation is schedule, it is saved into the pre-allocated buckets of the evaluation result
+            if(evaluate_scheduled || ts.evaluate_this_step){
+                ts.evaluate_this_step = false;
+                auto evaluate_fn = [&device, &ts](auto& result, auto& rng){
+                    // typename TS::EVALUATION_ACTOR_TYPE evaluation_actor;
+                    // malloc(device, evaluation_actor);
+                    auto actor = get_actor(ts);
+                    // copy(device, device, actor, evaluation_actor);
+                    evaluate(device, ts.env_eval, ts.ui, actor, ts.actor_deterministic_evaluation_state, ts.actor_deterministic_evaluation_buffers, ts.evaluation_buffer, result, rng, ts.evaluation_mode);
+                    // free(device, evaluation_actor);
+                    log(device, device.logger, "Step: ", ts.step, "/", CONFIG::CORE_PARAMETERS::STEP_LIMIT, " Mean return: ", result.returns_mean, " Mean episode length: ", result.episode_length_mean);
+                    add_scalar(device, device.logger, "evaluation/return/mean", result.returns_mean);
+                    add_scalar(device, device.logger, "evaluation/return/std", result.returns_std);
+                    add_scalar(device, device.logger, "evaluation/episode_length/mean", result.episode_length_mean);
+                    add_scalar(device, device.logger, "evaluation/episode_length/std", result.episode_length_std);
+                    add_scalar(device, device.logger, "evaluation/share_terminated", result.share_terminated);
+                };
+                if(evaluate_scheduled){
+                    auto& result = get(ts.evaluation_results, 0, evaluation_index);
+                    auto& rng = ts.rng_eval;
+                    evaluate_fn(result, rng);
+                }
+                else{
+                    utils::typing::remove_reference_t<decltype(get(ts.evaluation_results, 0, evaluation_index))> result;
+                    auto& rng = ts.rng_eval_on_demand;
+                    evaluate_fn(result, rng);
+                }
+            }
+        }
+        bool finished = step(device, static_cast<typename STATE::NEXT&>(ts));
+        return finished;
+    }
+    template <typename DEVICE, typename PARAMETERS, typename utils::typing::enable_if<utils::typing::is_same_v<typename PARAMETERS::TAG, rl::loop::steps::evaluation::ParametersTag>>::type* = nullptr>
+    RL_TOOLS_FUNCTION_PLACEMENT void log(DEVICE& device, PARAMETERS){}
+    template <typename DEVICE, typename CONFIG, typename utils::typing::enable_if<utils::typing::is_same_v<typename CONFIG::TAG, rl::loop::steps::evaluation::ConfigTag>>::type* = nullptr>
+    RL_TOOLS_FUNCTION_PLACEMENT void log(DEVICE& device, CONFIG){
+        log(device, typename CONFIG::NEXT{});
+    }
+}
+RL_TOOLS_NAMESPACE_WRAPPER_END
+
+
+#endif

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/loop/steps/evaluation/persist.h

@@ -0,0 +1,40 @@
+#include "../../../../version.h"
+#if (defined(RL_TOOLS_DISABLE_INCLUDE_GUARDS) || !defined(RL_TOOLS_RL_LOOP_STEPS_EVALUATION_PERSIST_H)) && (RL_TOOLS_USE_THIS_VERSION == 1)
+#pragma once
+#define RL_TOOLS_RL_LOOP_STEPS_EVALUATION_PERSIST_H
+#include "state.h"
+#include "../../../../random/persist.h"
+RL_TOOLS_NAMESPACE_WRAPPER_START
+namespace rl_tools{
+    template <typename DEVICE, typename T_CONFIG, typename GROUP>
+    void save(DEVICE& device, rl::loop::steps::evaluation::State<T_CONFIG>& ts, GROUP& group){
+        using STATE = rl::loop::steps::evaluation::State<T_CONFIG>;
+        save(device, static_cast<typename STATE::NEXT&>(ts), group);
+        auto evaluation_group = create_group(device, group, "evaluation");
+        auto rng_eval_group = create_group(device, evaluation_group, "rng_eval");
+        save(device, ts.rng_eval, rng_eval_group);
+        auto rng_eval_on_demand_group = create_group(device, evaluation_group, "rng_eval_on_demand");
+        save(device, ts.rng_eval_on_demand, rng_eval_on_demand_group);
+        save_binary(device, &ts.env_eval_parameters, 1, evaluation_group, "env_eval_parameters");
+    }
+    template <typename DEVICE, typename T_CONFIG, typename GROUP>
+    bool load(DEVICE& device, rl::loop::steps::evaluation::State<T_CONFIG>& ts, GROUP& group){
+        using STATE = rl::loop::steps::evaluation::State<T_CONFIG>;
+        bool success = load(device, static_cast<typename STATE::NEXT&>(ts), group);
+        auto evaluation_group = get_group(device, group, "evaluation");
+        auto rng_eval_group = get_group(device, evaluation_group, "rng_eval");
+        bool step_result = load(device, ts.rng_eval, rng_eval_group);
+        if(!step_result){ log(device, device.logger, "Evaluation loop load failed: rng_eval"); }
+        success &= step_result;
+        auto rng_eval_on_demand_group = get_group(device, evaluation_group, "rng_eval_on_demand");
+        step_result = load(device, ts.rng_eval_on_demand, rng_eval_on_demand_group);
+        if(!step_result){ log(device, device.logger, "Evaluation loop load failed: rng_eval_on_demand"); }
+        success &= step_result;
+        step_result = load_binary(device, &ts.env_eval_parameters, 1, evaluation_group, "env_eval_parameters");
+        if(!step_result){ log(device, device.logger, "Evaluation loop load failed: env_eval_parameters"); }
+        success &= step_result;
+        return success;
+    }
+}
+RL_TOOLS_NAMESPACE_WRAPPER_END
+#endif

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/loop/steps/evaluation/state.h

@@ -0,0 +1,36 @@
+#include "../../../../version.h"
+#if (defined(RL_TOOLS_DISABLE_INCLUDE_GUARDS) || !defined(RL_TOOLS_RL_LOOP_STEPS_EVALUATION_STATE_H)) && (RL_TOOLS_USE_THIS_VERSION == 1)
+#pragma once
+#define RL_TOOLS_RL_LOOP_STEPS_EVALUATION_STATE_H
+
+#include "../../../../rl/utils/evaluation/evaluation.h"
+
+RL_TOOLS_NAMESPACE_WRAPPER_START
+namespace rl_tools::rl::loop::steps::evaluation{
+    template<typename T_CONFIG, typename T_NEXT = typename T_CONFIG::NEXT::template State<typename T_CONFIG::NEXT>>
+    struct State: T_NEXT {
+        using CONFIG = T_CONFIG;
+        using NEXT = T_NEXT;
+        using TYPE_POLICY = typename CONFIG::TYPE_POLICY;
+        using TI = typename CONFIG::TI;
+        rl::utils::evaluation::Buffer<rl::utils::evaluation::BufferSpecification<typename CONFIG::EVALUATION_SPEC, CONFIG::DYNAMIC_ALLOCATION>> evaluation_buffer;
+        using RESULT_TYPE = rl::utils::evaluation::Result<typename CONFIG::EVALUATION_RESULT_SPEC>;
+        Matrix<matrix::Specification<RESULT_TYPE, TI, 1, CONFIG::EVALUATION_PARAMETERS::N_EVALUATIONS, NEXT::CONFIG::DYNAMIC_ALLOCATION>> evaluation_results;
+        typename CONFIG::RNG rng_eval, rng_eval_on_demand;
+        Mode<typename CONFIG::EVALUATION_PARAMETERS::EVALUATION_MODE> evaluation_mode;
+        using EVALUATION_ACTOR_TYPE_BATCH_SIZE = typename CONFIG::NN::ACTOR_TYPE::template CHANGE_BATCH_SIZE<TI, CONFIG::EVALUATION_PARAMETERS::NUM_EVALUATION_EPISODES>;
+        using EVALUATION_ACTOR_TYPE = typename EVALUATION_ACTOR_TYPE_BATCH_SIZE::template CHANGE_CAPABILITY<nn::capability::Forward<NEXT::CONFIG::DYNAMIC_ALLOCATION>>;
+        typename EVALUATION_ACTOR_TYPE::template State<CONFIG::DYNAMIC_ALLOCATION> actor_deterministic_evaluation_state;
+        typename EVALUATION_ACTOR_TYPE::template Buffer<NEXT::CONFIG::DYNAMIC_ALLOCATION> actor_deterministic_evaluation_buffers;
+        typename NEXT::CONFIG::ENVIRONMENT_EVALUATION env_eval;
+        typename NEXT::CONFIG::ENVIRONMENT_EVALUATION::Parameters env_eval_parameters;
+        typename CONFIG::UI ui;
+        bool evaluate_this_step = false;
+    };
+}
+RL_TOOLS_NAMESPACE_WRAPPER_END
+#endif
+
+
+
+