hyperdrone 0.1.0__py3-none-any.whl

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/environments/flag/operations_cpu.h

@@ -0,0 +1,212 @@
+#include "../../../version.h"
+#if (defined(RL_TOOLS_DISABLE_INCLUDE_GUARDS) || !defined(RL_TOOLS_RL_ENVIRONMENTS_FLAG_OPERATIONS_CPU_H)) && (RL_TOOLS_USE_THIS_VERSION == 1)
+#pragma once
+#define RL_TOOLS_RL_ENVIRONMENTS_FLAG_OPERATIONS_CPU_H
+
+#include "environment.h"
+#include "operations_generic.h"
+
+#include <string>
+
+RL_TOOLS_NAMESPACE_WRAPPER_START
+namespace rl_tools{
+    template <typename DEVICE, typename SPEC>
+    std::string json(DEVICE&, rl::environments::Flag<SPEC>& env){
+        std::string json = "{";
+        json += "}";
+        return json;
+    }
+    template <typename DEVICE, typename SPEC>
+    std::string json(DEVICE&, rl::environments::Flag<SPEC>& env, typename rl::environments::Flag<SPEC>::Parameters& parameters){
+        using PARAMS = typename rl::environments::Flag<SPEC>::Parameters;
+        std::string json = "{";
+        json += "\"BOARD_SIZE\":" + std::to_string(PARAMS::BOARD_SIZE) + ",";
+        json += "\"DT\":" + std::to_string(PARAMS::DT) + ",";
+        json += "\"MAX_ACCELERATION\":" + std::to_string(PARAMS::MAX_ACCELERATION) + ",";
+        json += "\"MAX_VELOCITY\":" + std::to_string(PARAMS::MAX_VELOCITY) + ",";
+        json += "\"FLAG_DISTANCE_THRESHOLD\":" + std::to_string(PARAMS::FLAG_DISTANCE_THRESHOLD) + ",";
+        json += "\"flag_positions\":[";
+        json += "[" + std::to_string(parameters.flag_positions[0][0]) + "," + std::to_string(parameters.flag_positions[0][1]) + "],";
+        json += "[" + std::to_string(parameters.flag_positions[1][0]) + "," + std::to_string(parameters.flag_positions[1][1]) + "]";
+        json += "]";
+        json += "}";
+        return json;
+    }
+    template <typename DEVICE, typename SPEC>
+    std::string json(DEVICE&, rl::environments::Flag<SPEC>& env, typename rl::environments::Flag<SPEC>::Parameters& parameters, typename rl::environments::Flag<SPEC>::State& state){
+        using TI = typename SPEC::TI;
+        std::string json = "{";
+        json += "\"position\":[" + std::to_string(state.position[0]) + "," + std::to_string(state.position[1]) + "],";
+        json += "\"velocity\":[" + std::to_string(state.velocity[0]) + "," + std::to_string(state.velocity[1]) + "],";
+        json += "\"state_machine\":" + std::to_string(static_cast<TI>(state.state_machine)) + ",";
+        json += "\"step\":" + std::to_string(state.step);
+        json += "}";
+        return json;
+    }
+
+    template <typename DEVICE, typename SPEC>
+    std::string get_ui(DEVICE& device, rl::environments::Flag<SPEC>& env){
+        // Implement the functions `export async function render(ui_state, parameters, state, action)` and `export async function init(canvas, parameters, options)` and `export` them so that they are available as ES6 imports
+        // Please have a look at https://studio.rl.tools which helps you create render functions interactively
+        std::string ui = R"RL_TOOLS_LITERAL(
+
+
+export async function init(canvas, options) {
+    return {
+        ctx: canvas.getContext('2d'),
+        trace: []
+    }
+}
+
+export async function episode_init(ui_state, parameters) {
+    // Clear previous traces
+    ui_state.trace = [];
+}
+
+export async function render(ui_state, parameters, state, action) {
+    const ctx = ui_state.ctx;
+    ctx.clearRect(0, 0, ctx.canvas.width, ctx.canvas.height);
+
+    const canvasWidth = ctx.canvas.width;
+    const scale = canvasWidth / parameters.BOARD_SIZE;
+
+    const agentX = state.position[0] * scale;
+    const agentY = state.position[1] * scale;
+
+    // Record trace
+    ui_state.trace.push({
+        x: agentX,
+        y: agentY,
+        sm: state.state_machine
+    });
+
+//    // Draw board
+//    ctx.strokeStyle = 'black';
+//    ctx.lineWidth = 2;
+//    ctx.strokeRect(0, 0, parameters.BOARD_SIZE * scale, parameters.BOARD_SIZE * scale);
+
+
+    const colors = ['red', 'blue'];
+    const transparentColors = ['rgba(255, 0, 0, 0.1)', 'rgba(0, 0, 255, 0.1)'];
+
+    parameters.flag_positions.forEach(([x, y], index) => {
+      const color = colors[index] || 'red';
+      const transparentColor = transparentColors[index] || 'rgba(255, 0, 0, 0.1)';
+
+      const flagX = x * scale;
+      const flagY = y * scale;
+
+      // Draw the flag (small circle)
+      ctx.beginPath();
+      ctx.arc(flagX, flagY, 5, 0, 2 * Math.PI);
+      ctx.fillStyle = color;
+      ctx.fill();
+      ctx.stroke();
+
+      ctx.beginPath();
+      ctx.arc(flagX, flagY, parameters.FLAG_DISTANCE_THRESHOLD * scale, 0, 2 * Math.PI);
+      ctx.fillStyle = transparentColor;
+      ctx.fill();
+    });
+
+    if(!("step" in state)){
+        // Draw origin
+        ctx.beginPath();
+        ctx.arc(0, 0, 5, 0, 2 * Math.PI);
+        ctx.fillStyle = 'black';
+        ctx.fill();
+        ctx.stroke();
+
+        // Transparent area around origin
+        ctx.beginPath();
+        ctx.arc(0, 0, parameters.FLAG_DISTANCE_THRESHOLD * scale, 0, 2 * Math.PI);
+        ctx.fillStyle = 'rgba(0, 0, 0, 0.1)';
+        ctx.fill();
+    }
+
+
+    // Determine current target and highlight it with the threshold radius
+    let targetX, targetY;
+    if (state.state_machine === 0) {
+        // INITIAL: target is flag
+        targetX = parameters.flag_positions[0][0] * scale;
+        targetY = parameters.flag_positions[0][1] * scale;
+    } else {
+        targetX = parameters.flag_positions[1][0] * scale;
+        targetY = parameters.flag_positions[1][1] * scale;
+    }
+    ctx.beginPath();
+    ctx.arc(targetX, targetY, parameters.FLAG_DISTANCE_THRESHOLD * scale, 0, 2 * Math.PI);
+    ctx.strokeStyle = '#c93c52';
+    ctx.lineWidth = 5;
+    ctx.stroke();
+
+    // Draw trace with different colors depending on state machine
+    for (let i = 1; i < ui_state.trace.length; i++) {
+        const prev = ui_state.trace[i-1];
+        const curr = ui_state.trace[i];
+
+        let color = 'gray';
+        if (prev.sm === 1) color = 'purple';
+        if (prev.sm === 2) color = 'orange';
+
+        ctx.beginPath();
+        ctx.moveTo(prev.x, prev.y);
+        ctx.lineTo(curr.x, curr.y);
+        ctx.strokeStyle = color;
+        ctx.lineWidth = 5;
+        ctx.stroke();
+    }
+
+    // Draw agent
+    ctx.beginPath();
+    ctx.arc(agentX, agentY, 35, 0, 2 * Math.PI);
+    ctx.fillStyle = '#7DB9B6';
+    ctx.fill();
+
+//    // Draw velocity vector
+//    const velX = state.velocity[0] * scale * 0.5;
+//    const velY = state.velocity[1] * scale * 0.5;
+//    ctx.beginPath();
+//    ctx.moveTo(agentX, agentY);
+//    ctx.lineTo(agentX + velX, agentY + velY);
+//    ctx.strokeStyle = 'blue';
+//    ctx.lineWidth = 5;
+//    ctx.stroke();
+//
+//    // Draw action vector (scaled by MAX_ACCELERATION)
+//    const accX = action[0] * parameters.MAX_ACCELERATION/(parameters.BOARD_SIZE*3) * scale * 0.5;
+//    const accY = action[1] * parameters.MAX_ACCELERATION/(parameters.BOARD_SIZE*3) * scale * 0.5;
+//    ctx.beginPath();
+//    ctx.moveTo(agentX, agentY);
+//    ctx.lineTo(agentX + accX, agentY + accY);
+//    ctx.strokeStyle = 'green';
+//    ctx.lineWidth = 5;
+//    ctx.stroke();
+
+}
+
+
+        )RL_TOOLS_LITERAL";
+        return ui;
+    }
+
+    template <typename DEVICE, typename SPEC>
+    std::string get_description(DEVICE& device, rl::environments::Flag<SPEC>& env){
+        using ENVIRONMENT = rl::environments::Flag<SPEC>;
+        std::string description;
+        description += "In this environment the agent is randomly placed on a 2D board of size " + std::to_string(SPEC::PARAMETERS::BOARD_SIZE) + "x" + std::to_string(SPEC::PARAMETERS::BOARD_SIZE) + ". ";
+        description += "The agent can move in 2D space with a maximum acceleration of " + std::to_string(SPEC::PARAMETERS::MAX_ACCELERATION) + " and a maximum velocity of " + std::to_string(SPEC::PARAMETERS::MAX_VELOCITY) + " (timestep DT=" + std::to_string(SPEC::PARAMETERS::DT) + "). ";
+        description += "Furthermore at the beginning of each episode two flags are sampled uniformly at random on the board. ";
+        description += "On the first step (only) the agent is revealed the positions of the flags. ";
+        description += "The goal of the agent is to visit the flags in order (within a distance of " + std::to_string(SPEC::PARAMETERS::FLAG_DISTANCE_THRESHOLD) + "). ";
+        description += "The agent receives a reward of " + std::to_string(SPEC::PARAMETERS::REWARD_SCALE) + " for visiting each flag and a penalty of -100 for going out of bounds. ";
+        description += "The episode ends when the agent visits the second flag or goes out of bounds (max " + std::to_string(ENVIRONMENT::EPISODE_STEP_LIMIT) + " steps). ";
+        description += "To achieve this, the agent must memorize the positions of the flags that are only revealed in the first step. We've added a second flag, because in the case of only one flag, theoretically, the agent could \"cheat\" by accelerating fast in the direction of the flag at the first state, and then maintaining that velocity. ";
+        description += "With the second flag, there is no way the agent can encode this information into the state of the environment, and hence it must memorize it internally.";
+        return description;
+    }
+}
+RL_TOOLS_NAMESPACE_WRAPPER_END
+
+#endif

hyperdrone/_vendor/rl-tools/include/rl_tools/rl/environments/flag/operations_generic.h

@@ -0,0 +1,215 @@
+#include "../../../version.h"
+#if (defined(RL_TOOLS_DISABLE_INCLUDE_GUARDS) || !defined(RL_TOOLS_RL_ENVIRONMENTS_FLAG_OPERATIONS_GENERIC_H)) && (RL_TOOLS_USE_THIS_VERSION == 1)
+#pragma once
+#define RL_TOOLS_RL_ENVIRONMENTS_FLAG_OPERATIONS_GENERIC_H
+#include "environment.h"
+#include "../operations_generic.h"
+RL_TOOLS_NAMESPACE_WRAPPER_START
+namespace rl_tools{
+    template<typename DEVICE, typename SPEC>
+    RL_TOOLS_FUNCTION_PLACEMENT static void malloc(DEVICE& device, const rl::environments::Flag<SPEC>& env){ }
+    template<typename DEVICE, typename SPEC>
+    RL_TOOLS_FUNCTION_PLACEMENT static void free(DEVICE& device, const rl::environments::Flag<SPEC>& env){ }
+    template<typename DEVICE, typename SPEC>
+    RL_TOOLS_FUNCTION_PLACEMENT static void init(DEVICE& device, const rl::environments::Flag<SPEC>& env){ }
+    template<typename DEVICE, typename SPEC, typename RNG>
+    RL_TOOLS_FUNCTION_PLACEMENT static void sample_initial_parameters(DEVICE& device, const rl::environments::Flag<SPEC>& env, typename rl::environments::Flag<SPEC>::Parameters& parameters, RNG& rng){
+        using T = typename SPEC::T;
+        parameters.flag_positions[0][0] = random::uniform_real_distribution(device.random, (T)0, SPEC::PARAMETERS::BOARD_SIZE, rng);
+        parameters.flag_positions[0][1] = random::uniform_real_distribution(device.random, (T)0, SPEC::PARAMETERS::BOARD_SIZE, rng);
+        parameters.flag_positions[1][0] = random::uniform_real_distribution(device.random, (T)0, SPEC::PARAMETERS::BOARD_SIZE, rng);
+        parameters.flag_positions[1][1] = random::uniform_real_distribution(device.random, (T)0, SPEC::PARAMETERS::BOARD_SIZE, rng);
+    }
+    template<typename DEVICE, typename SPEC>
+    RL_TOOLS_FUNCTION_PLACEMENT static void initial_parameters(DEVICE& device, const rl::environments::Flag<SPEC>& env, typename rl::environments::Flag<SPEC>::Parameters& parameters){
+        parameters.flag_positions[0][0] = SPEC::PARAMETERS::BOARD_SIZE * 1.0 / 4;
+        parameters.flag_positions[0][1] = SPEC::PARAMETERS::BOARD_SIZE * 1.0 / 4;
+        parameters.flag_positions[1][0] = SPEC::PARAMETERS::BOARD_SIZE * 3.0 / 4;
+        parameters.flag_positions[1][1] = SPEC::PARAMETERS::BOARD_SIZE * 3.0 / 4;
+    }
+    template<typename DEVICE, typename SPEC>
+    RL_TOOLS_FUNCTION_PLACEMENT static void initial_state(DEVICE& device, const rl::environments::Flag<SPEC>& env, typename rl::environments::Flag<SPEC>::Parameters& parameters, typename rl::environments::Flag<SPEC>::State& state){
+        state.position[0] = SPEC::PARAMETERS::BOARD_SIZE / 2;
+        state.position[1] = SPEC::PARAMETERS::BOARD_SIZE / 2;
+        state.velocity[0] = 0;
+        state.velocity[1] = 0;
+        state.state_machine = rl::environments::Flag<SPEC>::State::StateMachine::INITIAL;
+        state.step = 0;
+    }
+    template<typename DEVICE, typename SPEC, typename RNG>
+    RL_TOOLS_FUNCTION_PLACEMENT static void sample_initial_state(DEVICE& device, const rl::environments::Flag<SPEC>& env, typename rl::environments::Flag<SPEC>::Parameters& parameters, typename rl::environments::Flag<SPEC>::State& state, RNG& rng){
+        using T = typename SPEC::T;
+        using PARAMETERS = typename rl::environments::Flag<SPEC>::Parameters;
+        state.position[0] = random::uniform_real_distribution(device.random, (T)0, SPEC::PARAMETERS::BOARD_SIZE, rng);
+        state.position[1] = random::uniform_real_distribution(device.random, (T)0, SPEC::PARAMETERS::BOARD_SIZE, rng);
+        state.velocity[0] = 0;
+        state.velocity[1] = 0;
+        if constexpr(PARAMETERS::SAMPLE_INITIAL_STATE_WITH_FLAG_1_VISITED){
+            if(random::uniform_int_distribution(device.random, 0, 1, rng) == 0){
+                state.state_machine = rl::environments::Flag<SPEC>::State::StateMachine::INITIAL;
+            }
+            else{
+                state.state_machine = rl::environments::Flag<SPEC>::State::StateMachine::FLAG_1_VISITED;
+            }
+        }
+        else{
+            state.state_machine = rl::environments::Flag<SPEC>::State::StateMachine::INITIAL;
+        }
+        state.step = 0;
+    }
+    template<typename DEVICE, typename SPEC, typename ACTION_SPEC, typename RNG>
+    RL_TOOLS_FUNCTION_PLACEMENT typename SPEC::T step(DEVICE& device, const rl::environments::Flag<SPEC>& env, typename rl::environments::Flag<SPEC>::Parameters& parameters, const typename rl::environments::Flag<SPEC>::State& state, const Matrix<ACTION_SPEC>& action, typename rl::environments::Flag<SPEC>::State& next_state, RNG& rng) {
+        static_assert(ACTION_SPEC::ROWS == 1);
+        static_assert(ACTION_SPEC::COLS == 2);
+        using namespace rl::environments::pendulum;
+        using T = typename SPEC::T;
+        using PARAMS = typename SPEC::PARAMETERS;
+        using STATE = typename rl::environments::Flag<SPEC>::State;
+        T u_normalised[2];
+        u_normalised[0] = math::clamp(device.math, get(action, 0, 0), (T)-1, (T)1);
+        u_normalised[1] = math::clamp(device.math, get(action, 0, 1), (T)-1, (T)1);
+        T u[2];
+        u[0] = PARAMS::MAX_ACCELERATION * u_normalised[0];
+        u[1] = PARAMS::MAX_ACCELERATION * u_normalised[1];
+        next_state.position[0] = state.position[0] + state.velocity[0] * PARAMS::DT;
+        next_state.position[1] = state.position[1] + state.velocity[1] * PARAMS::DT;
+        next_state.velocity[0] = state.velocity[0] + u[0] * PARAMS::DT;
+        next_state.velocity[1] = state.velocity[1] + u[1] * PARAMS::DT;
+        next_state.position[0] = math::clamp(device.math, next_state.position[0], (T)0, PARAMS::BOARD_SIZE);
+        next_state.position[1] = math::clamp(device.math, next_state.position[1], (T)0, PARAMS::BOARD_SIZE);
+        next_state.velocity[0] = math::clamp(device.math, next_state.velocity[0], (T)-PARAMS::MAX_VELOCITY, PARAMS::MAX_VELOCITY);
+        next_state.velocity[1] = math::clamp(device.math, next_state.velocity[1], (T)-PARAMS::MAX_VELOCITY, PARAMS::MAX_VELOCITY);
+
+        T distance_to_flag_1 = math::sqrt(device.math, (next_state.position[0] - parameters.flag_positions[0][0]) * (next_state.position[0] - parameters.flag_positions[0][0]) + (next_state.position[1] - parameters.flag_positions[0][1]) * (next_state.position[1] - parameters.flag_positions[0][1]));
+        T distance_to_flag_2 = math::sqrt(device.math, (next_state.position[0] - parameters.flag_positions[1][0]) * (next_state.position[0] - parameters.flag_positions[1][0]) + (next_state.position[1] - parameters.flag_positions[1][1]) * (next_state.position[1] - parameters.flag_positions[1][1]));
+        switch(state.state_machine){
+            case STATE::StateMachine::INITIAL:
+                if(distance_to_flag_1 < PARAMS::FLAG_DISTANCE_THRESHOLD){
+                    next_state.state_machine = STATE::StateMachine::FLAG_1_VISITED;
+                }
+                else{
+                    next_state.state_machine = STATE::StateMachine::INITIAL;
+                }
+                break;
+            case STATE::StateMachine::FLAG_1_VISITED:
+                if(distance_to_flag_2 < PARAMS::FLAG_DISTANCE_THRESHOLD){
+                    next_state.state_machine = STATE::StateMachine::FLAG_2_VISITED;
+                }
+                else{
+                    next_state.state_machine = STATE::StateMachine::FLAG_1_VISITED;
+                }
+                break;
+            default:
+                std::cout << "unexpected state: " << static_cast<typename DEVICE::index_t>(state.state_machine) << std::endl;
+                std::exit(1);
+                break;
+        }
+        next_state.step = state.step + 1;
+        return SPEC::PARAMETERS::DT;
+    }
+    template<typename DEVICE, typename SPEC, typename ACTION_SPEC, typename RNG>
+    RL_TOOLS_FUNCTION_PLACEMENT static typename SPEC::T reward(DEVICE& device, const rl::environments::Flag<SPEC>& env, typename rl::environments::Flag<SPEC>::Parameters& parameters, const typename rl::environments::Flag<SPEC>::State& state, const Matrix<ACTION_SPEC>& action, const typename rl::environments::Flag<SPEC>::State& next_state, RNG& rng){
+        using T = typename SPEC::T;
+        using ENVIRONMENT = rl::environments::Flag<SPEC>;
+        using PARAMETERS = typename ENVIRONMENT::Parameters;
+        using STATE = typename rl::environments::Flag<SPEC>::State;
+        T reward = 0;
+        bool visited_flag_1 = state.state_machine != STATE::StateMachine::FLAG_1_VISITED && next_state.state_machine == STATE::StateMachine::FLAG_1_VISITED;
+        bool visited_flag_2 = state.state_machine != STATE::StateMachine::FLAG_2_VISITED && next_state.state_machine == STATE::StateMachine::FLAG_2_VISITED;
+        if(visited_flag_1 || visited_flag_2){
+            reward = 1.0 * PARAMETERS::REWARD_SCALE;
+        }
+        else{
+            static_assert(ENVIRONMENT::EPISODE_STEP_LIMIT > 2);
+            // reward = -1.0/(ENVIRONMENT::EPISODE_STEP_LIMIT - 2);
+        }
+        bool out_of_boundary = next_state.position[0] == 0 || next_state.position[0] == PARAMETERS::BOARD_SIZE;
+        out_of_boundary = out_of_boundary || next_state.position[1] == 0 || next_state.position[1] == PARAMETERS::BOARD_SIZE;
+        if (out_of_boundary){
+            reward -= 100;
+        }
+        return reward;
+    }
+
+    template<typename DEVICE, typename SPEC, typename OBS_TYPE_SPEC, typename OBS_SPEC, typename RNG>
+    RL_TOOLS_FUNCTION_PLACEMENT static void observe(DEVICE& device, const rl::environments::Flag<SPEC>& env, const typename rl::environments::Flag<SPEC>::Parameters& parameters, const typename rl::environments::Flag<SPEC>::State& state, const typename rl::environments::flag::Observation<OBS_TYPE_SPEC>&, Matrix<OBS_SPEC>& observation, RNG& rng){
+        static_assert(OBS_SPEC::ROWS == 1);
+        static_assert(OBS_SPEC::COLS == 11);
+        using T = typename SPEC::T;
+        using STATE_MACHINE = typename rl::environments::Flag<SPEC>::State::StateMachine;
+        set(observation, 0, 0, state.position[0]);
+        set(observation, 0, 1, state.position[1]);
+        set(observation, 0, 2, state.velocity[0]);
+        set(observation, 0, 3, state.velocity[1]);
+        switch(state.state_machine){
+            case STATE_MACHINE::INITIAL:
+                set(observation, 0, 4, 1);
+                set(observation, 0, 5, -1);
+                set(observation, 0, 6, -1);
+                break;
+            case STATE_MACHINE::FLAG_1_VISITED:
+                set(observation, 0, 4, -1);
+                set(observation, 0, 5, 1);
+                set(observation, 0, 6, -1);
+                break;
+            case STATE_MACHINE::FLAG_2_VISITED:
+                set(observation, 0, 4, -1);
+                set(observation, 0, 5, -1);
+                set(observation, 0, 6, 1);
+                break;
+        }
+        if(state.step <= 0){
+            set(observation, 0,  7, parameters.flag_positions[0][0]);
+            set(observation, 0,  8, parameters.flag_positions[0][1]);
+            set(observation, 0,  9, parameters.flag_positions[1][0]);
+            set(observation, 0, 10, parameters.flag_positions[1][1]);
+        }
+        else{
+            set(observation, 0,  7, -1);
+            set(observation, 0,  8, -1);
+            set(observation, 0,  9, -1);
+            set(observation, 0, 10, -1);
+        }
+    }
+
+    template<typename DEVICE, typename SPEC, typename OBS_TYPE_SPEC, typename OBS_SPEC, typename RNG>
+    RL_TOOLS_FUNCTION_PLACEMENT static void observe(DEVICE& device, const rl::environments::Flag<SPEC>& env, const typename rl::environments::Flag<SPEC>::Parameters& parameters, const typename rl::environments::Flag<SPEC>::State& state, const typename rl::environments::flag::ObservationPrivileged<OBS_TYPE_SPEC>&, Matrix<OBS_SPEC>& observation, RNG& rng) {
+        static_assert(OBS_SPEC::ROWS == 1);
+        static_assert(OBS_SPEC::COLS == 11);
+        using T = typename SPEC::T;
+        observe(device, env, parameters, state, typename rl::environments::flag::Observation<OBS_TYPE_SPEC>{}, observation, rng);
+        set(observation, 0,  7, parameters.flag_positions[0][0]);
+        set(observation, 0,  8, parameters.flag_positions[0][1]);
+        set(observation, 0,  9, parameters.flag_positions[1][0]);
+        set(observation, 0, 10, parameters.flag_positions[1][1]);
+    }
+    template<typename DEVICE, typename SPEC, typename RNG>
+    RL_TOOLS_FUNCTION_PLACEMENT static bool terminated(DEVICE& device, const rl::environments::Flag<SPEC>& env, typename rl::environments::Flag<SPEC>::Parameters& parameters, const typename rl::environments::Flag<SPEC>::State state, RNG& rng){
+        using PARAMS = typename SPEC::PARAMETERS;
+        bool out_of_boundary = state.position[0] == 0 || state.position[0] == PARAMS::BOARD_SIZE;
+        out_of_boundary = out_of_boundary || state.position[1] == 0 || state.position[1] == PARAMS::BOARD_SIZE;
+        return state.state_machine == rl::environments::Flag<SPEC>::State::StateMachine::FLAG_2_VISITED || out_of_boundary;
+    }
+    template<typename DEVICE, typename T, typename TI>
+    RL_TOOLS_FUNCTION_PLACEMENT static T abs_diff(DEVICE& device, const rl::environments::flag::State<T, TI>& s1, const rl::environments::flag::State<T, TI>& s2){
+        T diff = 0;
+        diff += math::abs(device.math, s1.position[0] - s2.position[0]);
+        diff += math::abs(device.math, s1.position[1] - s2.position[1]);
+        diff += math::abs(device.math, s1.velocity[0] - s2.velocity[0]);
+        diff += math::abs(device.math, s1.velocity[1] - s2.velocity[1]);
+        diff += (s1.state_machine != s2.state_machine) ? 1 : 0;
+        diff += math::abs(device.math, (T)s1.step - (T)s2.step);
+        return diff;
+    }
+    template<typename DEVICE, typename T, typename TI, TI T_MAX_EPISODE_LENGTH, bool T_ACTOR_PRIVILEGED_OBSERVATION, bool T_CRITIC_PRIVILEGED_OBSERVATION>
+    RL_TOOLS_FUNCTION_PLACEMENT static T abs_diff(DEVICE& device, const rl::environments::flag::DefaultParameters<T, TI, T_MAX_EPISODE_LENGTH, T_ACTOR_PRIVILEGED_OBSERVATION, T_CRITIC_PRIVILEGED_OBSERVATION>& p1, const rl::environments::flag::DefaultParameters<T, TI, T_MAX_EPISODE_LENGTH, T_ACTOR_PRIVILEGED_OBSERVATION, T_CRITIC_PRIVILEGED_OBSERVATION>& p2){
+        T diff = 0;
+        diff += math::abs(device.math, p1.flag_positions[0][0] - p2.flag_positions[0][0]);
+        diff += math::abs(device.math, p1.flag_positions[0][1] - p2.flag_positions[0][1]);
+        diff += math::abs(device.math, p1.flag_positions[1][0] - p2.flag_positions[1][0]);
+        diff += math::abs(device.math, p1.flag_positions[1][1] - p2.flag_positions[1][1]);
+        return diff;
+    }
+}
+RL_TOOLS_NAMESPACE_WRAPPER_END
+#endif