rlmodule 0.0.5__py3-none-any.whl

rlmodule/__init__.py

@@ -0,0 +1,3 @@
+# TODO this is not very good way
+from .source.rlmodel import RLModel, SharedRLModel, GaussianLayer, DeterministicLayer # noqa: F401, F403
+from .source.modules import MLP # noqa: F401, F403

rlmodule/skrl/examples/gymnasium/skrl_gymnasium_pendulum.py

@@ -0,0 +1,120 @@
+import gymnasium as gym
+
+import torch
+import torch.nn as nn
+
+from rlmodule import SharedRLModel, GaussianLayer, DeterministicLayer, MLP
+
+# import the skrl components to build the RL system
+from skrl.agents.torch.ppo import PPO, PPO_DEFAULT_CONFIG
+
+from skrl.envs.wrappers.torch.gymnasium_envs import GymnasiumWrapper
+#from skrl.envs.wrappers.torch import wrap_env
+from skrl.memories.torch import RandomMemory
+from skrl.resources.preprocessors.torch import RunningStandardScaler
+from skrl.resources.schedulers.torch import KLAdaptiveRL
+from skrl.trainers.torch import SequentialTrainer
+from skrl.utils import set_seed
+from skrl.envs.wrappers.torch.gymnasium_envs import GymnasiumWrapper
+
+# todo set seed
+# seed for reproducibility
+set_seed()  # e.g. `set_seed(42)` for fixed seed
+
+def get_shared_model(env):
+    # instantiate the agent's models (function approximators).
+    params = {'input_size': env.observation_space.shape[0], 'hidden_units': [64, 64, 64], 'activations': ['relu', 'relu', 'relu']}
+    net = MLP(params)
+    
+    model = SharedRLModel(
+        observation_space=env.observation_space.shape[0], # TODO not do shape[0] but process box normally
+        action_space=env.observation_space.shape[0],
+        device=device,
+        network=net,
+        output_layer={
+            'policy': GaussianLayer(
+                input_size=64,  # TODO
+                output_size=env.action_space.shape[0],
+                device=device,
+                output_activation=nn.Tanh(),
+                clip_actions=False,
+                clip_log_std=True,
+                min_log_std=-1.2,
+                max_log_std=2,
+                initial_log_std=0.2,
+            ),
+            'value': DeterministicLayer(
+                input_size=64,
+                output_size=1,
+                device=device,
+                output_activation=nn.Identity(),
+            ),
+        },
+    ).to(device) # TODO careful about device
+
+    return {'policy': model, 'value': model}
+
+
+# load and wrap the gymnasium environment.
+# note: the environment version may change depending on the gymnasium version
+try:
+    env = gym.vector.make("Pendulum-v1", num_envs=4, asynchronous=False)
+except (gym.error.DeprecatedEnv, gym.error.VersionNotFound) as e:
+    env_id = [spec for spec in gym.envs.registry if spec.startswith("Pendulum-v")][0]
+    print("Pendulum-v1 not found. Trying {}".format(env_id))
+    env = gym.vector.make(env_id, num_envs=4, asynchronous=False)
+
+env = GymnasiumWrapper(env)
+
+device = env.device
+
+
+# instantiate a memory as rollout buffer (any memory can be used for this)
+memory = RandomMemory(memory_size=1024, num_envs=env.num_envs, device=device)
+
+models = get_shared_model(env)
+
+print(models)
+
+
+# configure and instantiate the agent (visit its documentation to see all the options)
+# https://skrl.readthedocs.io/en/latest/api/agents/ppo.html#configuration-and-hyperparameters
+cfg = PPO_DEFAULT_CONFIG.copy()
+cfg["rollouts"] = 1024  # memory_size
+cfg["learning_epochs"] = 10
+cfg["mini_batches"] = 32
+cfg["discount_factor"] = 0.9
+cfg["lambda"] = 0.95
+cfg["learning_rate"] = 1e-3
+cfg["learning_rate_scheduler"] = KLAdaptiveRL
+cfg["learning_rate_scheduler_kwargs"] = {"kl_threshold": 0.008}
+cfg["grad_norm_clip"] = 0.5
+cfg["ratio_clip"] = 0.2
+cfg["value_clip"] = 0.2
+cfg["clip_predicted_values"] = False
+cfg["entropy_loss_scale"] = 0.0
+cfg["value_loss_scale"] = 0.5
+cfg["kl_threshold"] = 0
+cfg["state_preprocessor"] = RunningStandardScaler
+cfg["state_preprocessor_kwargs"] = {"size": env.observation_space, "device": device}
+cfg["value_preprocessor"] = RunningStandardScaler
+cfg["value_preprocessor_kwargs"] = {"size": 1, "device": device}
+# logging to TensorBoard and write checkpoints (in timesteps)
+cfg["experiment"]["write_interval"] = 500
+cfg["experiment"]["checkpoint_interval"] = 5000
+cfg["experiment"]["directory"] = "runs/torch/Pendulum"
+
+agent = PPO(models=models,
+            memory=memory,
+            cfg=cfg,
+            observation_space=env.observation_space,
+            action_space=env.action_space,
+            device=device)
+
+
+# configure and instantiate the RL trainer
+cfg_trainer = {"timesteps": 100000, "headless": True}
+trainer = SequentialTrainer(cfg=cfg_trainer, env=env, agents=[agent])
+
+# start training
+trainer.train()

rlmodule/skrl/examples/gymnasium/skrl_gymnasium_pendulum_no_module.py

@@ -0,0 +1,123 @@
+import gymnasium as gym
+
+import torch
+import torch.nn as nn
+
+# import the skrl components to build the RL system
+from skrl.agents.torch.ppo import PPO, PPO_DEFAULT_CONFIG
+
+from skrl.envs.wrappers.torch.gymnasium_envs import GymnasiumWrapper
+# from skrl.envs.wrappers.torch import wrap_env
+from skrl.memories.torch import RandomMemory
+from skrl.models.torch import DeterministicMixin, GaussianMixin, Model
+from skrl.resources.preprocessors.torch import RunningStandardScaler
+from skrl.resources.schedulers.torch import KLAdaptiveRL
+from skrl.trainers.torch import SequentialTrainer
+from skrl.utils import set_seed
+
+
+# seed for reproducibility
+set_seed()  # e.g. `set_seed(42)` for fixed seed
+
+
+# define models (stochastic and deterministic models) using mixins
+class Policy(GaussianMixin, Model):
+    def __init__(self, observation_space, action_space, device, clip_actions=False,
+                 clip_log_std=True, min_log_std=-20, max_log_std=2, reduction="sum"):
+        Model.__init__(self, observation_space, action_space, device)
+        GaussianMixin.__init__(self, clip_actions, clip_log_std, min_log_std, max_log_std, reduction)
+
+        self.net = nn.Sequential(nn.Linear(self.num_observations, 64),
+                                 nn.ReLU(),
+                                 nn.Linear(64, 64),
+                                 nn.ReLU(),
+                                 nn.Linear(64, self.num_actions))
+        self.log_std_parameter = nn.Parameter(torch.zeros(self.num_actions))
+
+    def compute(self, inputs, role):
+        # Pendulum-v1 action_space is -2 to 2
+        return 2 * torch.tanh(self.net(inputs["states"])), self.log_std_parameter, {}
+
+class Value(DeterministicMixin, Model):
+    def __init__(self, observation_space, action_space, device, clip_actions=False):
+        Model.__init__(self, observation_space, action_space, device)
+        DeterministicMixin.__init__(self, clip_actions)
+
+        self.net = nn.Sequential(nn.Linear(self.num_observations, 64),
+                                 nn.ReLU(),
+                                 nn.Linear(64, 64),
+                                 nn.ReLU(),
+                                 nn.Linear(64, 1))
+
+    def compute(self, inputs, role):
+        return self.net(inputs["states"]), {}
+
+
+# load and wrap the gymnasium environment.
+# note: the environment version may change depending on the gymnasium version
+try:
+    env = gym.vector.make("Pendulum-v1", num_envs=4, asynchronous=False)
+except (gym.error.DeprecatedEnv, gym.error.VersionNotFound) as e:
+    env_id = [spec for spec in gym.envs.registry if spec.startswith("Pendulum-v")][0]
+    print("Pendulum-v1 not found. Trying {}".format(env_id))
+    env = gym.vector.make(env_id, num_envs=4, asynchronous=False)
+
+env = GymnasiumWrapper(env)
+
+device = env.device
+
+
+# instantiate a memory as rollout buffer (any memory can be used for this)
+memory = RandomMemory(memory_size=1024, num_envs=env.num_envs, device=device)
+
+
+# instantiate the agent's models (function approximators).
+# PPO requires 2 models, visit its documentation for more details
+# https://skrl.readthedocs.io/en/latest/api/agents/ppo.html#models
+models = {}
+models["policy"] = Policy(env.observation_space, env.action_space, device, clip_actions=True)
+models["value"] = Value(env.observation_space, env.action_space, device)
+
+print(models)
+
+# configure and instantiate the agent (visit its documentation to see all the options)
+# https://skrl.readthedocs.io/en/latest/api/agents/ppo.html#configuration-and-hyperparameters
+cfg = PPO_DEFAULT_CONFIG.copy()
+cfg["rollouts"] = 1024  # memory_size
+cfg["learning_epochs"] = 10
+cfg["mini_batches"] = 32
+cfg["discount_factor"] = 0.9
+cfg["lambda"] = 0.95
+cfg["learning_rate"] = 1e-3
+cfg["learning_rate_scheduler"] = KLAdaptiveRL
+cfg["learning_rate_scheduler_kwargs"] = {"kl_threshold": 0.008}
+cfg["grad_norm_clip"] = 0.5
+cfg["ratio_clip"] = 0.2
+cfg["value_clip"] = 0.2
+cfg["clip_predicted_values"] = False
+cfg["entropy_loss_scale"] = 0.0
+cfg["value_loss_scale"] = 0.5
+cfg["kl_threshold"] = 0
+cfg["state_preprocessor"] = RunningStandardScaler
+cfg["state_preprocessor_kwargs"] = {"size": env.observation_space, "device": device}
+cfg["value_preprocessor"] = RunningStandardScaler
+cfg["value_preprocessor_kwargs"] = {"size": 1, "device": device}
+# logging to TensorBoard and write checkpoints (in timesteps)
+cfg["experiment"]["write_interval"] = 500
+cfg["experiment"]["checkpoint_interval"] = 5000
+cfg["experiment"]["directory"] = "runs/torch/Pendulum"
+
+agent = PPO(models=models,
+            memory=memory,
+            cfg=cfg,
+            observation_space=env.observation_space,
+            action_space=env.action_space,
+            device=device)
+
+
+# configure and instantiate the RL trainer
+cfg_trainer = {"timesteps": 100000, "headless": True}
+trainer = SequentialTrainer(cfg=cfg_trainer, env=env, agents=[agent])
+
+# start training
+trainer.train()

rlmodule/source/deprecated.py

@@ -0,0 +1,378 @@
+from typing import Any, Mapping, Tuple, Union
+
+import gym
+import gymnasium
+
+import torch
+from torch.distributions import Normal
+
+
+class GaussianMixin:
+    def __init__(self,
+                 clip_actions: bool = False,
+                 clip_log_std: bool = True,
+                 min_log_std: float = -20,
+                 max_log_std: float = 2,
+                 reduction: str = "sum",
+                 role: str = "") -> None:
+        """Gaussian mixin model (stochastic model)
+
+        :param clip_actions: Flag to indicate whether the actions should be clipped to the action space (default: ``False``)
+        :type clip_actions: bool, optional
+        :param clip_log_std: Flag to indicate whether the log standard deviations should be clipped (default: ``True``)
+        :type clip_log_std: bool, optional
+        :param min_log_std: Minimum value of the log standard deviation if ``clip_log_std`` is True (default: ``-20``)
+        :type min_log_std: float, optional
+        :param max_log_std: Maximum value of the log standard deviation if ``clip_log_std`` is True (default: ``2``)
+        :type max_log_std: float, optional
+        :param reduction: Reduction method for returning the log probability density function: (default: ``"sum"``).
+                          Supported values are ``"mean"``, ``"sum"``, ``"prod"`` and ``"none"``. If "``none"``, the log probability density
+                          function is returned as a tensor of shape ``(num_samples, num_actions)`` instead of ``(num_samples, 1)``
+        :type reduction: str, optional
+        :param role: Role play by the model (default: ``""``)
+        :type role: str, optional
+
+        :raises ValueError: If the reduction method is not valid
+
+        Example::
+
+            # define the model
+            >>> import torch
+            >>> import torch.nn as nn
+            >>> from skrl.models.torch import Model, GaussianMixin
+            >>>
+            >>> class Policy(GaussianMixin, Model):
+            ...     def __init__(self, observation_space, action_space, device="cuda:0",
+            ...                  clip_actions=False, clip_log_std=True, min_log_std=-20, max_log_std=2, reduction="sum"):
+            ...         Model.__init__(self, observation_space, action_space, device)
+            ...         GaussianMixin.__init__(self, clip_actions, clip_log_std, min_log_std, max_log_std, reduction)
+            ...
+            ...         self.net = nn.Sequential(nn.Linear(self.num_observations, 32),
+            ...                                  nn.ELU(),
+            ...                                  nn.Linear(32, 32),
+            ...                                  nn.ELU(),
+            ...                                  nn.Linear(32, self.num_actions))
+            ...         self.log_std_parameter = nn.Parameter(torch.zeros(self.num_actions))
+            ...
+            ...     def compute(self, inputs, role):
+            ...         return self.net(inputs["states"]), self.log_std_parameter, {}
+            ...
+            >>> # given an observation_space: gym.spaces.Box with shape (60,)
+            >>> # and an action_space: gym.spaces.Box with shape (8,)
+            >>> model = Policy(observation_space, action_space)
+            >>>
+            >>> print(model)
+            Policy(
+              (net): Sequential(
+                (0): Linear(in_features=60, out_features=32, bias=True)
+                (1): ELU(alpha=1.0)
+                (2): Linear(in_features=32, out_features=32, bias=True)
+                (3): ELU(alpha=1.0)
+                (4): Linear(in_features=32, out_features=8, bias=True)
+              )
+            )
+        """
+        self._clip_actions = clip_actions and (issubclass(type(self.action_space), gym.Space) or \
+            issubclass(type(self.action_space), gymnasium.Space))
+
+        if self._clip_actions:
+            # TODO(ll) the code assume it will be inherited ... self.device is not set anywhere.  
+            self._clip_actions_min = torch.tensor(self.action_space.low, device=self.device, dtype=torch.float32)
+            self._clip_actions_max = torch.tensor(self.action_space.high, device=self.device, dtype=torch.float32)
+
+        self._clip_log_std = clip_log_std
+        self._log_std_min = min_log_std
+        self._log_std_max = max_log_std
+
+        self._log_std = None
+        self._num_samples = None
+        self._distribution = None
+
+        if reduction not in ["mean", "sum", "prod", "none"]:
+            raise ValueError("reduction must be one of 'mean', 'sum', 'prod' or 'none'")
+        self._reduction = torch.mean if reduction == "mean" else torch.sum if reduction == "sum" \
+            else torch.prod if reduction == "prod" else None
+
+    # TODO(ll) keep for backward compatibility
+    def act(self,
+            inputs: Mapping[str, Union[torch.Tensor, Any]],
+            role: str = "") -> Tuple[torch.Tensor, Union[torch.Tensor, None], Mapping[str, Union[torch.Tensor, Any]]]:
+        """Act stochastically in response to the state of the environment
+
+        :param inputs: Model inputs. The most common keys are:
+
+                       - ``"states"``: state of the environment used to make the decision
+                       - ``"taken_actions"``: actions taken by the policy for the given states
+        :type inputs: dict where the values are typically torch.Tensor
+        :param role: Role play by the model (default: ``""``)
+        :type role: str, optional
+
+        :return: Model output. The first component is the action to be taken by the agent.
+                 The second component is the log of the probability density function.
+                 The third component is a dictionary containing the mean actions ``"mean_actions"``
+                 and extra output values
+        :rtype: tuple of torch.Tensor, torch.Tensor or None, and dict
+
+        Example::
+
+            >>> # given a batch of sample states with shape (4096, 60)
+            >>> actions, log_prob, outputs = model.act({"states": states})
+            >>> print(actions.shape, log_prob.shape, outputs["mean_actions"].shape)
+            torch.Size([4096, 8]) torch.Size([4096, 1]) torch.Size([4096, 8])
+        """
+        # map from states/observations to mean actions and log standard deviations
+        mean_actions, log_std, outputs = self.compute(inputs, role)
+
+        return self.forward(mean_actions, inputs.get('taken_actions', None), log_std, outputs)
+
+        # # clamp log standard deviations
+        # if self._clip_log_std:
+        #     log_std = torch.clamp(log_std, self._log_std_min, self._log_std_max)
+
+        # self._log_std = log_std
+        # self._num_samples = mean_actions.shape[0]
+
+        # # distribution
+        # self._distribution = Normal(mean_actions, log_std.exp())
+
+        # # sample using the reparameterization trick
+        # actions = self._distribution.rsample()
+
+        # # clip actions
+        # if self._clip_actions:
+        #     actions = torch.clamp(actions, min=self._clip_actions_min, max=self._clip_actions_max)
+
+        # # log of the probability density function
+        # log_prob = self._distribution.log_prob(inputs.get("taken_actions", actions))
+        # if self._reduction is not None:
+        #     log_prob = self._reduction(log_prob, dim=-1)
+        # if log_prob.dim() != actions.dim():
+        #     log_prob = log_prob.unsqueeze(-1)
+
+        # outputs["mean_actions"] = mean_actions
+        # return actions, log_prob, outputs
+    
+
+    def forward(self,
+            mean_actions, taken_actions, log_std, outputs) -> Tuple[torch.Tensor, Union[torch.Tensor, None], Mapping[str, Union[torch.Tensor, Any]]]:
+        # TODO(ll) comment, input types 
+        """Act stochastically in response to the state of the environment
+
+        :param inputs: Model inputs. The most common keys are:
+
+                       - ``"states"``: state of the environment used to make the decision
+                       - ``"taken_actions"``: actions taken by the policy for the given states
+        :type inputs: dict where the values are typically torch.Tensor
+        :param role: Role play by the model (default: ``""``)
+        :type role: str, optional
+
+        :return: Model output. The first component is the action to be taken by the agent.
+                 The second component is the log of the probability density function.
+                 The third component is a dictionary containing the mean actions ``"mean_actions"``
+                 and extra output values
+        :rtype: tuple of torch.Tensor, torch.Tensor or None, and dict
+
+        Example::
+
+            >>> # given a batch of sample states with shape (4096, 60)
+            >>> actions, log_prob, outputs = model.act({"states": states})
+            >>> print(actions.shape, log_prob.shape, outputs["mean_actions"].shape)
+            torch.Size([4096, 8]) torch.Size([4096, 1]) torch.Size([4096, 8])
+        """
+
+        # clamp log standard deviations
+        if self._clip_log_std:
+            log_std = torch.clamp(log_std, self._log_std_min, self._log_std_max)
+
+        self._log_std = log_std
+        self._num_samples = mean_actions.shape[0]
+
+        # distribution
+        self._distribution = Normal(mean_actions, log_std.exp())
+
+        # sample using the reparameterization trick
+        actions = self._distribution.rsample()
+
+        # clip actions
+        if self._clip_actions:
+            actions = torch.clamp(actions, min=self._clip_actions_min, max=self._clip_actions_max)
+        
+        if taken_actions is None:
+            taken_actions = actions
+        
+        # log of the probability density function
+        log_prob = self._distribution.log_prob(taken_actions)
+        if self._reduction is not None:
+            log_prob = self._reduction(log_prob, dim=-1)
+        if log_prob.dim() != actions.dim():
+            log_prob = log_prob.unsqueeze(-1)
+
+        outputs["mean_actions"] = mean_actions
+        return actions, log_prob, outputs
+    
+
+    def get_entropy(self, role: str = "") -> torch.Tensor:
+        """Compute and return the entropy of the model
+
+        :return: Entropy of the model
+        :rtype: torch.Tensor
+        :param role: Role play by the model (default: ``""``)
+        :type role: str, optional
+
+        Example::
+
+            >>> entropy = model.get_entropy()
+            >>> print(entropy.shape)
+            torch.Size([4096, 8])
+        """
+        if self._distribution is None:
+            return torch.tensor(0.0, device=self.device)
+        return self._distribution.entropy().to(self.device)
+
+    def get_log_std(self, role: str = "") -> torch.Tensor:
+        """Return the log standard deviation of the model
+
+        :return: Log standard deviation of the model
+        :rtype: torch.Tensor
+        :param role: Role play by the model (default: ``""``)
+        :type role: str, optional
+
+        Example::
+
+            >>> log_std = model.get_log_std()
+            >>> print(log_std.shape)
+            torch.Size([4096, 8])
+        """
+        return self._log_std.repeat(self._num_samples, 1)
+
+    def distribution(self, role: str = "") -> torch.distributions.Normal:
+        """Get the current distribution of the model
+
+        :return: Distribution of the model
+        :rtype: torch.distributions.Normal
+        :param role: Role play by the model (default: ``""``)
+        :type role: str, optional
+
+        Example::
+
+            >>> distribution = model.distribution()
+            >>> print(distribution)
+            Normal(loc: torch.Size([4096, 8]), scale: torch.Size([4096, 8]))
+        """
+        return self._distribution
+    
+
+
+
+class DeterministicMixin:
+    def __init__(self, clip_actions: bool = False, role: str = "") -> None:
+        """Deterministic mixin model (deterministic model)
+
+        :param clip_actions: Flag to indicate whether the actions should be clipped to the action space (default: ``False``)
+        :type clip_actions: bool, optional
+        :param role: Role play by the model (default: ``""``)
+        :type role: str, optional
+
+        Example::
+
+            # define the model
+            >>> import torch
+            >>> import torch.nn as nn
+            >>> from skrl.models.torch import Model, DeterministicMixin
+            >>>
+            >>> class Value(DeterministicMixin, Model):
+            ...     def __init__(self, observation_space, action_space, device="cuda:0", clip_actions=False):
+            ...         Model.__init__(self, observation_space, action_space, device)
+            ...         DeterministicMixin.__init__(self, clip_actions)
+            ...
+            ...         self.net = nn.Sequential(nn.Linear(self.num_observations, 32),
+            ...                                  nn.ELU(),
+            ...                                  nn.Linear(32, 32),
+            ...                                  nn.ELU(),
+            ...                                  nn.Linear(32, 1))
+            ...
+            ...     def compute(self, inputs, role):
+            ...         return self.net(inputs["states"]), {}
+            ...
+            >>> # given an observation_space: gym.spaces.Box with shape (60,)
+            >>> # and an action_space: gym.spaces.Box with shape (8,)
+            >>> model = Value(observation_space, action_space)
+            >>>
+            >>> print(model)
+            Value(
+              (net): Sequential(
+                (0): Linear(in_features=60, out_features=32, bias=True)
+                (1): ELU(alpha=1.0)
+                (2): Linear(in_features=32, out_features=32, bias=True)
+                (3): ELU(alpha=1.0)
+                (4): Linear(in_features=32, out_features=1, bias=True)
+              )
+            )
+        """
+        self._clip_actions = clip_actions and (issubclass(type(self.action_space), gym.Space) or \
+            issubclass(type(self.action_space), gymnasium.Space))
+
+        if self._clip_actions:
+            self._clip_actions_min = torch.tensor(self.action_space.low, device=self.device, dtype=torch.float32)
+            self._clip_actions_max = torch.tensor(self.action_space.high, device=self.device, dtype=torch.float32)
+
+    def act(self,
+            inputs: Mapping[str, Union[torch.Tensor, Any]],
+            role: str = "") -> Tuple[torch.Tensor, Union[torch.Tensor, None], Mapping[str, Union[torch.Tensor, Any]]]:
+        """Act deterministically in response to the state of the environment
+
+        :param inputs: Model inputs. The most common keys are:
+
+                       - ``"states"``: state of the environment used to make the decision
+                       - ``"taken_actions"``: actions taken by the policy for the given states
+        :type inputs: dict where the values are typically torch.Tensor
+        :param role: Role play by the model (default: ``""``)
+        :type role: str, optional
+
+        :return: Model output. The first component is the action to be taken by the agent.
+                 The second component is ``None``. The third component is a dictionary containing extra output values
+        :rtype: tuple of torch.Tensor, torch.Tensor or None, and dict
+
+        Example::
+
+            >>> # given a batch of sample states with shape (4096, 60)
+            >>> actions, _, outputs = model.act({"states": states})
+            >>> print(actions.shape, outputs)
+            torch.Size([4096, 1]) {}
+        """
+        # map from observations/states to actions
+        actions, outputs = self.compute(inputs, role)
+
+        # clip actions
+        if self._clip_actions:
+            actions = torch.clamp(actions, min=self._clip_actions_min, max=self._clip_actions_max)
+
+        return actions, None, outputs
+    
+    def forward(self, actions, outputs) -> Tuple[torch.Tensor, Union[torch.Tensor, None], Mapping[str, Union[torch.Tensor, Any]]]:
+        """Act deterministically in response to the state of the environment
+
+        :param inputs: Model inputs. The most common keys are:
+
+                       - ``"states"``: state of the environment used to make the decision
+                       - ``"taken_actions"``: actions taken by the policy for the given states
+        :type inputs: dict where the values are typically torch.Tensor
+        :param role: Role play by the model (default: ``""``)
+        :type role: str, optional
+
+        :return: Model output. The first component is the action to be taken by the agent.
+                 The second component is ``None``. The third component is a dictionary containing extra output values
+        :rtype: tuple of torch.Tensor, torch.Tensor or None, and dict
+
+        Example::
+
+            >>> # given a batch of sample states with shape (4096, 60)
+            >>> actions, _, outputs = model.act({"states": states})
+            >>> print(actions.shape, outputs)
+            torch.Size([4096, 1]) {}
+        """
+        # clip actions
+        if self._clip_actions:
+            actions = torch.clamp(actions, min=self._clip_actions_min, max=self._clip_actions_max)
+
+        return actions, None, outputs