rlmodule 0.1.6.2__tar.gz → 0.1.6.3__tar.gz

{rlmodule-0.1.6.2/rlmodule.egg-info → rlmodule-0.1.6.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rlmodule
-Version: 0.1.6.2
+Version: 0.1.6.3
 Summary: Flexible reinforcement learning models instantiators library
 Author: Lopatovsky
 Author-email: lopatovsky@gmail.com
@@ -79,7 +79,7 @@ rm -rf runs
 
 ### Run chosen example
 ```
-python3 rlmodule/skrl/torch/gymnasium/pendulum_ppo_mlp_separate_model.py
+python3 rlmodule/skrl/torch/examples/gymnasium/pendulum_ppo_mlp_separate_model.py
 ```
 
 Optional: observe run results in Tensorboard

{rlmodule-0.1.6.2 → rlmodule-0.1.6.3}/README.md

@@ -46,7 +46,7 @@ rm -rf runs
 
 ### Run chosen example
 ```
-python3 rlmodule/skrl/torch/gymnasium/pendulum_ppo_mlp_separate_model.py
+python3 rlmodule/skrl/torch/examples/gymnasium/pendulum_ppo_mlp_separate_model.py
 ```
 
 Optional: observe run results in Tensorboard

{rlmodule-0.1.6.2 → rlmodule-0.1.6.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "rlmodule"
-version = "0.1.6.2"
+version = "0.1.6.3"
 description = "Flexible reinforcement learning models instantiators library"
 readme = "README.md"
 requires-python = ">=3.6"

rlmodule-0.1.6.3/rlmodule/skrl/torch/examples/gymnasium/cnn_experimental.py

@@ -0,0 +1,152 @@
+from typing import Tuple
+
+import os
+from datetime import datetime
+import gymnasium as gym
+
+# import the skrl components to build the RL system
+from skrl.agents.torch.ppo import PPO, PPO_DEFAULT_CONFIG, PPO_RNN
+from skrl.envs.wrappers.torch.gymnasium_envs import GymnasiumWrapper
+from skrl.memories.torch import RandomMemory
+from skrl.resources.preprocessors.torch import RunningStandardScaler
+from skrl.resources.schedulers.torch import KLAdaptiveRL
+from skrl.trainers.torch import SequentialTrainer
+from skrl.utils import set_seed
+
+import numpy as np
+import torch.nn as nn
+
+from rlmodule.skrl.torch import SharedRLModelCfg, build_model
+from rlmodule.skrl.torch.network import CnnCfg
+from rlmodule.skrl.torch.output_layer import DeterministicLayerCfg, GaussianLayerCfg, ParameterLogStdCfg
+
+
+# CNN is in an experimental phase. Some combination of inputs may not be yet supported
+
+
+class DummyEnv(gym.Env):
+    """Dummy env producing random observations of given shape."""
+
+    def __init__(self):
+        super(DummyEnv, self).__init__()
+        # Define observation space:
+        self.observation_space = gym.spaces.Box(low=0, high=255, shape=(169,), dtype=np.uint8)
+        # Define action space: Continuous values between -1.0 and 1.0, with 6 action dimensions
+        self.action_space = gym.spaces.Box(low=-1.0, high=1.0, shape=(6,), dtype=np.float32)
+
+    def reset(self) -> Tuple[np.ndarray, dict]:
+        # Reset environment and return initial random observation
+        observation = self._get_random_observation()
+        return observation, {}
+
+    def step(self, action: np.ndarray) -> Tuple[np.ndarray, float, bool, bool, dict]:
+        # Create a new random observation
+        observation = self._get_random_observation()
+        reward = 0.0  # No meaningful reward in this dummy environment
+        terminated = False  # This dummy environment never ends
+        truncated = False
+        info = {}
+        return observation, reward, terminated, truncated, info
+
+    def _get_random_observation(self) -> np.ndarray:
+        # Generate a random observation with values between 0 and 255
+        return np.random.randint(0, 256, 169, dtype=np.uint8)
+
+
+def get_model(env):
+    """Instantiate the agent's models (function approximators)."""
+
+    net_cfg = CnnCfg(
+        input_states=(13, 13, 1),
+        layers=[
+            nn.Conv2d(in_channels=1, out_channels=64, kernel_size=3, stride=2),
+            nn.ReLU(),
+            nn.Flatten(),
+        ],
+    )
+
+    model = build_model(
+        SharedRLModelCfg(
+            network=net_cfg,
+            device=device,
+            policy_output_layer=GaussianLayerCfg(
+                output_size=env.action_space,
+                log_std=ParameterLogStdCfg(
+                    min_log_std=-1.2,
+                    max_log_std=2,
+                    initial_log_std=0.0,
+                ),
+            ),
+            value_output_layer=DeterministicLayerCfg(),
+        )
+    )
+
+    print(model)
+    return {"policy": model, "value": model}
+
+
+# set seed for reproducibility
+seed = 42
+set_seed(seed)
+
+env = DummyEnv()
+# env = gym.make_vec("ALE/Pong-v5", num_envs=4, vectorization_mode="sync")
+env = GymnasiumWrapper(env)
+
+device = env.device
+
+# instantiate a memory as rollout buffer (any memory can be used for this)
+memory = RandomMemory(memory_size=1024, num_envs=env.num_envs, device=device)
+
+models = get_model(env)
+
+# configure and instantiate the agent (visit its documentation to see all the options)
+# https://skrl.readthedocs.io/en/latest/api/agents/ppo.html#configuration-and-hyperparameters
+cfg = PPO_DEFAULT_CONFIG.copy()
+cfg["rollouts"] = 1024  # memory_size
+cfg["learning_epochs"] = 10
+cfg["mini_batches"] = 32
+cfg["discount_factor"] = 0.9
+cfg["lambda"] = 0.95
+cfg["learning_rate"] = 1e-3
+cfg["learning_rate_scheduler"] = KLAdaptiveRL
+cfg["learning_rate_scheduler_kwargs"] = {"kl_threshold": 0.008}
+cfg["grad_norm_clip"] = 0.5
+cfg["ratio_clip"] = 0.2
+cfg["value_clip"] = 0.2
+cfg["clip_predicted_values"] = False
+cfg["entropy_loss_scale"] = 0.0
+cfg["value_loss_scale"] = 0.5
+cfg["kl_threshold"] = 0
+cfg["state_preprocessor"] = RunningStandardScaler
+cfg["state_preprocessor_kwargs"] = {"size": env.observation_space, "device": device}
+cfg["value_preprocessor"] = RunningStandardScaler
+cfg["value_preprocessor_kwargs"] = {"size": 1, "device": device}
+# logging to TensorBoard and write checkpoints (in timesteps)
+cfg["experiment"]["write_interval"] = 500
+cfg["experiment"]["checkpoint_interval"] = 5000
+cfg["experiment"]["directory"] = "runs/skrl/torch/"
+cfg["experiment"]["experiment_name"] = (
+    os.path.splitext(os.path.basename(__file__))[0] + "_" + datetime.now().strftime("%Y-%m-%d_%H:%M:%S")
+)
+
+params = {
+    "models": models,
+    "memory": memory,
+    "cfg": cfg,
+    "observation_space": env.observation_space,
+    "action_space": env.action_space,
+    "device": device,
+}
+
+if models["policy"].is_rnn:
+    agent = PPO_RNN(**params)
+else:
+    agent = PPO(**params)
+
+# configure and instantiate the RL trainer
+cfg_trainer = {"timesteps": 100000, "headless": True}
+trainer = SequentialTrainer(cfg=cfg_trainer, env=env, agents=[agent])
+
+# start training
+trainer.train()

{rlmodule-0.1.6.2/rlmodule/skrl/torch → rlmodule-0.1.6.3/rlmodule/skrl/torch/examples}/gymnasium/pendulum_ppo_mlp_separate_model.py

@@ -27,9 +27,6 @@ def get_model(env):
         activation=nn.ReLU,
     )
 
-    print(env.action_space)
-    print(env.observation_space)
-
     policy_model = build_model(
         RLModelCfg(
             network=net_cfg,

rlmodule-0.1.6.3/rlmodule/skrl/torch/network/__init__.py

@@ -0,0 +1,29 @@
+__all__ = [
+    # modules:
+    "MLP",
+    "RNN",
+    "GRU",
+    "LSTM",
+    "RnnBase",
+    "RnnMlp",
+    "RnnMlpWithForwardedInput",
+    # configs:
+    "NetworkCfg",
+    "MlpCfg",
+    "RnnBaseCfg",
+    "RnnCfg",
+    "GruCfg",
+    "LstmCfg",
+    "RnnMlpCfg",
+]
+from rlmodule.source.network import CNN, GRU, LSTM, MLP, RNN, RnnBase, RnnMlp, RnnMlpWithForwardedInput  # noqa: F401
+from rlmodule.source.network_cfg import (  # noqa: F401
+    CnnCfg,
+    GruCfg,
+    LstmCfg,
+    MlpCfg,
+    NetworkCfg,
+    RnnBaseCfg,
+    RnnCfg,
+    RnnMlpCfg,
+)

{rlmodule-0.1.6.2 → rlmodule-0.1.6.3}/rlmodule/source/build.py

@@ -10,9 +10,9 @@ def build_model(cfg: BaseRLCfg):
     actions_as_input = hasattr(network_cfg, "input_actions") and network_cfg.input_actions is not None
 
     # process input size
-    network_cfg.input_states = get_space_size(network_cfg.input_states)
+    network_cfg.input_size = get_space_size(network_cfg.input_states)
     if actions_as_input:
-        network_cfg.input_states += get_space_size(network_cfg.input_actions)
+        network_cfg.input_size += get_space_size(network_cfg.input_actions)
 
     # build base network of function approximator
     net = network_cfg.module(network_cfg)
@@ -21,10 +21,10 @@ def build_model(cfg: BaseRLCfg):
         net.input_actions = True
 
     # get output size to be used as output layer input
-    network_output_size = get_output_size(net, network_cfg.input_states)
+    network_output_size = get_output_size(net, network_cfg.input_size)
 
     def build_output_layer(layer_cfg):
-        return layer_cfg.class_type(device=cfg.device, input_states=network_output_size, cfg=layer_cfg)
+        return layer_cfg.class_type(device=cfg.device, input_size=network_output_size, cfg=layer_cfg)
 
     # build function approximator
     if isinstance(cfg, RLModelCfg):

{rlmodule-0.1.6.2 → rlmodule-0.1.6.3}/rlmodule/source/network.py

@@ -1,6 +1,8 @@
 import torch
 import torch.nn as nn
 
+from rlmodule.source.utils import get_cnn_shape
+
 
 class MLP(nn.Module):
     """Configurable multilayer perceptron module.
@@ -10,7 +12,7 @@ class MLP(nn.Module):
     Example of use:
 
         cfg = MlpCfg(
-            input_states = 517,
+            input_size = 517,
             hidden_units = [2048, 1024, 1024, 512],
             activation = nn.ELU,
         )
@@ -22,7 +24,7 @@ class MLP(nn.Module):
 
         # input layer
         layers = [
-            nn.Linear(cfg.input_states, cfg.hidden_units[0]),
+            nn.Linear(cfg.input_size, cfg.hidden_units[0]),
             cfg.activation(),
         ]
 
@@ -66,7 +68,7 @@ class LSTM(RnnBase):
     Example of use:
 
         cfg = LstmCfg(
-            input_states = 517,
+            input_size = 517,
             num_envs = 2048,
             num_layers = 1,
             hidden_size = 512 + 256,
@@ -79,7 +81,7 @@ class LSTM(RnnBase):
         super().__init__(cfg)
 
         self.lstm = nn.LSTM(
-            input_size=cfg.input_states,
+            input_size=cfg.input_size,
             hidden_size=cfg.hidden_size,
             num_layers=cfg.num_layers,
             batch_first=True,
@@ -235,7 +237,7 @@ class RNN(RnnModule):
     Example of use:
 
         cfg = RnnCfg(
-            input_states = 517,
+            input_size = 517,
             num_envs = 2048,
             num_layers = 1,
             hidden_size = 512 + 256,
@@ -248,7 +250,7 @@ class RNN(RnnModule):
         super().__init__(cfg)
 
         self.rnn = nn.RNN(
-            input_size=cfg.input_states,
+            input_size=cfg.input_size,
             hidden_size=cfg.hidden_size,
             num_layers=cfg.num_layers,
             batch_first=True,
@@ -266,7 +268,7 @@ class GRU(RnnModule):
     Example of use:
 
         cfg = GruCfg(
-            input_states = 517,
+            input_size = 517,
             num_envs = 2048,
             num_layers = 1,
             hidden_size = 512 + 256,
@@ -279,7 +281,7 @@ class GRU(RnnModule):
         super().__init__(cfg)
 
         self.rnn = nn.GRU(
-            input_size=cfg.input_states,
+            input_size=cfg.input_size,
             hidden_size=cfg.hidden_size,
             num_layers=cfg.num_layers,
             batch_first=True,
@@ -298,7 +300,7 @@ class RnnMlp(RnnBase):
     1) Example of use (with RNN module):
 
         cfg = RnnMlpCfg(
-                input_states = 517,
+                input_size = 517,
                 rnn = RnnCfg(
                     num_envs = 2048,
                     num_layers = 1,
@@ -315,7 +317,7 @@ class RnnMlp(RnnBase):
     2) Example of use (with GRU module):
 
         cfg = RnnMlpCfg(
-                input_states = 517,
+                input_size = 517,
                 rnn = GruCfg(
                     num_envs = 2048,
                     num_layers = 1,
@@ -332,7 +334,7 @@ class RnnMlp(RnnBase):
     3) Example of use (with LSTM module):
 
         cfg = RnnMlpCfg(
-                input_states = 517,
+                input_size = 517,
                 rnn = LstmCfg(
                     num_envs = 2048,
                     num_layers = 1,
@@ -349,9 +351,9 @@ class RnnMlp(RnnBase):
 
     def __init__(self, cfg):
         super().__init__(cfg.rnn)
-        cfg.rnn.input_states = cfg.input_states
+        cfg.rnn.input_size = cfg.input_size
         self.rnn = cfg.rnn.module(cfg.rnn)
-        cfg.mlp.input_states = self.hidden_size
+        cfg.mlp.input_size = self.hidden_size
         self.mlp = MLP(cfg.mlp)
 
     def forward(self, states, terminated, rnn_inputs):
@@ -372,7 +374,7 @@ class RnnMlpWithForwardedInput(RnnBase):
     1) Example of use (with RNN module):
 
         cfg = RnnMlpCfg(
-                input_states = 517,
+                input_size = 517,
                 module = RnnMlpWithForwardedInput,
                 rnn = RnnCfg(
                     num_envs = 2048,
@@ -390,7 +392,7 @@ class RnnMlpWithForwardedInput(RnnBase):
     2) Example of use (with GRU module):
 
         cfg = RnnMlpCfg(
-                input_states = 517,
+                input_size = 517,
                 module = RnnMlpWithForwardedInput,
                 rnn = GruCfg(
                     num_envs = 2048,
@@ -408,7 +410,7 @@ class RnnMlpWithForwardedInput(RnnBase):
     3) Example of use (with LSTM module):
 
         cfg = RnnMlpCfg(
-                input_states = 517,
+                input_size = 517,
                 module = RnnMlpWithForwardedInput,
                 rnn = LstmCfg(
                     num_envs = 2048,
@@ -430,9 +432,9 @@ class RnnMlpWithForwardedInput(RnnBase):
 
     def __init__(self, cfg):
         super().__init__(cfg.rnn)
-        cfg.rnn.input_states = cfg.input_states
+        cfg.rnn.input_size = cfg.input_size
         self.rnn = cfg.rnn.module(cfg.rnn)
-        cfg.mlp.input_states = cfg.input_states + self.hidden_size
+        cfg.mlp.input_size = cfg.input_size + self.hidden_size
         self.mlp = MLP(cfg.mlp)
 
     def forward(self, states, terminated, rnn_inputs):
@@ -442,93 +444,40 @@ class RnnMlpWithForwardedInput(RnnBase):
         return self.mlp(mlp_input), output_dict
 
 
-# CNN - coming soon
+class CNN(nn.Module):
+    """Experimental implementation of Convolutional neural network.
+    Architecture is defined by providing modules_cfg.MlpCfg.
 
-# def get_cnn_layer(params):
-#     """
-#     Create a CNN layer based on the provided parameters and activation function.
-
-#     Args:
-#         params (dict): Dictionary containing the parameters for the layer.
-#             Expected keys are:
-#                 - 'type' (str): Type of the layer ('conv' for convolutional, 'pool' for pooling,
-#                                                    'dense' for fully connected).
-#                 - 'in_channels' (int): Number of input channels (required for 'conv' type).
-#                 - 'out_channels' (int): Number of output channels (required for 'conv' type).
-#                 - 'kernel_size' (int or tuple): Size of the kernel (required for 'conv' or 'pool' type).
-#                 - 'stride' (int or tuple): Stride of the convolution or pooling operation
-#                                            (required for 'conv' or 'pool' type).
-#                 - 'in_features' (int): Number of input features (required for 'dense' type).
-#                 - 'out_features' (int): Number of output features (required for 'dense' type).
-#                 - 'activation' (str): Activation function to use after the layer (only for 'conv' and 'dense' types).
-
-#     Returns:
-#         list: List containing the created layer(s). For 'conv' type, it includes the convolutional
-#               layer followed by the activation function. For 'pool' type, it includes only the pooling layer.
-#               For 'dense' type, it includes the fully connected layer followed by the activation function.
-
-#     Raises:
-#         ValueError: If the 'type' specified in params is not supported.
-#     """
-#     if params['type'] == 'conv':
-#         return [
-#             nn.Conv2d(
-#                 in_channels=params['in_channels'],
-#                 out_channels=params['out_channels'],
-#                 kernel_size=params['kernel_size'],
-#                 stride=params['stride'],
-#             ),
-#             _get_activation_function(params['activation']),
-#         ]
-#     elif params['type'] == 'pool':
-#         return [
-#             nn.MaxPool2d(
-#                 kernel_size=params['kernel_size'],
-#                 stride=params['stride'],
-#             )
-#         ]
-#     elif params['type'] == 'dense':
-#         return [
-#             nn.Flatten(),  # if there is 2D layer before need to be flatten to 1D.
-#             nn.Linear(
-#                 in_features=params['in_features'],
-#                 out_features=params['out_features'],
-#             ),
-#             _get_activation_function(params['activation']),
-#         ]
-#     else:
-#         raise ValueError(f"Unsupported layer type: {params['type']}")
-
-
-# class CNN(nn.Module):
-#     def __init__(self, params):
-#         super().__init__()
+    Warning: CNN is in an experimental phase. Some combination of inputs may not be yet supported.
 
-#         layers = sum([get_cnn_layer(layer_params) for layer_params in params['layers']], [])
-#         self.cnn = nn.Sequential(*layers, nn.Flatten())
+    Example of use:
 
-#     def forward(self, input):
-#         return self.cnn(input)
+        cfg = CnnCfg(
+            input_size = (13, 13, 1)  # input states in order (x,y,channels)
+            layers=[
+                nn.Conv2d(in_channels=1, out_channels=64, kernel_size=3, stride=2),
+                nn.ReLU(),
+                nn.Flatten()
+            ]
+        )
+        net = CNN(cfg)
+    """
 
+    def __init__(self, cfg):
+        super().__init__()
+        self._input_size = cfg.input_size
 
-# def example_CNN():
-#     params = {
-#         'input_shape': [1, 13, 13],
-#         'layers': [
-#             {'type': 'conv', 'kernel_size': 3, 'stride': 2, 'in_channels': 1,
-#              'out_channels': 32, 'activation': 'relu'},
-#             {
-#                 'type': 'conv',
-#                 'kernel_size': 3,
-#                 'stride': 1,
-#                 'in_channels': 32,
-#                 'out_channels': 64,
-#                 'activation': 'relu',
-#             },
-#         ],
-#     }
+        self._input_shape = get_cnn_shape(cfg.input_states)
 
-#     return CNN(params)
+        modules = []
+        for layer in cfg.layers:
+            modules.append(layer)
+
+        self.cnn = nn.Sequential(*modules)
+
+    def forward(self, input):
+        # Cnn expects inputs in shape (batch, channels, x, y)
+        return self.cnn(input.view(-1, *self._input_shape).permute(0, 3, 1, 2))
 
 
 # class TripleCnnAndMlp(nn.Module):
@@ -570,7 +519,7 @@ class RnnMlpWithForwardedInput(RnnBase):
 
 #         self.cnns = nn.ModuleList([CNN(cnn_params) for _ in range(self.cnn_number)])
 
-#         mlp_params['input_states'] = self.prefix_length + self.cnn_number * get_output_size(
+#         mlp_params['input_size'] = self.prefix_length + self.cnn_number * get_output_size(
 #             self.cnns[0], self.input_shape
 #         )
 
@@ -630,13 +579,13 @@ class RnnMlpWithForwardedInput(RnnBase):
 
 #         self.cnns = nn.ModuleList([CNN(cnn_params) for _ in range(self.cnn_number)])
 
-#         rnn_params['input_states'] = self.prefix_length + self.cnn_number * get_output_size(
+#         rnn_params['input_size'] = self.prefix_length + self.cnn_number * get_output_size(
 #             self.cnns[0], self.input_shape
 #         )
 
 #         self.rnn = rnn_class(rnn_params)
 
-#         mlp_params['input_states'] = self.rnn.hidden_size + self.rnn.input_states
+#         mlp_params['input_size'] = self.rnn.hidden_size + self.rnn.input_size
 
 #         self.mlp = MLP(mlp_params)
 

{rlmodule-0.1.6.2 → rlmodule-0.1.6.3}/rlmodule/source/network_cfg.py

@@ -8,7 +8,7 @@ import gymnasium
 import torch.nn as nn
 
 from rlmodule import logger
-from rlmodule.source.network import GRU, LSTM, MLP, RNN, RnnBase, RnnMlp
+from rlmodule.source.network import CNN, GRU, LSTM, MLP, RNN, RnnBase, RnnMlp
 
 
 # use isaac-lab native configclass if available to avoid it being declared twice
@@ -22,6 +22,8 @@ except ImportError:
 @configclass
 class NetworkCfg:
     module: Union[nn.Module, Callable[..., nn.Module]] = MISSING
+    input_size: int = None
+    """Size of all network inputs. This field should be automatically derived from input_states/input_actions."""
     input_states: Union[int, Sequence[int], gym.Space, gymnasium.Space] = None
     """Observations passed to the network as an input."""
     input_actions: Union[int, Sequence[int], gym.Space, gymnasium.Space] = None
@@ -65,3 +67,16 @@ class RnnMlpCfg(NetworkCfg):
 
     rnn: RnnBaseCfg = MISSING
     mlp: MlpCfg = MISSING
+
+
+@configclass
+class CnnCfg(NetworkCfg):
+    """Config for convolutional neural network.
+    For network to work properly the input states need to be specified as a triple in order (x,y,channels)
+    Warning: CNN is in an experimental phase. Some combination of inputs may not be yet supported.
+    """
+
+    module: type[CNN] = CNN
+
+    layers: Sequence[nn.Module] = MISSING
+    """Layers of convolutional neural network."""

{rlmodule-0.1.6.2 → rlmodule-0.1.6.3}/rlmodule/source/output_layer.py

@@ -11,7 +11,7 @@ from rlmodule.source.utils import get_space_size
 
 
 class OutputLayer(nn.Module):
-    def __init__(self, device: Union[str, torch.device], input_states: int, cfg):
+    def __init__(self, device: Union[str, torch.device], input_size: int, cfg):
         """Base class for OutputLayer class hierarchy."""
         super().__init__()
         self.device = device
@@ -26,26 +26,26 @@ class OutputLayer(nn.Module):
             self._clip_actions_min = torch.tensor(cfg.output_size.low, device=self.device, dtype=torch.float32)
             self._clip_actions_max = torch.tensor(cfg.output_size.high, device=self.device, dtype=torch.float32)
 
-        self._input_states = input_states
+        self._input_size = input_size
         self._output_size = get_space_size(cfg.output_size)
 
         self._output_scale = cfg.output_scale
 
 
 class GaussianLayer(OutputLayer):
-    def __init__(self, device: Union[str, torch.device], input_states: int, cfg):
+    def __init__(self, device: Union[str, torch.device], input_size: int, cfg):
         """Gaussian output layer
 
         :param device: Device on which a tensor/array is or will be allocated
         :type device: str or torch.device
-        :param input_states
+        :param input_size
         :type int
         :param cfg: Configuration of Gaussian output layer
         :type OutputLayerCfg
 
         :raises ValueError: If the reduction method is not valid
         """
-        super().__init__(device, input_states, cfg)
+        super().__init__(device, input_size, cfg)
 
         self._cached_log_std = None
         self._num_samples = None
@@ -59,9 +59,9 @@ class GaussianLayer(OutputLayer):
             else torch.sum if cfg.reduction == "sum" else torch.prod if cfg.reduction == "prod" else None
         )
 
-        self._net = nn.Sequential(nn.Linear(self._input_states, self._output_size), cfg.output_activation())
+        self._net = nn.Sequential(nn.Linear(self._input_size, self._output_size), cfg.output_activation())
 
-        self._log_std = cfg.log_std.class_type(device, input_states, self._output_size, cfg.log_std)
+        self._log_std = cfg.log_std.class_type(device, input_size, self._output_size, cfg.log_std)
 
     def forward(self, input, taken_actions, outputs_dict):
         """Act stochastically in response to the state of the environment
@@ -185,20 +185,20 @@ class GaussianLayer(OutputLayer):
 
 
 class DeterministicLayer(OutputLayer):
-    def __init__(self, device: Union[str, torch.device], input_states: int, cfg):
+    def __init__(self, device: Union[str, torch.device], input_size: int, cfg):
         """Deterministic output layer
 
         :param device: Device on which a tensor/array is or will be allocated
         :type device: str or torch.device
-        :param input_states
+        :param input_size
         :type int
         :param cfg: Configuration of Gaussian output layer
         :type OutputLayerCfg
         """
 
-        super().__init__(device, input_states, cfg)
+        super().__init__(device, input_size, cfg)
 
-        self._net = nn.Sequential(nn.Linear(input_states, cfg.output_size), cfg.output_activation())
+        self._net = nn.Sequential(nn.Linear(input_size, cfg.output_size), cfg.output_activation())
 
     def forward(self, input, taken_actions, outputs_dict):
         """Act deterministically in response to the state of the environment

{rlmodule-0.1.6.2 → rlmodule-0.1.6.3}/rlmodule/source/utils.py

@@ -28,6 +28,35 @@ def get_output_size(module, input_shape):
         return module(dummy_input).view(-1).shape[0]
 
 
+def get_cnn_shape(space: Union[int, Sequence[int], gym.Space, gymnasium.Space]):
+    """Experimental function to get shape of cnn input.
+
+    :param space: Space or shape from which to obtain the number of elements
+    :type space: int, sequence of int, gym.Space, or gymnasium.Space
+
+    :raises ValueError: If the space is not supported
+    :rtype: tuple
+
+    Example::
+        # from sequence of int
+        >>> model._get_space_size([2, 3])
+        (2, 3)
+
+        # Box space
+        >>> space = gym.spaces.Box(low=-1, high=1, shape=(2, 3))
+        >>> model._get_space_size(space)
+        (2, 3)
+
+        # Other types are currently not supported as a cnn input
+    """
+    if type(space) in [tuple, list]:
+        return tuple(space)
+    elif issubclass(type(space), gym.spaces.Box):
+        return space.shape
+    else:
+        raise ValueError(f"Space type {type(space)} is not supported as cnn input.")
+
+
 def get_space_size(
     space: Union[int, Sequence[int], gym.Space, gymnasium.Space], number_of_elements: bool = True
 ) -> int:

{rlmodule-0.1.6.2 → rlmodule-0.1.6.3/rlmodule.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rlmodule
-Version: 0.1.6.2
+Version: 0.1.6.3
 Summary: Flexible reinforcement learning models instantiators library
 Author: Lopatovsky
 Author-email: lopatovsky@gmail.com
@@ -79,7 +79,7 @@ rm -rf runs
 
 ### Run chosen example
 ```
-python3 rlmodule/skrl/torch/gymnasium/pendulum_ppo_mlp_separate_model.py
+python3 rlmodule/skrl/torch/examples/gymnasium/pendulum_ppo_mlp_separate_model.py
 ```
 
 Optional: observe run results in Tensorboard

{rlmodule-0.1.6.2 → rlmodule-0.1.6.3}/rlmodule.egg-info/SOURCES.txt

@@ -9,16 +9,17 @@ rlmodule.egg-info/requires.txt
 rlmodule.egg-info/top_level.txt
 rlmodule/skrl/__init__.py
 rlmodule/skrl/torch/__init__.py
-rlmodule/skrl/torch/gymnasium/pendulum_ppo_mlp_separate_model.py
-rlmodule/skrl/torch/gymnasium/pendulum_ppo_mlp_shared_model.py
-rlmodule/skrl/torch/gymnasium/pendulum_ppo_rnn_forwarded_input_shared_model.py
-rlmodule/skrl/torch/gymnasium/pendulum_ppo_rnn_separate_model.py
-rlmodule/skrl/torch/gymnasium/pendulum_ppo_rnn_shared_model.py
-rlmodule/skrl/torch/gymnasium/pendulum_sac_mlp_separate_model.py
-rlmodule/skrl/torch/gymnasium/skrl_no_module.py
-rlmodule/skrl/torch/gymnasium/skrl_no_module_rnn.py
-rlmodule/skrl/torch/gymnasium/skrl_sac.py
-rlmodule/skrl/torch/gymnasium/temp.py
+rlmodule/skrl/torch/examples/gymnasium/cnn_experimental.py
+rlmodule/skrl/torch/examples/gymnasium/pendulum_ppo_mlp_separate_model.py
+rlmodule/skrl/torch/examples/gymnasium/pendulum_ppo_mlp_shared_model.py
+rlmodule/skrl/torch/examples/gymnasium/pendulum_ppo_rnn_forwarded_input_shared_model.py
+rlmodule/skrl/torch/examples/gymnasium/pendulum_ppo_rnn_separate_model.py
+rlmodule/skrl/torch/examples/gymnasium/pendulum_ppo_rnn_shared_model.py
+rlmodule/skrl/torch/examples/gymnasium/pendulum_sac_mlp_separate_model.py
+rlmodule/skrl/torch/examples/gymnasium/skrl_no_module.py
+rlmodule/skrl/torch/examples/gymnasium/skrl_no_module_rnn.py
+rlmodule/skrl/torch/examples/gymnasium/skrl_sac.py
+rlmodule/skrl/torch/examples/gymnasium/temp.py
 rlmodule/skrl/torch/network/__init__.py
 rlmodule/skrl/torch/output_layer/__init__.py
 rlmodule/source/__init__.py

rlmodule-0.1.6.2/rlmodule/skrl/torch/network/__init__.py

@@ -1,20 +0,0 @@
-__all__ = [
-    # modules:
-    "MLP",
-    "RNN",
-    "GRU",
-    "LSTM",
-    "RnnBase",
-    "RnnMlp",
-    "RnnMlpWithForwardedInput",
-    # configs:
-    "NetworkCfg",
-    "MlpCfg",
-    "RnnBaseCfg",
-    "RnnCfg",
-    "GruCfg",
-    "LstmCfg",
-    "RnnMlpCfg",
-]
-from rlmodule.source.network import GRU, LSTM, MLP, RNN, RnnBase, RnnMlp, RnnMlpWithForwardedInput  # noqa: F401
-from rlmodule.source.network_cfg import GruCfg, LstmCfg, MlpCfg, NetworkCfg, RnnBaseCfg, RnnCfg, RnnMlpCfg  # noqa: F401