rlmodule 0.1.6.2__tar.gz → 0.1.7__tar.gz

{rlmodule-0.1.6.2/rlmodule.egg-info → rlmodule-0.1.7}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: rlmodule
-Version: 0.1.6.2
+Version: 0.1.7
 Summary: Flexible reinforcement learning models instantiators library
 Author: Lopatovsky
 Author-email: lopatovsky@gmail.com
@@ -18,18 +18,22 @@ Classifier: Intended Audience :: Science/Research
 Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Classifier: Operating System :: OS Independent
-Requires-Python: >=3.6
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: gym
 Requires-Dist: gymnasium==0.29.0
 Requires-Dist: torch>=1.9
-Requires-Dist: skrl>=1.3.0
+Requires-Dist: skrl>=2.1
 Provides-Extra: skrl
-Requires-Dist: skrl>=1.3.0; extra == "skrl"
+Requires-Dist: skrl>=2.1; extra == "skrl"
 Provides-Extra: all
-Requires-Dist: skrl>=1.3.0; extra == "all"
+Requires-Dist: skrl>=2.1; extra == "all"
+Dynamic: license-file
 
 # rlmodule
 Flexible reinforcement learning models instantiators library
@@ -77,9 +81,17 @@ Note: sometimes installation may fail, if there is a run/ dir present, you may n
 rm -rf runs
 ```
 
+### Deep policies: LayerNorm and Residual MLPs
+
+`rlmodule` supports per-layer LayerNorm on `MlpCfg` and a SimBa-style residual
+MLP (`ResidualMlpCfg`). These work for both separated and shared
+architectures, with independent per-head granularity via the new
+`OutputLayerCfg.head` field. See [docs/layer_norm_and_residual.md](docs/layer_norm_and_residual.md)
+for motivation, references, and copy-pasteable cfg snippets.
+
 ### Run chosen example
 ```
-python3 rlmodule/skrl/torch/gymnasium/pendulum_ppo_mlp_separate_model.py
+python3 rlmodule/skrl/torch/examples/gymnasium/pendulum_ppo_mlp_separate_model.py
 ```
 
 Optional: observe run results in Tensorboard

{rlmodule-0.1.6.2 → rlmodule-0.1.7}/README.md

@@ -44,9 +44,17 @@ Note: sometimes installation may fail, if there is a run/ dir present, you may n
 rm -rf runs
 ```
 
+### Deep policies: LayerNorm and Residual MLPs
+
+`rlmodule` supports per-layer LayerNorm on `MlpCfg` and a SimBa-style residual
+MLP (`ResidualMlpCfg`). These work for both separated and shared
+architectures, with independent per-head granularity via the new
+`OutputLayerCfg.head` field. See [docs/layer_norm_and_residual.md](docs/layer_norm_and_residual.md)
+for motivation, references, and copy-pasteable cfg snippets.
+
 ### Run chosen example
 ```
-python3 rlmodule/skrl/torch/gymnasium/pendulum_ppo_mlp_separate_model.py
+python3 rlmodule/skrl/torch/examples/gymnasium/pendulum_ppo_mlp_separate_model.py
 ```
 
 Optional: observe run results in Tensorboard

{rlmodule-0.1.6.2 → rlmodule-0.1.7}/pyproject.toml

@@ -1,9 +1,13 @@
+[build-system]
+requires = ["setuptools>=61.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
 [project]
 name = "rlmodule"
-version = "0.1.6.2"
+version = "0.1.7"
 description = "Flexible reinforcement learning models instantiators library"
 readme = "README.md"
-requires-python = ">=3.6"
+requires-python = ">=3.10"
 license = {text = "MIT License"}
 authors = [
   {name = "Lopatovsky"},
@@ -20,24 +24,27 @@ classifiers = [
   "Topic :: Scientific/Engineering",
   "Topic :: Scientific/Engineering :: Artificial Intelligence",
   "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
   "Operating System :: OS Independent",
 ]
 dependencies = [
   "gym",
   "gymnasium==0.29.0",
   "torch>=1.9",
-  "skrl>=1.3.0",
+  "skrl>=2.1",
 ]
 
 [project.optional-dependencies]
 
 # TODO(ll) only examples should depend on skrl, not the main code.
 skrl = [
-  "skrl>=1.3.0",
+  "skrl>=2.1",
 ]
 
 all = [
-  "skrl>=1.3.0",
+  "skrl>=2.1",
 ]
 
 # urls
@@ -49,6 +56,10 @@ all = [
 "Say Thanks!" = "https://github.com/lopatovsky"
 "Source" = "https://github.com/FabricaAI/rlmodule"
 
+[tool.setuptools.packages.find]
+include = ["rlmodule*"]
+exclude = ["tests*", "docs*"]
+
 [tool.yapf]
 # run: yapf -p -m -i -r <folder>
 based_on_style = "pep8"
@@ -94,3 +105,9 @@ sections = [
 ]
 no_lines_before = "THIRDPARTY"
 skip = ["docs"]
+
+
+[tool.pytest.ini_options]
+markers = [
+  "slow: marks tests as slow (skip with: -m 'not slow')",
+]

{rlmodule-0.1.6.2 → rlmodule-0.1.7}/rlmodule/skrl/torch/__init__.py

@@ -1,3 +1,3 @@
-__all__ = ["build_modelBaseRLCfg", "RLModelCfg", "SeparatedRLModelCfg", "SharedRLModelCfg"]
+__all__ = ["build_model", "BaseRLCfg", "RLModelCfg", "SeparatedRLModelCfg", "SharedRLModelCfg"]
 from rlmodule.source.build import build_model  # noqa: F401
 from rlmodule.source.rlmodel_cfg import BaseRLCfg, RLModelCfg, SeparatedRLModelCfg, SharedRLModelCfg  # noqa: F401

rlmodule-0.1.7/rlmodule/skrl/torch/examples/gymnasium/cnn_experimental.py

@@ -0,0 +1,152 @@
+from typing import Tuple
+
+import os
+from datetime import datetime
+import gymnasium as gym
+
+# import the skrl components to build the RL system
+from skrl.agents.torch.ppo import PPO, PPO_DEFAULT_CONFIG, PPO_RNN
+from skrl.envs.wrappers.torch.gymnasium_envs import GymnasiumWrapper
+from skrl.memories.torch import RandomMemory
+from skrl.resources.preprocessors.torch import RunningStandardScaler
+from skrl.resources.schedulers.torch import KLAdaptiveRL
+from skrl.trainers.torch import SequentialTrainer
+from skrl.utils import set_seed
+
+import numpy as np
+import torch.nn as nn
+
+from rlmodule.skrl.torch import SharedRLModelCfg, build_model
+from rlmodule.skrl.torch.network import CnnCfg
+from rlmodule.skrl.torch.output_layer import DeterministicLayerCfg, GaussianLayerCfg, ParameterLogStdCfg
+
+
+# CNN is in an experimental phase. Some combination of inputs may not be yet supported
+
+
+class DummyEnv(gym.Env):
+    """Dummy env producing random observations of given shape."""
+
+    def __init__(self):
+        super(DummyEnv, self).__init__()
+        # Define observation space:
+        self.observation_space = gym.spaces.Box(low=0, high=255, shape=(169,), dtype=np.uint8)
+        # Define action space: Continuous values between -1.0 and 1.0, with 6 action dimensions
+        self.action_space = gym.spaces.Box(low=-1.0, high=1.0, shape=(6,), dtype=np.float32)
+
+    def reset(self) -> Tuple[np.ndarray, dict]:
+        # Reset environment and return initial random observation
+        observation = self._get_random_observation()
+        return observation, {}
+
+    def step(self, action: np.ndarray) -> Tuple[np.ndarray, float, bool, bool, dict]:
+        # Create a new random observation
+        observation = self._get_random_observation()
+        reward = 0.0  # No meaningful reward in this dummy environment
+        terminated = False  # This dummy environment never ends
+        truncated = False
+        info = {}
+        return observation, reward, terminated, truncated, info
+
+    def _get_random_observation(self) -> np.ndarray:
+        # Generate a random observation with values between 0 and 255
+        return np.random.randint(0, 256, 169, dtype=np.uint8)
+
+
+def get_model(env):
+    """Instantiate the agent's models (function approximators)."""
+
+    net_cfg = CnnCfg(
+        input_states=(13, 13, 1),
+        layers=[
+            nn.Conv2d(in_channels=1, out_channels=64, kernel_size=3, stride=2),
+            nn.ReLU(),
+            nn.Flatten(),
+        ],
+    )
+
+    model = build_model(
+        SharedRLModelCfg(
+            network=net_cfg,
+            device=device,
+            policy_output_layer=GaussianLayerCfg(
+                output_size=env.action_space,
+                log_std=ParameterLogStdCfg(
+                    min_log_std=-1.2,
+                    max_log_std=2,
+                    initial_log_std=0.0,
+                ),
+            ),
+            value_output_layer=DeterministicLayerCfg(),
+        )
+    )
+
+    print(model)
+    return {"policy": model, "value": model}
+
+
+# set seed for reproducibility
+seed = 42
+set_seed(seed)
+
+env = DummyEnv()
+# env = gym.make_vec("ALE/Pong-v5", num_envs=4, vectorization_mode="sync")
+env = GymnasiumWrapper(env)
+
+device = env.device
+
+# instantiate a memory as rollout buffer (any memory can be used for this)
+memory = RandomMemory(memory_size=1024, num_envs=env.num_envs, device=device)
+
+models = get_model(env)
+
+# configure and instantiate the agent (visit its documentation to see all the options)
+# https://skrl.readthedocs.io/en/latest/api/agents/ppo.html#configuration-and-hyperparameters
+cfg = PPO_DEFAULT_CONFIG.copy()
+cfg["rollouts"] = 1024  # memory_size
+cfg["learning_epochs"] = 10
+cfg["mini_batches"] = 32
+cfg["discount_factor"] = 0.9
+cfg["lambda"] = 0.95
+cfg["learning_rate"] = 1e-3
+cfg["learning_rate_scheduler"] = KLAdaptiveRL
+cfg["learning_rate_scheduler_kwargs"] = {"kl_threshold": 0.008}
+cfg["grad_norm_clip"] = 0.5
+cfg["ratio_clip"] = 0.2
+cfg["value_clip"] = 0.2
+cfg["clip_predicted_values"] = False
+cfg["entropy_loss_scale"] = 0.0
+cfg["value_loss_scale"] = 0.5
+cfg["kl_threshold"] = 0
+cfg["state_preprocessor"] = RunningStandardScaler
+cfg["state_preprocessor_kwargs"] = {"size": env.observation_space, "device": device}
+cfg["value_preprocessor"] = RunningStandardScaler
+cfg["value_preprocessor_kwargs"] = {"size": 1, "device": device}
+# logging to TensorBoard and write checkpoints (in timesteps)
+cfg["experiment"]["write_interval"] = 500
+cfg["experiment"]["checkpoint_interval"] = 5000
+cfg["experiment"]["directory"] = "runs/skrl/torch/"
+cfg["experiment"]["experiment_name"] = (
+    os.path.splitext(os.path.basename(__file__))[0] + "_" + datetime.now().strftime("%Y-%m-%d_%H:%M:%S")
+)
+
+params = {
+    "models": models,
+    "memory": memory,
+    "cfg": cfg,
+    "observation_space": env.observation_space,
+    "action_space": env.action_space,
+    "device": device,
+}
+
+if models["policy"].is_rnn:
+    agent = PPO_RNN(**params)
+else:
+    agent = PPO(**params)
+
+# configure and instantiate the RL trainer
+cfg_trainer = {"timesteps": 100000, "headless": True}
+trainer = SequentialTrainer(cfg=cfg_trainer, env=env, agents=[agent])
+
+# start training
+trainer.train()

{rlmodule-0.1.6.2/rlmodule/skrl/torch → rlmodule-0.1.7/rlmodule/skrl/torch/examples}/gymnasium/pendulum_ppo_mlp_separate_model.py

@@ -27,9 +27,6 @@ def get_model(env):
         activation=nn.ReLU,
     )
 
-    print(env.action_space)
-    print(env.observation_space)
-
     policy_model = build_model(
         RLModelCfg(
             network=net_cfg,

rlmodule-0.1.7/rlmodule/skrl/torch/network/__init__.py

@@ -0,0 +1,59 @@
+__all__ = [
+    # modules:
+    "MLP",
+    "RNN",
+    "GRU",
+    "LSTM",
+    "RnnBase",
+    "RnnMlp",
+    "RnnMlpWithForwardedInput",
+    "ResidualMlp",
+    "CnnBackbone",
+    "CnnMlp",
+    "CnnProprioMlp",
+    "GraphNetwork",
+    # configs:
+    "NetworkCfg",
+    "MlpCfg",
+    "ResidualMlpCfg",
+    "RnnBaseCfg",
+    "RnnCfg",
+    "GruCfg",
+    "LstmCfg",
+    "RnnMlpCfg",
+    "CnnCfg",
+    "CnnBackboneCfg",
+    "CnnMlpCfg",
+    "CnnProprioMlpCfg",
+    "GraphNetworkCfg",
+]
+from rlmodule.source.network import (  # noqa: F401
+    CNN,
+    GRU,
+    LSTM,
+    MLP,
+    RNN,
+    CnnBackbone,
+    CnnMlp,
+    CnnProprioMlp,
+    GraphNetwork,
+    ResidualMlp,
+    RnnBase,
+    RnnMlp,
+    RnnMlpWithForwardedInput,
+)
+from rlmodule.source.network_cfg import (  # noqa: F401
+    CnnBackboneCfg,
+    CnnCfg,
+    CnnMlpCfg,
+    CnnProprioMlpCfg,
+    GraphNetworkCfg,
+    GruCfg,
+    LstmCfg,
+    MlpCfg,
+    NetworkCfg,
+    ResidualMlpCfg,
+    RnnBaseCfg,
+    RnnCfg,
+    RnnMlpCfg,
+)

{rlmodule-0.1.6.2 → rlmodule-0.1.7}/rlmodule/source/build.py

@@ -10,9 +10,9 @@ def build_model(cfg: BaseRLCfg):
     actions_as_input = hasattr(network_cfg, "input_actions") and network_cfg.input_actions is not None
 
     # process input size
-    network_cfg.input_states = get_space_size(network_cfg.input_states)
+    network_cfg.input_size = get_space_size(network_cfg.input_states)
     if actions_as_input:
-        network_cfg.input_states += get_space_size(network_cfg.input_actions)
+        network_cfg.input_size += get_space_size(network_cfg.input_actions)
 
     # build base network of function approximator
     net = network_cfg.module(network_cfg)
@@ -21,10 +21,10 @@ def build_model(cfg: BaseRLCfg):
         net.input_actions = True
 
     # get output size to be used as output layer input
-    network_output_size = get_output_size(net, network_cfg.input_states)
+    network_output_size = get_output_size(net, network_cfg.input_size)
 
     def build_output_layer(layer_cfg):
-        return layer_cfg.class_type(device=cfg.device, input_states=network_output_size, cfg=layer_cfg)
+        return layer_cfg.class_type(device=cfg.device, input_size=network_output_size, cfg=layer_cfg)
 
     # build function approximator
     if isinstance(cfg, RLModelCfg):

{rlmodule-0.1.6.2 → rlmodule-0.1.7}/rlmodule/source/model.py

@@ -109,29 +109,47 @@ class Model(torch.nn.Module):
             >>> model.get_specification()
             {'rnn': {'sizes': [(1, 4, 64), (1, 4, 64)]}}
         """
-        if self._lstm:
-            return {
-                "rnn": {
-                    "sequence_length": self._net.sequence_length,
-                    "sizes": [
-                        (
-                            self._net.num_layers,
-                            self._net.num_envs,
-                            self._net.hidden_size,
-                        ),  # hidden states (D ∗ num_layers, N, Hout)
-                        (self._net.num_layers, self._net.num_envs, self._net.hidden_size),
-                    ],
+        if self._lstm or self._rnn:
+            num_layers = self._net.num_layers
+            num_envs = self._net.num_envs
+            hidden_size = self._net.hidden_size
+            # ``num_envs`` is only used here to size the SKRL-side hidden-state
+            # buffer; it MUST match the trainer's vector-env count. If it's
+            # misconfigured (<= 0 or inconsistent), SKRL silently allocates a
+            # wrong-sized tensor and the mismatch only surfaces later inside
+            # the LSTM/RNN forward as an opaque shape error. Catch it here.
+            if num_envs is None or num_envs <= 0:
+                raise ValueError(
+                    f"RnnBaseCfg.num_envs must be a positive integer matching the trainer's "
+                    f"vector-env count; got {num_envs!r}."
+                )
+            if num_layers is None or num_layers <= 0:
+                raise ValueError(
+                    f"RnnBaseCfg.num_layers must be a positive integer; got {num_layers!r}."
+                )
+            if hidden_size is None or hidden_size <= 0:
+                raise ValueError(
+                    f"RnnBaseCfg.hidden_size must be a positive integer; got {hidden_size!r}."
+                )
+
+            size = (num_layers, num_envs, hidden_size)
+            if self._lstm:
+                return {
+                    "rnn": {
+                        "sequence_length": self._net.sequence_length,
+                        # hidden states (D * num_layers, N, Hout)
+                        # cell states   (D * num_layers, N, Hcell)
+                        "sizes": [size, size],
+                    }
                 }
-            }  # cell states   (D ∗ num_layers, N, Hcell)
-        elif self._rnn:
             return {
                 "rnn": {
                     "sequence_length": self._net.sequence_length,
-                    "sizes": [(self._net.num_layers, self._net.num_envs, self._net.hidden_size)],
+                    # hidden states (D * num_layers, N, Hout)
+                    "sizes": [size],
                 }
-            }  # hidden states (D ∗ num_layers, N, Hout)
-        else:
-            return {}
+            }
+        return {}
 
     def tensor_to_space(
         self, tensor: torch.Tensor, space: Union[gym.Space, gymnasium.Space], start: int = 0
@@ -196,27 +214,25 @@ class Model(torch.nn.Module):
 
     def random_act(
         self, inputs: Mapping[str, Union[torch.Tensor, Any]], role: str = ""
-    ) -> Tuple[torch.Tensor, None, Mapping[str, Union[torch.Tensor, Any]]]:
-        """Act randomly according to the action space
-
-        :param inputs: Model inputs. The most common keys are:
+    ) -> Tuple[torch.Tensor, Mapping[str, Union[torch.Tensor, Any]]]:
+        """Act randomly according to the action space.
 
-                       - ``"states"``: state of the environment used to make the decision
-                       - ``"taken_actions"``: actions taken by the policy for the given states
-        :type inputs: dict where the values are typically torch.Tensor
-        :param role: Role play by the model (default: ``""``)
-        :type role: str, optional
+        Returns a 2-tuple ``(actions, outputs_dict)`` matching the
+        SKRL 2.x ``model.act`` contract.
+        """
 
-        :raises NotImplementedError: Unsupported action space
+        # SKRL 2.x agents pass the actor input under ``observations``;
+        # fall back to ``states`` when a model is invoked directly.
+        anchor = inputs.get("observations")
+        if anchor is None:
+            anchor = inputs["states"]
+        batch_size = anchor.shape[0]
 
-        :return: Model output. The first component is the action to be taken by the agent
-        :rtype: tuple of torch.Tensor, None, and dict
-        """
         # discrete action space (Discrete)
         if issubclass(type(self.action_space), gym.spaces.Discrete) or issubclass(
             type(self.action_space), gymnasium.spaces.Discrete
         ):
-            return torch.randint(self.action_space.n, (inputs["states"].shape[0], 1), device=self.device), None, {}
+            return torch.randint(self.action_space.n, (batch_size, 1), device=self.device), {}
         # continuous action space (Box)
         elif issubclass(type(self.action_space), gym.spaces.Box) or issubclass(
             type(self.action_space), gymnasium.spaces.Box
@@ -228,8 +244,7 @@ class Model(torch.nn.Module):
                 )
 
             return (
-                self._random_distribution.sample(sample_shape=(inputs["states"].shape[0], self.num_actions)),
-                None,
+                self._random_distribution.sample(sample_shape=(batch_size, self.num_actions)),
                 {},
             )
         else:
@@ -372,22 +387,18 @@ class Model(torch.nn.Module):
         """
         return self.forward(inputs, role)
 
-    def set_mode(self, mode: str) -> None:
-        """Set the model mode (training or evaluation)
+    def enable_training_mode(self, enabled: bool = True) -> None:
+        """Enable or disable training mode.
 
-        :param mode: Mode: ``"train"`` for training or ``"eval"`` for evaluation.
-            See `torch.nn.Module.train
-            <https://pytorch.org/docs/stable/generated/torch.nn.Module.html#torch.nn.Module.train>`_
-        :type mode: str
+        SKRL 2.x agents call this from
+        ``Agent.enable_models_training_mode`` to toggle the underlying
+        :py:meth:`torch.nn.Module.train` flag.
 
-        :raises ValueError: If the mode is not ``"train"`` or ``"eval"``
+        :param enabled: Set training mode if ``True``, evaluation mode if
+            ``False`` (default: ``True``).
         """
-        if mode == "train":
-            self.train(True)
-        elif mode == "eval":
-            self.train(False)
-        else:
-            raise ValueError("Invalid mode. Use 'train' for training or 'eval' for evaluation")
+
+        self.train(enabled)
 
     def save(self, path: str, state_dict: Optional[dict] = None) -> None:
         """Save the model to the specified path