rlmodule 0.1.6.3__tar.gz → 0.2.0__tar.gz

{rlmodule-0.1.6.3/rlmodule.egg-info → rlmodule-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: rlmodule
-Version: 0.1.6.3
+Version: 0.2.0
 Summary: Flexible reinforcement learning models instantiators library
 Author: Lopatovsky
 Author-email: lopatovsky@gmail.com
@@ -18,18 +18,22 @@ Classifier: Intended Audience :: Science/Research
 Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Classifier: Operating System :: OS Independent
-Requires-Python: >=3.6
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: gym
 Requires-Dist: gymnasium==0.29.0
 Requires-Dist: torch>=1.9
-Requires-Dist: skrl>=1.3.0
+Requires-Dist: skrl>=2.1
 Provides-Extra: skrl
-Requires-Dist: skrl>=1.3.0; extra == "skrl"
+Requires-Dist: skrl>=2.1; extra == "skrl"
 Provides-Extra: all
-Requires-Dist: skrl>=1.3.0; extra == "all"
+Requires-Dist: skrl>=2.1; extra == "all"
+Dynamic: license-file
 
 # rlmodule
 Flexible reinforcement learning models instantiators library
@@ -77,6 +81,14 @@ Note: sometimes installation may fail, if there is a run/ dir present, you may n
 rm -rf runs
 ```
 
+### Deep policies: LayerNorm and Residual MLPs
+
+`rlmodule` supports per-layer LayerNorm on `MlpCfg` and a SimBa-style residual
+MLP (`ResidualMlpCfg`). These work for both separated and shared
+architectures, with independent per-head granularity via the new
+`OutputLayerCfg.head` field. See [docs/layer_norm_and_residual.md](docs/layer_norm_and_residual.md)
+for motivation, references, and copy-pasteable cfg snippets.
+
 ### Run chosen example
 ```
 python3 rlmodule/skrl/torch/examples/gymnasium/pendulum_ppo_mlp_separate_model.py

{rlmodule-0.1.6.3 → rlmodule-0.2.0}/README.md

@@ -44,6 +44,14 @@ Note: sometimes installation may fail, if there is a run/ dir present, you may n
 rm -rf runs
 ```
 
+### Deep policies: LayerNorm and Residual MLPs
+
+`rlmodule` supports per-layer LayerNorm on `MlpCfg` and a SimBa-style residual
+MLP (`ResidualMlpCfg`). These work for both separated and shared
+architectures, with independent per-head granularity via the new
+`OutputLayerCfg.head` field. See [docs/layer_norm_and_residual.md](docs/layer_norm_and_residual.md)
+for motivation, references, and copy-pasteable cfg snippets.
+
 ### Run chosen example
 ```
 python3 rlmodule/skrl/torch/examples/gymnasium/pendulum_ppo_mlp_separate_model.py

{rlmodule-0.1.6.3 → rlmodule-0.2.0}/pyproject.toml

@@ -1,9 +1,13 @@
+[build-system]
+requires = ["setuptools>=61.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
 [project]
 name = "rlmodule"
-version = "0.1.6.3"
+version = "0.2.0"
 description = "Flexible reinforcement learning models instantiators library"
 readme = "README.md"
-requires-python = ">=3.6"
+requires-python = ">=3.10"
 license = {text = "MIT License"}
 authors = [
   {name = "Lopatovsky"},
@@ -20,24 +24,27 @@ classifiers = [
   "Topic :: Scientific/Engineering",
   "Topic :: Scientific/Engineering :: Artificial Intelligence",
   "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
   "Operating System :: OS Independent",
 ]
 dependencies = [
   "gym",
   "gymnasium==0.29.0",
   "torch>=1.9",
-  "skrl>=1.3.0",
+  "skrl>=2.1",
 ]
 
 [project.optional-dependencies]
 
 # TODO(ll) only examples should depend on skrl, not the main code.
 skrl = [
-  "skrl>=1.3.0",
+  "skrl>=2.1",
 ]
 
 all = [
-  "skrl>=1.3.0",
+  "skrl>=2.1",
 ]
 
 # urls
@@ -49,6 +56,10 @@ all = [
 "Say Thanks!" = "https://github.com/lopatovsky"
 "Source" = "https://github.com/FabricaAI/rlmodule"
 
+[tool.setuptools.packages.find]
+include = ["rlmodule*"]
+exclude = ["tests*", "docs*"]
+
 [tool.yapf]
 # run: yapf -p -m -i -r <folder>
 based_on_style = "pep8"
@@ -94,3 +105,9 @@ sections = [
 ]
 no_lines_before = "THIRDPARTY"
 skip = ["docs"]
+
+
+[tool.pytest.ini_options]
+markers = [
+  "slow: marks tests as slow (skip with: -m 'not slow')",
+]

{rlmodule-0.1.6.3 → rlmodule-0.2.0}/rlmodule/skrl/torch/__init__.py

@@ -1,3 +1,3 @@
-__all__ = ["build_modelBaseRLCfg", "RLModelCfg", "SeparatedRLModelCfg", "SharedRLModelCfg"]
+__all__ = ["build_model", "BaseRLCfg", "RLModelCfg", "SeparatedRLModelCfg", "SharedRLModelCfg"]
 from rlmodule.source.build import build_model  # noqa: F401
 from rlmodule.source.rlmodel_cfg import BaseRLCfg, RLModelCfg, SeparatedRLModelCfg, SharedRLModelCfg  # noqa: F401

rlmodule-0.2.0/rlmodule/skrl/torch/network/__init__.py

@@ -0,0 +1,63 @@
+__all__ = [
+    # modules:
+    "MLP",
+    "RNN",
+    "GRU",
+    "LSTM",
+    "RnnBase",
+    "RnnMlp",
+    "RnnMlpWithForwardedInput",
+    "ResidualMlp",
+    "CnnBackbone",
+    "Cnn3dBackbone",
+    "CnnMlp",
+    "CnnProprioMlp",
+    "GraphNetwork",
+    # configs:
+    "NetworkCfg",
+    "MlpCfg",
+    "ResidualMlpCfg",
+    "RnnBaseCfg",
+    "RnnCfg",
+    "GruCfg",
+    "LstmCfg",
+    "RnnMlpCfg",
+    "CnnCfg",
+    "CnnBackboneCfg",
+    "Cnn3dBackboneCfg",
+    "CnnMlpCfg",
+    "CnnProprioMlpCfg",
+    "GraphNetworkCfg",
+]
+from rlmodule.source.network import (  # noqa: F401
+    CNN,
+    GRU,
+    LSTM,
+    MLP,
+    RNN,
+    Cnn3dBackbone,
+    CnnBackbone,
+    CnnMlp,
+    CnnProprioMlp,
+    GraphNetwork,
+    ResidualMlp,
+    RnnBase,
+    RnnMlp,
+    RnnMlpWithForwardedInput,
+)
+from rlmodule.source.network_cfg import (  # noqa: F401
+    Cnn3dBackboneCfg,
+    CnnBackboneCfg,
+    CnnCfg,
+    CnnMlpCfg,
+    CnnProprioMlpCfg,
+    GraphNetworkCfg,
+    GruCfg,
+    LstmCfg,
+    MlpCfg,
+    NetworkCfg,
+    ResidualMlpCfg,
+    RnnBaseCfg,
+    RnnCfg,
+    RnnMlpCfg,
+)

{rlmodule-0.1.6.3 → rlmodule-0.2.0}/rlmodule/source/log_std.py

@@ -102,18 +102,24 @@ class CombinedLogStd(LogStd):
     def __init__(self, device: Union[str, torch.device], input_size: int, output_size: int, cfg):
         super().__init__(device, input_size, output_size, cfg)
 
-        self._modules = [
-            module_cfg.class_type(device, input_size, output_size, module_cfg) for module_cfg in cfg.combined_modules
-        ]
+        # NOTE: must NOT be ``self._modules`` — that name is ``nn.Module``'s
+        # internal child-module registry. Assigning a plain list there
+        # silently breaks parameter registration (``.parameters()`` /
+        # ``.to()`` / ``state_dict()``), so the sub-LogStd params never reach
+        # the optimizer and their std stays frozen. Use a registered
+        # ``nn.ModuleList`` under a distinct attribute instead.
+        self._sub_log_stds = nn.ModuleList(
+            [module_cfg.class_type(device, input_size, output_size, module_cfg) for module_cfg in cfg.combined_modules]
+        )
 
         if cfg.combination_constants is None:
-            self._combination_constants = list(repeat(1.0, len(self._modules)))
+            self._combination_constants = list(repeat(1.0, len(self._sub_log_stds)))
         else:
             self._combination_constants = cfg.combination_constants
-            if len(self._modules) != len(self._combination_constants):
+            if len(self._sub_log_stds) != len(self._combination_constants):
                 raise ValueError(
                     f"Number of combination constants {len(self._combination_constants)} does not match number of"
-                    f" modules {len(self._modules)}"
+                    f" modules {len(self._sub_log_stds)}"
                 )
 
         if cfg.combination_method == "max":
@@ -128,6 +134,7 @@ class CombinedLogStd(LogStd):
     def forward(self, input: torch.Tensor) -> torch.Tensor:
         """Compute standard deviation as a combination from multiple Std modules."""
         collected_stds = [
-            self._combination_constants[module_id] * module(input) for module_id, module in enumerate(self._modules)
+            self._combination_constants[module_id] * module(input)
+            for module_id, module in enumerate(self._sub_log_stds)
         ]
         return self.clip_std(reduce(self._combination_method, collected_stds))

{rlmodule-0.1.6.3 → rlmodule-0.2.0}/rlmodule/source/model.py

@@ -3,14 +3,17 @@ from typing import Any, Mapping, Optional, Tuple, Union
 import collections
 import gym
 import gymnasium
+import logging
 
 import torch
 import torch.nn as nn
 
-from rlmodule import logger
 from rlmodule.source.utils import contains_rnn_module
 
 
+logger = logging.getLogger("rlmodule")
+
+
 class Model(torch.nn.Module):
     def __init__(self, device: Union[str, torch.device], network: nn.Module) -> None:
         """Base class representing a function approximator"""
@@ -109,29 +112,47 @@ class Model(torch.nn.Module):
             >>> model.get_specification()
             {'rnn': {'sizes': [(1, 4, 64), (1, 4, 64)]}}
         """
-        if self._lstm:
-            return {
-                "rnn": {
-                    "sequence_length": self._net.sequence_length,
-                    "sizes": [
-                        (
-                            self._net.num_layers,
-                            self._net.num_envs,
-                            self._net.hidden_size,
-                        ),  # hidden states (D ∗ num_layers, N, Hout)
-                        (self._net.num_layers, self._net.num_envs, self._net.hidden_size),
-                    ],
+        if self._lstm or self._rnn:
+            num_layers = self._net.num_layers
+            num_envs = self._net.num_envs
+            hidden_size = self._net.hidden_size
+            # ``num_envs`` is only used here to size the SKRL-side hidden-state
+            # buffer; it MUST match the trainer's vector-env count. If it's
+            # misconfigured (<= 0 or inconsistent), SKRL silently allocates a
+            # wrong-sized tensor and the mismatch only surfaces later inside
+            # the LSTM/RNN forward as an opaque shape error. Catch it here.
+            if num_envs is None or num_envs <= 0:
+                raise ValueError(
+                    f"RnnBaseCfg.num_envs must be a positive integer matching the trainer's "
+                    f"vector-env count; got {num_envs!r}."
+                )
+            if num_layers is None or num_layers <= 0:
+                raise ValueError(
+                    f"RnnBaseCfg.num_layers must be a positive integer; got {num_layers!r}."
+                )
+            if hidden_size is None or hidden_size <= 0:
+                raise ValueError(
+                    f"RnnBaseCfg.hidden_size must be a positive integer; got {hidden_size!r}."
+                )
+
+            size = (num_layers, num_envs, hidden_size)
+            if self._lstm:
+                return {
+                    "rnn": {
+                        "sequence_length": self._net.sequence_length,
+                        # hidden states (D * num_layers, N, Hout)
+                        # cell states   (D * num_layers, N, Hcell)
+                        "sizes": [size, size],
+                    }
                 }
-            }  # cell states   (D ∗ num_layers, N, Hcell)
-        elif self._rnn:
             return {
                 "rnn": {
                     "sequence_length": self._net.sequence_length,
-                    "sizes": [(self._net.num_layers, self._net.num_envs, self._net.hidden_size)],
+                    # hidden states (D * num_layers, N, Hout)
+                    "sizes": [size],
                 }
-            }  # hidden states (D ∗ num_layers, N, Hout)
-        else:
-            return {}
+            }
+        return {}
 
     def tensor_to_space(
         self, tensor: torch.Tensor, space: Union[gym.Space, gymnasium.Space], start: int = 0
@@ -196,27 +217,25 @@ class Model(torch.nn.Module):
 
     def random_act(
         self, inputs: Mapping[str, Union[torch.Tensor, Any]], role: str = ""
-    ) -> Tuple[torch.Tensor, None, Mapping[str, Union[torch.Tensor, Any]]]:
-        """Act randomly according to the action space
+    ) -> Tuple[torch.Tensor, Mapping[str, Union[torch.Tensor, Any]]]:
+        """Act randomly according to the action space.
 
-        :param inputs: Model inputs. The most common keys are:
-
-                       - ``"states"``: state of the environment used to make the decision
-                       - ``"taken_actions"``: actions taken by the policy for the given states
-        :type inputs: dict where the values are typically torch.Tensor
-        :param role: Role play by the model (default: ``""``)
-        :type role: str, optional
+        Returns a 2-tuple ``(actions, outputs_dict)`` matching the
+        SKRL 2.x ``model.act`` contract.
+        """
 
-        :raises NotImplementedError: Unsupported action space
+        # SKRL 2.x agents pass the actor input under ``observations``;
+        # fall back to ``states`` when a model is invoked directly.
+        anchor = inputs.get("observations")
+        if anchor is None:
+            anchor = inputs["states"]
+        batch_size = anchor.shape[0]
 
-        :return: Model output. The first component is the action to be taken by the agent
-        :rtype: tuple of torch.Tensor, None, and dict
-        """
         # discrete action space (Discrete)
         if issubclass(type(self.action_space), gym.spaces.Discrete) or issubclass(
             type(self.action_space), gymnasium.spaces.Discrete
         ):
-            return torch.randint(self.action_space.n, (inputs["states"].shape[0], 1), device=self.device), None, {}
+            return torch.randint(self.action_space.n, (batch_size, 1), device=self.device), {}
         # continuous action space (Box)
         elif issubclass(type(self.action_space), gym.spaces.Box) or issubclass(
             type(self.action_space), gymnasium.spaces.Box
@@ -228,8 +247,7 @@ class Model(torch.nn.Module):
                 )
 
             return (
-                self._random_distribution.sample(sample_shape=(inputs["states"].shape[0], self.num_actions)),
-                None,
+                self._random_distribution.sample(sample_shape=(batch_size, self.num_actions)),
                 {},
             )
         else:
@@ -372,22 +390,18 @@ class Model(torch.nn.Module):
         """
         return self.forward(inputs, role)
 
-    def set_mode(self, mode: str) -> None:
-        """Set the model mode (training or evaluation)
+    def enable_training_mode(self, enabled: bool = True) -> None:
+        """Enable or disable training mode.
 
-        :param mode: Mode: ``"train"`` for training or ``"eval"`` for evaluation.
-            See `torch.nn.Module.train
-            <https://pytorch.org/docs/stable/generated/torch.nn.Module.html#torch.nn.Module.train>`_
-        :type mode: str
+        SKRL 2.x agents call this from
+        ``Agent.enable_models_training_mode`` to toggle the underlying
+        :py:meth:`torch.nn.Module.train` flag.
 
-        :raises ValueError: If the mode is not ``"train"`` or ``"eval"``
+        :param enabled: Set training mode if ``True``, evaluation mode if
+            ``False`` (default: ``True``).
         """
-        if mode == "train":
-            self.train(True)
-        elif mode == "eval":
-            self.train(False)
-        else:
-            raise ValueError("Invalid mode. Use 'train' for training or 'eval' for evaluation")
+
+        self.train(enabled)
 
     def save(self, path: str, state_dict: Optional[dict] = None) -> None:
         """Save the model to the specified path