rlmodule 0.1.0__tar.gz → 0.1.5__tar.gz

rlmodule-0.1.5/LICENSE

@@ -0,0 +1,30 @@
+Copyright (c) 2024 by Fabrica AI Corp.
+
+All rights reserved.
+
+SPDX-License-Identifier: BSD-3-Clause
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice,
+   this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors
+   may be used to endorse or promote products derived from this software without
+   specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

{rlmodule-0.1.0 → rlmodule-0.1.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rlmodule
-Version: 0.1.0
+Version: 0.1.5
 Summary: Flexible reinforcement learning models instantiators library
 Author: Lopatovsky
 Author-email: lopatovsky@gmail.com

{rlmodule-0.1.0 → rlmodule-0.1.5}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "rlmodule"
-version = "0.1.0"
+version = "0.1.5"
 description = "Flexible reinforcement learning models instantiators library"
 readme = "README.md"
 requires-python = ">=3.6"

{rlmodule-0.1.0 → rlmodule-0.1.5}/rlmodule/__init__.py

@@ -1,4 +1,4 @@
 # TODO this is not very good way
 from .source.rlmodel import RLModel, SharedRLModel # noqa: F401, F403
 from .source.output_layer import GaussianLayer, DeterministicLayer # noqa: F401, F403
-from .source.modules import MLP # noqa: F401, F403
+from .source.network import MLP # noqa: F401, F403

rlmodule-0.1.5/rlmodule/skrl/torch/__init__.py

@@ -0,0 +1,9 @@
+__all__ = [
+    "build_model"
+    "BaseRLCfg",
+    "RLModelCfg",
+    "SeparatedRLModelCfg",
+    "SharedRLModelCfg"
+]
+from rlmodule.source.rlmodel_cfg import BaseRLCfg, RLModelCfg, SeparatedRLModelCfg, SharedRLModelCfg
+from rlmodule.source.build import build_model

{rlmodule-0.1.0/rlmodule/skrl → rlmodule-0.1.5/rlmodule/skrl/torch}/examples/gymnasium/skrl_gymnasium_pendulum.py

@@ -2,17 +2,10 @@ import gymnasium as gym
 
 import torch.nn as nn
 
-from rlmodule.source.model_cfg import SharedRLModelCfg
-from rlmodule.source.modules import RnnMlpWithForwardedInput
-from rlmodule.source.modules_cfg import LstmCfg, MlpCfg, RnnMlpCfg
-from rlmodule.source.output_layer_cfg import DeterministicLayerCfg, GaussianLayerCfg
-from rlmodule.source.model_builder import build_model
-
-# example how this could be called:
-# from rlmodule.skrl.torch.model import SharedRLModelCfg
-# from rlmodule.skrl.torch.module import LstmCfg
-# from rlmodule.skrl.torch.output_layer import DeterministicLayerCfg, GaussianLayerCfg
-# from rlmodule.skrl.torch import build_model
+from rlmodule.skrl.torch import build_model, SharedRLModelCfg, RLModelCfg, SeparatedRLModelCfg
+from rlmodule.skrl.torch.network import MlpCfg   
+from rlmodule.skrl.torch.output_layer import DeterministicLayerCfg, GaussianLayerCfg
+
 
 # import the skrl components to build the RL system
 from skrl.agents.torch.ppo import PPO, PPO_RNN, PPO_DEFAULT_CONFIG
@@ -38,7 +31,7 @@ set_seed()  # e.g. `set_seed(42)` for fixed seed
 # (Done) Move things logically
 # (Done) Handle input shapes better?
 # (Done except Cnn) Convert all rest modules to use Configclass
-# Rearrange library in the way it can be used for other rllibs and for torch/jax?
+# (Done) Rearrange library in the way it can be used for other rllibs and for torch/jax?
 # Create examples - mlp, rnn,gru,lstm, Lstmmlp, shared - separate , custom net by fcion
 #                 - examples from modules make in class header not function 
 # WRITE README tutorial
@@ -47,6 +40,8 @@ set_seed()  # e.g. `set_seed(42)` for fixed seed
 # extensive comments
 # Launch new version to pip
 # Import new version in Isaac-lab
+# Change structure of source
+# lazy linear? what is it ?
 
  
 
@@ -61,7 +56,7 @@ set_seed()  # e.g. `set_seed(42)` for fixed seed
 #         ),
 #         mlp = MlpCfg(
 #             hidden_units = [64, 64, 64],
-#             activations = [nn.ReLU(), nn.ReLU(), nn.ReLU()],
+#             activations = [nn.ReLU, nn.ReLU, nn.ReLU],
 #         ),
 #     )
 #     return RnnMlp(cfg)
@@ -72,7 +67,7 @@ def get_shared_model(env):
     net_cfg = MlpCfg(
         input_size = env.observation_space,
         hidden_units = [64, 64, 64],
-        activations = [nn.ReLU(), nn.ReLU(), nn.ReLU()],
+        activations = [nn.ReLU, nn.ReLU, nn.ReLU],
     )
 
     # 2
@@ -95,7 +90,7 @@ def get_shared_model(env):
     #     ),
     #     mlp = MlpCfg(
     #         hidden_units = [64, 64, 64],
-    #         activations = [nn.ReLU(), nn.ReLU(), nn.ReLU()],
+    #         activations = [nn.ReLU, nn.ReLU, nn.ReLU],
     #     ),
     # )
 
@@ -111,7 +106,7 @@ def get_shared_model(env):
     #     ),
     #     mlp = MlpCfg(
     #         hidden_units = [64, 64, 64],
-    #         activations = [nn.ReLU(), nn.ReLU(), nn.ReLU()],
+    #         activations = [nn.ReLU, nn.ReLU, nn.ReLU],
     #     ),
     # )
 
@@ -167,40 +162,35 @@ def get_shared_model(env):
     return {'policy': model, 'value': model}
 
 # TODO Update separate model to configclass
-# def get_separate_model(env):
-#     # instantiate the agent's models (function approximators).
-#     params = {'input_size': env.observation_space, 
-#               'hidden_units': [64, 64, 64], 
-#               'activations': ['relu', 'relu', 'relu']
-#               }
-#     net = MLP(params)
-
-#     policy_model = RLModel(
-#         network=net,
-#         device= device,
-#         output_layer = GaussianLayer(
-#                 input_size=64,  # TODO
-#                 output_size=env.action_space,
-#                 output_activation=nn.Tanh(),
-#                 clip_actions=False,
-#                 clip_log_std=True,
-#                 min_log_std=-1.2,
-#                 max_log_std=2,
-#                 initial_log_std=0.2,
-#             ),
-#     )
+def get_separated_model(env):
+    net_cfg = MlpCfg(
+        input_size = env.observation_space,
+        hidden_units = [64, 64, 64],
+        activations = [nn.ReLU, nn.ReLU, nn.ReLU],
+    )
 
-#     value_model = RLModel(
-#         network=net,
-#         device= device,
-#         output_layer = DeterministicLayer(
-#                 input_size=64,
-#                 output_size=1,
-#                 output_activation=nn.Identity(),
-#             ),
-#     )
+    policy_model = build_model( 
+        RLModelCfg(
+            network= net_cfg,
+            device= device,
+            output_layer= GaussianLayerCfg(
+                    output_size=env.action_space,
+                    min_log_std=-1.2,
+                    max_log_std=2,
+                    initial_log_std=0.2,
+                ),
+        )
+    )
+
+    value_model = build_model( 
+        RLModelCfg(
+            network= net_cfg,
+            device= device,
+            output_layer= DeterministicLayerCfg(),
+        )
+    )
 
-#     return {'policy': policy_model, 'value': value_model}
+    return {'policy': policy_model, 'value': value_model}
 
 
 # load and wrap the gymnasium environment.
@@ -221,6 +211,7 @@ device = env.device
 memory = RandomMemory(memory_size=1024, num_envs=env.num_envs, device=device)
 
 models = get_shared_model(env)
+#models = get_separated_model(env)
 
 print(models)
 

rlmodule-0.1.5/rlmodule/skrl/torch/network/__init__.py

@@ -0,0 +1,8 @@
+__all__ = [
+    # modules:
+    "MLP", "RNN", "GRU", "LSTM", "RnnBase", "RnnMlp",
+    # configs:
+    "NetworkCfg", "MlpCfg", "RnnBaseCfg", "RnnCfg", "GruCfg", "LstmCfg", "RnnMlpCfg"
+]
+from rlmodule.source.network import MLP, RNN, GRU, LSTM, RnnBase, RnnMlp
+from rlmodule.source.network_cfg import NetworkCfg, MlpCfg, RnnBaseCfg, RnnCfg, GruCfg, LstmCfg, RnnMlpCfg

rlmodule-0.1.5/rlmodule/skrl/torch/output_layer/__init__.py

@@ -0,0 +1,5 @@
+__all__ = [
+    "DeterministicLayerCfg",
+    "GaussianLayerCfg"
+]
+from rlmodule.source.output_layer_cfg import DeterministicLayerCfg, GaussianLayerCfg

rlmodule-0.1.0/rlmodule/source/model_builder.py → rlmodule-0.1.5/rlmodule/source/build.py

@@ -1,4 +1,4 @@
-from rlmodule.source.model_cfg import BaseRLCfg, RLModelCfg, SharedRLModelCfg
+from rlmodule.source.rlmodel_cfg import BaseRLCfg, RLModelCfg, SharedRLModelCfg
 from rlmodule.source.utils import get_output_size
 
 #TODO this, or inside RLModel class that have a config passed

rlmodule-0.1.0/rlmodule/source/modules.py → rlmodule-0.1.5/rlmodule/source/network.py

@@ -26,13 +26,13 @@ class MLP(nn.Module):
         # input layer
         layers = [
             nn.Linear(cfg.input_size, cfg.hidden_units[0]),
-            cfg.activations[0],
+            cfg.activations[0](),
         ]
 
         # hidden layers
         for i in range(len(cfg.hidden_units) - 1):
             layers.append(nn.Linear(cfg.hidden_units[i], cfg.hidden_units[i + 1]))
-            layers.append(cfg.activations[i + 1])
+            layers.append(cfg.activations[i + 1]())
 
         self.mlp = nn.Sequential(*layers)
 

rlmodule-0.1.0/rlmodule/source/modules_cfg.py → rlmodule-0.1.5/rlmodule/source/network_cfg.py

@@ -6,7 +6,7 @@ from dataclasses import MISSING
 from collections.abc import Callable
 from typing import List, Sequence, Union
 
-from rlmodule.source.modules import GRU, LSTM, MLP, RNN, RnnBase, RnnMlp
+from rlmodule.source.network import GRU, LSTM, MLP, RNN, RnnBase, RnnMlp
 
 # use isaac-lab native configclass if available to avoid it being declared twice
 try:
@@ -26,7 +26,7 @@ class MlpCfg(NetworkCfg):
     module: type[MLP] = MLP
 
     hidden_units: List[int] = MISSING
-    activations: List[nn.Module] = MISSING
+    activations: List[type[nn.Module]] = MISSING
     
 
 @configclass

{rlmodule-0.1.0 → rlmodule-0.1.5}/rlmodule/source/output_layer.py

@@ -68,7 +68,7 @@ class GaussianLayer(OutputLayer):
         
         self.net = nn.Sequential(
             nn.Linear(input_size,  cfg.output_size), 
-            cfg.output_activation
+            cfg.output_activation()
         )
 
         self.log_std_parameter = nn.Parameter(

{rlmodule-0.1.0 → rlmodule-0.1.5}/rlmodule/source/output_layer_cfg.py

@@ -23,7 +23,7 @@ class OutputLayerCfg:
 @configclass
 class GaussianLayerCfg(OutputLayerCfg):
     class_type: type[GaussianLayer] = GaussianLayer
-    output_activation: nn.Module = nn.Tanh()
+    output_activation: type[nn.Module] = nn.Tanh
 
     clip_log_std: bool = True
     min_log_std: float = -20

{rlmodule-0.1.0 → rlmodule-0.1.5}/rlmodule/source/rlmodel.py

@@ -78,13 +78,8 @@ class RLModel(Model):
         self._lstm = contains_rnn_module( self, nn.LSTM)
         self._rnn = contains_rnn_module( self, (nn.LSTM, nn.RNN, nn.GRU))
         
-        
-        print("!rnn: ", self._rnn)
-        print("!lstm: ", self._lstm)
-
-
-        print("!rnn: ", self._rnn)
-        print("!lstm: ", self._lstm)
+        print("is rnn: ", self._rnn)
+        print("is lstm: ", self._lstm)
 
         # TODO(ll) one liner:
         # self.net = nn.Sequential(network, self._get_num_units_by_shape(self.output_shape), torch.nn.Tanh())
@@ -120,13 +115,6 @@ class RLModel(Model):
         else:
             return {}
         
-    # TODO(ll) RNN & GRU has this simplier specification
-    # def get_specification(self):
-
-    #     # batch size (N) is the number of envs
-    #     return {"rnn": {"sequence_length": self.sequence_length,
-    #                     "sizes": [(self.num_layers, self.num_envs, self.hidden_size)]}}  # hidden states (D ∗ num_layers, N, Hout)
-        
     def get_entropy(self, role: str = "") -> torch.Tensor:
         """Compute and return the entropy of the model
 
@@ -163,14 +151,16 @@ class RLModel(Model):
             inputs: Mapping[str, Union[torch.Tensor, Any]],
             role: str = "") -> Tuple[torch.Tensor, Union[torch.Tensor, None], Mapping[str, Union[torch.Tensor, Any]]]:
         
+        states = inputs["states"]
+        
         # TODO(ll) once shared model is design, you may rethink this to do by call some function pointer or whatever.
         # or consider inheritance GaussianModelRNN
         # TODO(ll) if one of the models is not rnn ... and PPO_RNN is called what happens?
         if self._rnn:
-            output, output_dict = self.net(self._get_all_inputs(inputs), inputs.get('terminated', None), inputs['rnn'])
+            output, output_dict = self.net(states, inputs.get('terminated', None), inputs['rnn'])
         else:
 
-            output = self.net(self._get_all_inputs(inputs))
+            output = self.net(states)
             output_dict = {}
 
         # TODO(ll) passing by independetly as together sequential has just one input in forward
@@ -188,7 +178,7 @@ class RLModel(Model):
 
         # TODO(ll) output scale removed .. check that tanh where is
         # return output * self.instantiator_output_scale, self.log_std_parameter, {}
-        return output, self.log_std_parameter, output_dict
+        # return output, self.log_std_parameter, output_dict
 
 class SharedRLModel(Model):
     def __init__(self, 
@@ -253,8 +243,8 @@ class SharedRLModel(Model):
         self._lstm = contains_rnn_module( self, nn.LSTM)
         self._rnn = contains_rnn_module( self, (nn.LSTM, nn.RNN, nn.GRU))
         
-        print("!rnn: ", self._rnn)
-        print("!lstm: ", self._lstm)
+        print("is rnn: ", self._rnn)
+        print("is lstm: ", self._lstm)
 
         # caching for shared forward pass for policy and value roles
         self._cached_states = None

rlmodule-0.1.0/rlmodule/source/model_cfg.py → rlmodule-0.1.5/rlmodule/source/rlmodel_cfg.py

@@ -30,4 +30,11 @@ class RLModelCfg(BaseRLCfg):
 class SharedRLModelCfg(BaseRLCfg):
     class_type: type[SharedRLModel] = SharedRLModel
     policy_output_layer: OutputLayerCfg = MISSING
-    value_output_layer: OutputLayerCfg = MISSING
+    value_output_layer: OutputLayerCfg = MISSING
+
+
+@configclass
+class SeparatedRLModelCfg:
+    """Convenience class to hold two separate instances of RLModelCfg. One for policy and one for value model."""
+    policy: RLModelCfg = MISSING
+    value: RLModelCfg = MISSING

{rlmodule-0.1.0 → rlmodule-0.1.5}/rlmodule.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rlmodule
-Version: 0.1.0
+Version: 0.1.5
 Summary: Flexible reinforcement learning models instantiators library
 Author: Lopatovsky
 Author-email: lopatovsky@gmail.com

{rlmodule-0.1.0 → rlmodule-0.1.5}/rlmodule.egg-info/SOURCES.txt

@@ -8,18 +8,21 @@ rlmodule.egg-info/dependency_links.txt
 rlmodule.egg-info/requires.txt
 rlmodule.egg-info/top_level.txt
 rlmodule/skrl/__init__.py
-rlmodule/skrl/examples/gymnasium/skrl_gymnasium_pendulum.py
-rlmodule/skrl/examples/gymnasium/skrl_gymnasium_pendulum_no_module.py
+rlmodule/skrl/torch/__init__.py
+rlmodule/skrl/torch/examples/gymnasium/skrl_gymnasium_pendulum.py
+rlmodule/skrl/torch/examples/gymnasium/skrl_gymnasium_pendulum_no_module.py
+rlmodule/skrl/torch/network/__init__.py
+rlmodule/skrl/torch/output_layer/__init__.py
 rlmodule/source/__init__.py
+rlmodule/source/build.py
 rlmodule/source/deprecated.py
 rlmodule/source/model.py
-rlmodule/source/model_builder.py
-rlmodule/source/model_cfg.py
-rlmodule/source/modules.py
-rlmodule/source/modules_cfg.py
+rlmodule/source/network.py
+rlmodule/source/network_cfg.py
 rlmodule/source/output_layer.py
 rlmodule/source/output_layer_cfg.py
 rlmodule/source/rlmodel.py
+rlmodule/source/rlmodel_cfg.py
 rlmodule/source/utils.py
 rlmodule/source/nvidia_utils/__init__.py
 rlmodule/source/nvidia_utils/configclass.py

rlmodule-0.1.0/LICENSE

@@ -1,21 +0,0 @@
-MIT License
-
-Copyright (c) 2024 FabricaAI
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.