rlmodule 0.0.5__tar.gz → 0.1.5__tar.gz

rlmodule-0.1.5/LICENSE

@@ -0,0 +1,30 @@
+Copyright (c) 2024 by Fabrica AI Corp.
+
+All rights reserved.
+
+SPDX-License-Identifier: BSD-3-Clause
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice,
+   this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors
+   may be used to endorse or promote products derived from this software without
+   specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

{rlmodule-0.0.5 → rlmodule-0.1.5}/PKG-INFO

@@ -1,8 +1,9 @@
 Metadata-Version: 2.1
 Name: rlmodule
-Version: 0.0.5
+Version: 0.1.5
 Summary: Flexible reinforcement learning models instantiators library
 Author: Lopatovsky
+Author-email: lopatovsky@gmail.com
 Maintainer: Lopatovsky
 License: MIT License
 Project-URL: Homepage, https://github.com/FabricaAI/rlmodule
@@ -24,11 +25,55 @@ License-File: LICENSE
 Requires-Dist: gym
 Requires-Dist: gymnasium
 Requires-Dist: torch>=1.9
-Requires-Dist: skrl>=1.2.0
+Requires-Dist: skrl>=1.3.0
 Provides-Extra: skrl
-Requires-Dist: skrl>=1.2.0; extra == "skrl"
+Requires-Dist: skrl>=1.3.0; extra == "skrl"
 Provides-Extra: all
-Requires-Dist: skrl>=1.2.0; extra == "all"
+Requires-Dist: skrl>=1.3.0; extra == "all"
 
 # rlmodule
 Flexible reinforcement learning models instantiators library
+
+
+## How to run
+
+### Install rlmodule from local code
+
+- Make sure you are in base rlmodule dit.
+
+- Start virtual env.
+```
+python3 -m venv venv
+source venv/bin/activate
+```
+- Install library from local code 
+```
+pip install -e .
+```
+Note: sometimes installation may fail, if there is a run/ dir present, you may need to remove it (TODO: fix)
+```
+rm -rf runs
+```
+### Run chosen example
+```
+python3 rlmodule/skrl/examples/gymnasium/skrl_gymnasium_pendulum.py
+```
+
+
+## Update new version to PIP
+
+```
+pip install build twine
+```
+
+```
+rm -rf runs
+python -m build
+```
+
+```
+twine upload dist/*
+```
+
+
+

rlmodule-0.1.5/README.md

@@ -0,0 +1,46 @@
+# rlmodule
+Flexible reinforcement learning models instantiators library
+
+
+## How to run
+
+### Install rlmodule from local code
+
+- Make sure you are in base rlmodule dit.
+
+- Start virtual env.
+```
+python3 -m venv venv
+source venv/bin/activate
+```
+- Install library from local code 
+```
+pip install -e .
+```
+Note: sometimes installation may fail, if there is a run/ dir present, you may need to remove it (TODO: fix)
+```
+rm -rf runs
+```
+### Run chosen example
+```
+python3 rlmodule/skrl/examples/gymnasium/skrl_gymnasium_pendulum.py
+```
+
+
+## Update new version to PIP
+
+```
+pip install build twine
+```
+
+```
+rm -rf runs
+python -m build
+```
+
+```
+twine upload dist/*
+```
+
+
+

{rlmodule-0.0.5 → rlmodule-0.1.5}/pyproject.toml

@@ -1,16 +1,18 @@
 [project]
 name = "rlmodule"
-version = "0.0.5"
+version = "0.1.5"
 description = "Flexible reinforcement learning models instantiators library"
 readme = "README.md"
 requires-python = ">=3.6"
 license = {text = "MIT License"}
 authors = [
-  {name = "Lopatovsky"},
+  {name = "Lopatovsky"}, 
+  {email = "lopatovsky@gmail.com"},
 ]
 maintainers = [
   {name = "Lopatovsky"},
 ]
+
 keywords = ["reinforcement-learning", "machine-learning", "reinforcement", "machine", "learning", "rl", "rl-models", "rl-modules", "models", "modules"]
 classifiers = [
   "License :: OSI Approved :: MIT License",
@@ -24,18 +26,18 @@ dependencies = [
   "gym",
   "gymnasium",
   "torch>=1.9",
-  "skrl>=1.2.0",
+  "skrl>=1.3.0",
 ]
 
 [project.optional-dependencies]
 
 # TODO(ll) only examples should depend on skrl, not the main code.
 skrl = [
-  "skrl>=1.2.0",
+  "skrl>=1.3.0",
 ]
 
 all = [
-  "skrl>=1.2.0",
+  "skrl>=1.3.0",
 ]
 
 # urls

rlmodule-0.1.5/rlmodule/__init__.py

@@ -0,0 +1,4 @@
+# TODO this is not very good way
+from .source.rlmodel import RLModel, SharedRLModel # noqa: F401, F403
+from .source.output_layer import GaussianLayer, DeterministicLayer # noqa: F401, F403
+from .source.network import MLP # noqa: F401, F403

rlmodule-0.1.5/rlmodule/skrl/torch/__init__.py

@@ -0,0 +1,9 @@
+__all__ = [
+    "build_model"
+    "BaseRLCfg",
+    "RLModelCfg",
+    "SeparatedRLModelCfg",
+    "SharedRLModelCfg"
+]
+from rlmodule.source.rlmodel_cfg import BaseRLCfg, RLModelCfg, SeparatedRLModelCfg, SharedRLModelCfg
+from rlmodule.source.build import build_model

rlmodule-0.1.5/rlmodule/skrl/torch/examples/gymnasium/skrl_gymnasium_pendulum.py

@@ -0,0 +1,267 @@
+import gymnasium as gym
+
+import torch.nn as nn
+
+from rlmodule.skrl.torch import build_model, SharedRLModelCfg, RLModelCfg, SeparatedRLModelCfg
+from rlmodule.skrl.torch.network import MlpCfg   
+from rlmodule.skrl.torch.output_layer import DeterministicLayerCfg, GaussianLayerCfg
+
+
+# import the skrl components to build the RL system
+from skrl.agents.torch.ppo import PPO, PPO_RNN, PPO_DEFAULT_CONFIG
+
+from skrl.envs.wrappers.torch.gymnasium_envs import GymnasiumWrapper
+#from skrl.envs.wrappers.torch import wrap_env
+from skrl.memories.torch import RandomMemory
+from skrl.resources.preprocessors.torch import RunningStandardScaler
+from skrl.resources.schedulers.torch import KLAdaptiveRL
+from skrl.trainers.torch import SequentialTrainer
+from skrl.utils import set_seed
+from skrl.envs.wrappers.torch.gymnasium_envs import GymnasiumWrapper
+
+# todo set seed
+# seed for reproducibility
+set_seed()  # e.g. `set_seed(42)` for fixed seed
+
+#TODO next
+# (Done) check how to do own @configclass .. or use one from isaac by default (print something to know version)
+# (Done) RNN code modify to use data class
+# (Done) RnnMLP
+# (Done) Custom function Network
+# (Done) Move things logically
+# (Done) Handle input shapes better?
+# (Done except Cnn) Convert all rest modules to use Configclass
+# (Done) Rearrange library in the way it can be used for other rllibs and for torch/jax?
+# Create examples - mlp, rnn,gru,lstm, Lstmmlp, shared - separate , custom net by fcion
+#                 - examples from modules make in class header not function 
+# WRITE README tutorial
+# Run & fix pre-commit
+# annotate cfgs in modules - why doesn't work
+# extensive comments
+# Launch new version to pip
+# Import new version in Isaac-lab
+# Change structure of source
+# lazy linear? what is it ?
+
+ 
+
+# def example_module(cfg):
+#     cfg = RnnMlpCfg(
+#         input_size = env.observation_space,
+#         rnn = LstmCfg(
+#             num_envs = env.num_envs,
+#             num_layers = 1,
+#             hidden_size = 32,
+#             sequence_length = 16,
+#         ),
+#         mlp = MlpCfg(
+#             hidden_units = [64, 64, 64],
+#             activations = [nn.ReLU, nn.ReLU, nn.ReLU],
+#         ),
+#     )
+#     return RnnMlp(cfg)
+
+def get_shared_model(env):
+    # instantiate the agent's models (function approximators).
+
+    net_cfg = MlpCfg(
+        input_size = env.observation_space,
+        hidden_units = [64, 64, 64],
+        activations = [nn.ReLU, nn.ReLU, nn.ReLU],
+    )
+
+    # 2
+    # net_cfg = LstmCfg(
+    #     input_size = env.observation_space,
+    #     num_envs = env.num_envs,
+    #     num_layers = 1,
+    #     hidden_size = 32,
+    #     sequence_length = 16,
+    # )
+
+    # 3
+    # net_cfg = RnnMlpCfg(
+    #     input_size = env.observation_space,
+    #     rnn = LstmCfg(
+    #         num_envs = env.num_envs,
+    #         num_layers = 1,
+    #         hidden_size = 32,
+    #         sequence_length = 16,
+    #     ),
+    #     mlp = MlpCfg(
+    #         hidden_units = [64, 64, 64],
+    #         activations = [nn.ReLU, nn.ReLU, nn.ReLU],
+    #     ),
+    # )
+
+    # 3.5 
+    # net_cfg = RnnMlpCfg(
+    #     input_size = env.observation_space,
+    #     module = RnnMlpWithForwardedInput,
+    #     rnn = LstmCfg(
+    #         num_envs = env.num_envs,
+    #         num_layers = 1,
+    #         hidden_size = 32,
+    #         sequence_length = 16,
+    #     ),
+    #     mlp = MlpCfg(
+    #         hidden_units = [64, 64, 64],
+    #         activations = [nn.ReLU, nn.ReLU, nn.ReLU],
+    #     ),
+    # )
+
+    # 4
+    # net_cfg = NetworkCfg( input_size = env.observation_space,
+    #                       module = example_module)
+
+    # variant IV - all config
+    # net also config (optional)?
+    model = build_model( 
+        SharedRLModelCfg(
+            network= net_cfg,
+            device= device,
+            policy_output_layer= GaussianLayerCfg(
+                    output_size=env.action_space,
+                    min_log_std=-1.2,
+                    max_log_std=2,
+                    initial_log_std=0.2,
+                ),
+            value_output_layer= DeterministicLayerCfg(),
+        )
+    )
+    # RLCfg(
+    #     Model
+    #     AgentCfg
+    #     MemoryCfg
+    # )
+
+    # build_model(cfg.model)
+
+    # cfg.agent.algoclass(
+    #     cfg.agent.algo_config
+    #     cfg.memory
+    # )
+
+    # SharedRLModelCfg(RLModelCfg){
+    #     class = SharedRLModel
+
+    # }
+
+    # net
+    # output layers.class()
+
+    # cfg.model.class(  net, output_layers )
+
+    #configclass -> dataclass - easy (Ron said)
+
+
+    # variant V:
+    # ideas?
+
+
+    return {'policy': model, 'value': model}
+
+# TODO Update separate model to configclass
+def get_separated_model(env):
+    net_cfg = MlpCfg(
+        input_size = env.observation_space,
+        hidden_units = [64, 64, 64],
+        activations = [nn.ReLU, nn.ReLU, nn.ReLU],
+    )
+
+    policy_model = build_model( 
+        RLModelCfg(
+            network= net_cfg,
+            device= device,
+            output_layer= GaussianLayerCfg(
+                    output_size=env.action_space,
+                    min_log_std=-1.2,
+                    max_log_std=2,
+                    initial_log_std=0.2,
+                ),
+        )
+    )
+
+    value_model = build_model( 
+        RLModelCfg(
+            network= net_cfg,
+            device= device,
+            output_layer= DeterministicLayerCfg(),
+        )
+    )
+
+    return {'policy': policy_model, 'value': value_model}
+
+
+# load and wrap the gymnasium environment.
+# note: the environment version may change depending on the gymnasium version
+try:
+    env = gym.vector.make("Pendulum-v1", num_envs=4, asynchronous=False)
+except (gym.error.DeprecatedEnv, gym.error.VersionNotFound) as e:
+    env_id = [spec for spec in gym.envs.registry if spec.startswith("Pendulum-v")][0]
+    print("Pendulum-v1 not found. Trying {}".format(env_id))
+    env = gym.vector.make(env_id, num_envs=4, asynchronous=False)
+
+env = GymnasiumWrapper(env)
+
+device = env.device
+
+
+# instantiate a memory as rollout buffer (any memory can be used for this)
+memory = RandomMemory(memory_size=1024, num_envs=env.num_envs, device=device)
+
+models = get_shared_model(env)
+#models = get_separated_model(env)
+
+print(models)
+
+
+# configure and instantiate the agent (visit its documentation to see all the options)
+# https://skrl.readthedocs.io/en/latest/api/agents/ppo.html#configuration-and-hyperparameters
+cfg = PPO_DEFAULT_CONFIG.copy()
+cfg["rollouts"] = 1024  # memory_size
+cfg["learning_epochs"] = 10
+cfg["mini_batches"] = 32
+cfg["discount_factor"] = 0.9
+cfg["lambda"] = 0.95
+cfg["learning_rate"] = 1e-3
+cfg["learning_rate_scheduler"] = KLAdaptiveRL
+cfg["learning_rate_scheduler_kwargs"] = {"kl_threshold": 0.008}
+cfg["grad_norm_clip"] = 0.5
+cfg["ratio_clip"] = 0.2
+cfg["value_clip"] = 0.2
+cfg["clip_predicted_values"] = False
+cfg["entropy_loss_scale"] = 0.0
+cfg["value_loss_scale"] = 0.5
+cfg["kl_threshold"] = 0
+cfg["state_preprocessor"] = RunningStandardScaler
+cfg["state_preprocessor_kwargs"] = {"size": env.observation_space, "device": device}
+cfg["value_preprocessor"] = RunningStandardScaler
+cfg["value_preprocessor_kwargs"] = {"size": 1, "device": device}
+# logging to TensorBoard and write checkpoints (in timesteps)
+cfg["experiment"]["write_interval"] = 500
+cfg["experiment"]["checkpoint_interval"] = 5000
+cfg["experiment"]["directory"] = "runs/torch/Pendulum"
+
+# TODO(nicer)
+if models['policy']._rnn:
+    agent = PPO_RNN(models=models,
+                memory=memory,
+                cfg=cfg,
+                observation_space=env.observation_space,
+                action_space=env.action_space,
+                device=device)
+else:
+    agent = PPO(models=models,
+                memory=memory,
+                cfg=cfg,
+                observation_space=env.observation_space,
+                action_space=env.action_space,
+                device=device)
+
+# configure and instantiate the RL trainer
+cfg_trainer = {"timesteps": 100000, "headless": True}
+trainer = SequentialTrainer(cfg=cfg_trainer, env=env, agents=[agent])
+
+# start training
+trainer.train()

rlmodule-0.1.5/rlmodule/skrl/torch/network/__init__.py

@@ -0,0 +1,8 @@
+__all__ = [
+    # modules:
+    "MLP", "RNN", "GRU", "LSTM", "RnnBase", "RnnMlp",
+    # configs:
+    "NetworkCfg", "MlpCfg", "RnnBaseCfg", "RnnCfg", "GruCfg", "LstmCfg", "RnnMlpCfg"
+]
+from rlmodule.source.network import MLP, RNN, GRU, LSTM, RnnBase, RnnMlp
+from rlmodule.source.network_cfg import NetworkCfg, MlpCfg, RnnBaseCfg, RnnCfg, GruCfg, LstmCfg, RnnMlpCfg

rlmodule-0.1.5/rlmodule/skrl/torch/output_layer/__init__.py

@@ -0,0 +1,5 @@
+__all__ = [
+    "DeterministicLayerCfg",
+    "GaussianLayerCfg"
+]
+from rlmodule.source.output_layer_cfg import DeterministicLayerCfg, GaussianLayerCfg

rlmodule-0.1.5/rlmodule/source/build.py

@@ -0,0 +1,33 @@
+from rlmodule.source.rlmodel_cfg import BaseRLCfg, RLModelCfg, SharedRLModelCfg
+from rlmodule.source.utils import get_output_size
+
+#TODO this, or inside RLModel class that have a config passed
+def build_model(cfg: BaseRLCfg):
+
+    #1) network = create # for now it is created, todo from config
+    
+    #2) get output size - what type?  ..what is Model init doing with action/observation space
+
+    net = cfg.network.module(cfg.network)
+    
+    network_output_size = get_output_size(net, cfg.network.input_size)
+
+    def build_output_layer(layer_cfg):
+        return layer_cfg.class_type( device = cfg.device, input_size = network_output_size, cfg = layer_cfg)
+    
+    if isinstance(cfg, RLModelCfg):
+        rl_model = cfg.class_type( device = cfg.device,
+                        network = net,
+                        output_layer = build_output_layer( cfg.output_layer)
+                     )
+    elif isinstance(cfg, SharedRLModelCfg):
+        rl_model = cfg.class_type( device = cfg.device,
+                        network = net,
+                        policy_output_layer = build_output_layer( cfg.policy_output_layer ),
+                        value_output_layer = build_output_layer( cfg.value_output_layer ),
+                     )
+    else:
+        raise TypeError(
+                f" Received unsupported class: '{type(cfg)}'."
+            )
+    return rl_model

{rlmodule-0.0.5 → rlmodule-0.1.5}/rlmodule/source/deprecated.py

@@ -72,13 +72,14 @@ class GaussianMixin:
               )
             )
         """
-        self._clip_actions = clip_actions and (issubclass(type(self.action_space), gym.Space) or \
-            issubclass(type(self.action_space), gymnasium.Space))
+        # TODO return this functionality
+        # self._clip_actions = clip_actions and (issubclass(type(self.action_space), gym.Space) or \
+        #     issubclass(type(self.action_space), gymnasium.Space))
 
-        if self._clip_actions:
-            # TODO(ll) the code assume it will be inherited ... self.device is not set anywhere.  
-            self._clip_actions_min = torch.tensor(self.action_space.low, device=self.device, dtype=torch.float32)
-            self._clip_actions_max = torch.tensor(self.action_space.high, device=self.device, dtype=torch.float32)
+        # if self._clip_actions:
+        #     # TODO(ll) the code assume it will be inherited ... self.device is not set anywhere.  
+        #     self._clip_actions_min = torch.tensor(self.action_space.low, device=self.device, dtype=torch.float32)
+        #     self._clip_actions_max = torch.tensor(self.action_space.high, device=self.device, dtype=torch.float32)
 
         self._clip_log_std = clip_log_std
         self._log_std_min = min_log_std
@@ -194,8 +195,9 @@ class GaussianMixin:
         actions = self._distribution.rsample()
 
         # clip actions
-        if self._clip_actions:
-            actions = torch.clamp(actions, min=self._clip_actions_min, max=self._clip_actions_max)
+        # TODO enable
+        # if self._clip_actions:
+        #     actions = torch.clamp(actions, min=self._clip_actions_min, max=self._clip_actions_max)
         
         if taken_actions is None:
             taken_actions = actions
@@ -309,12 +311,14 @@ class DeterministicMixin:
               )
             )
         """
-        self._clip_actions = clip_actions and (issubclass(type(self.action_space), gym.Space) or \
-            issubclass(type(self.action_space), gymnasium.Space))
+        pass
+        # TODO return this functionality
+        # self._clip_actions = clip_actions and (issubclass(type(self.action_space), gym.Space) or \
+        #     issubclass(type(self.action_space), gymnasium.Space))
 
-        if self._clip_actions:
-            self._clip_actions_min = torch.tensor(self.action_space.low, device=self.device, dtype=torch.float32)
-            self._clip_actions_max = torch.tensor(self.action_space.high, device=self.device, dtype=torch.float32)
+        # if self._clip_actions:
+        #     self._clip_actions_min = torch.tensor(self.action_space.low, device=self.device, dtype=torch.float32)
+        #     self._clip_actions_max = torch.tensor(self.action_space.high, device=self.device, dtype=torch.float32)
 
     def act(self,
             inputs: Mapping[str, Union[torch.Tensor, Any]],
@@ -344,8 +348,9 @@ class DeterministicMixin:
         actions, outputs = self.compute(inputs, role)
 
         # clip actions
-        if self._clip_actions:
-            actions = torch.clamp(actions, min=self._clip_actions_min, max=self._clip_actions_max)
+        # TODO enable
+        # if self._clip_actions:
+        #     actions = torch.clamp(actions, min=self._clip_actions_min, max=self._clip_actions_max)
 
         return actions, None, outputs
     
@@ -371,8 +376,8 @@ class DeterministicMixin:
             >>> print(actions.shape, outputs)
             torch.Size([4096, 1]) {}
         """
-        # clip actions
-        if self._clip_actions:
-            actions = torch.clamp(actions, min=self._clip_actions_min, max=self._clip_actions_max)
+        # clip actions # TODO enable
+        # if self._clip_actions:
+        #     actions = torch.clamp(actions, min=self._clip_actions_min, max=self._clip_actions_max)
 
         return actions, None, outputs