rlmodule 0.0.5__tar.gz → 0.1.0__tar.gz

{rlmodule-0.0.5 → rlmodule-0.1.0}/PKG-INFO

@@ -1,8 +1,9 @@
 Metadata-Version: 2.1
 Name: rlmodule
-Version: 0.0.5
+Version: 0.1.0
 Summary: Flexible reinforcement learning models instantiators library
 Author: Lopatovsky
+Author-email: lopatovsky@gmail.com
 Maintainer: Lopatovsky
 License: MIT License
 Project-URL: Homepage, https://github.com/FabricaAI/rlmodule
@@ -24,11 +25,55 @@ License-File: LICENSE
 Requires-Dist: gym
 Requires-Dist: gymnasium
 Requires-Dist: torch>=1.9
-Requires-Dist: skrl>=1.2.0
+Requires-Dist: skrl>=1.3.0
 Provides-Extra: skrl
-Requires-Dist: skrl>=1.2.0; extra == "skrl"
+Requires-Dist: skrl>=1.3.0; extra == "skrl"
 Provides-Extra: all
-Requires-Dist: skrl>=1.2.0; extra == "all"
+Requires-Dist: skrl>=1.3.0; extra == "all"
 
 # rlmodule
 Flexible reinforcement learning models instantiators library
+
+
+## How to run
+
+### Install rlmodule from local code
+
+- Make sure you are in base rlmodule dit.
+
+- Start virtual env.
+```
+python3 -m venv venv
+source venv/bin/activate
+```
+- Install library from local code 
+```
+pip install -e .
+```
+Note: sometimes installation may fail, if there is a run/ dir present, you may need to remove it (TODO: fix)
+```
+rm -rf runs
+```
+### Run chosen example
+```
+python3 rlmodule/skrl/examples/gymnasium/skrl_gymnasium_pendulum.py
+```
+
+
+## Update new version to PIP
+
+```
+pip install build twine
+```
+
+```
+rm -rf runs
+python -m build
+```
+
+```
+twine upload dist/*
+```
+
+
+

rlmodule-0.1.0/README.md

@@ -0,0 +1,46 @@
+# rlmodule
+Flexible reinforcement learning models instantiators library
+
+
+## How to run
+
+### Install rlmodule from local code
+
+- Make sure you are in base rlmodule dit.
+
+- Start virtual env.
+```
+python3 -m venv venv
+source venv/bin/activate
+```
+- Install library from local code 
+```
+pip install -e .
+```
+Note: sometimes installation may fail, if there is a run/ dir present, you may need to remove it (TODO: fix)
+```
+rm -rf runs
+```
+### Run chosen example
+```
+python3 rlmodule/skrl/examples/gymnasium/skrl_gymnasium_pendulum.py
+```
+
+
+## Update new version to PIP
+
+```
+pip install build twine
+```
+
+```
+rm -rf runs
+python -m build
+```
+
+```
+twine upload dist/*
+```
+
+
+

{rlmodule-0.0.5 → rlmodule-0.1.0}/pyproject.toml

@@ -1,16 +1,18 @@
 [project]
 name = "rlmodule"
-version = "0.0.5"
+version = "0.1.0"
 description = "Flexible reinforcement learning models instantiators library"
 readme = "README.md"
 requires-python = ">=3.6"
 license = {text = "MIT License"}
 authors = [
-  {name = "Lopatovsky"},
+  {name = "Lopatovsky"}, 
+  {email = "lopatovsky@gmail.com"},
 ]
 maintainers = [
   {name = "Lopatovsky"},
 ]
+
 keywords = ["reinforcement-learning", "machine-learning", "reinforcement", "machine", "learning", "rl", "rl-models", "rl-modules", "models", "modules"]
 classifiers = [
   "License :: OSI Approved :: MIT License",
@@ -24,18 +26,18 @@ dependencies = [
   "gym",
   "gymnasium",
   "torch>=1.9",
-  "skrl>=1.2.0",
+  "skrl>=1.3.0",
 ]
 
 [project.optional-dependencies]
 
 # TODO(ll) only examples should depend on skrl, not the main code.
 skrl = [
-  "skrl>=1.2.0",
+  "skrl>=1.3.0",
 ]
 
 all = [
-  "skrl>=1.2.0",
+  "skrl>=1.3.0",
 ]
 
 # urls

rlmodule-0.1.0/rlmodule/__init__.py

@@ -0,0 +1,4 @@
+# TODO this is not very good way
+from .source.rlmodel import RLModel, SharedRLModel # noqa: F401, F403
+from .source.output_layer import GaussianLayer, DeterministicLayer # noqa: F401, F403
+from .source.modules import MLP # noqa: F401, F403

rlmodule-0.1.0/rlmodule/skrl/examples/gymnasium/skrl_gymnasium_pendulum.py

@@ -0,0 +1,276 @@
+import gymnasium as gym
+
+import torch.nn as nn
+
+from rlmodule.source.model_cfg import SharedRLModelCfg
+from rlmodule.source.modules import RnnMlpWithForwardedInput
+from rlmodule.source.modules_cfg import LstmCfg, MlpCfg, RnnMlpCfg
+from rlmodule.source.output_layer_cfg import DeterministicLayerCfg, GaussianLayerCfg
+from rlmodule.source.model_builder import build_model
+
+# example how this could be called:
+# from rlmodule.skrl.torch.model import SharedRLModelCfg
+# from rlmodule.skrl.torch.module import LstmCfg
+# from rlmodule.skrl.torch.output_layer import DeterministicLayerCfg, GaussianLayerCfg
+# from rlmodule.skrl.torch import build_model
+
+# import the skrl components to build the RL system
+from skrl.agents.torch.ppo import PPO, PPO_RNN, PPO_DEFAULT_CONFIG
+
+from skrl.envs.wrappers.torch.gymnasium_envs import GymnasiumWrapper
+#from skrl.envs.wrappers.torch import wrap_env
+from skrl.memories.torch import RandomMemory
+from skrl.resources.preprocessors.torch import RunningStandardScaler
+from skrl.resources.schedulers.torch import KLAdaptiveRL
+from skrl.trainers.torch import SequentialTrainer
+from skrl.utils import set_seed
+from skrl.envs.wrappers.torch.gymnasium_envs import GymnasiumWrapper
+
+# todo set seed
+# seed for reproducibility
+set_seed()  # e.g. `set_seed(42)` for fixed seed
+
+#TODO next
+# (Done) check how to do own @configclass .. or use one from isaac by default (print something to know version)
+# (Done) RNN code modify to use data class
+# (Done) RnnMLP
+# (Done) Custom function Network
+# (Done) Move things logically
+# (Done) Handle input shapes better?
+# (Done except Cnn) Convert all rest modules to use Configclass
+# Rearrange library in the way it can be used for other rllibs and for torch/jax?
+# Create examples - mlp, rnn,gru,lstm, Lstmmlp, shared - separate , custom net by fcion
+#                 - examples from modules make in class header not function 
+# WRITE README tutorial
+# Run & fix pre-commit
+# annotate cfgs in modules - why doesn't work
+# extensive comments
+# Launch new version to pip
+# Import new version in Isaac-lab
+
+ 
+
+# def example_module(cfg):
+#     cfg = RnnMlpCfg(
+#         input_size = env.observation_space,
+#         rnn = LstmCfg(
+#             num_envs = env.num_envs,
+#             num_layers = 1,
+#             hidden_size = 32,
+#             sequence_length = 16,
+#         ),
+#         mlp = MlpCfg(
+#             hidden_units = [64, 64, 64],
+#             activations = [nn.ReLU(), nn.ReLU(), nn.ReLU()],
+#         ),
+#     )
+#     return RnnMlp(cfg)
+
+def get_shared_model(env):
+    # instantiate the agent's models (function approximators).
+
+    net_cfg = MlpCfg(
+        input_size = env.observation_space,
+        hidden_units = [64, 64, 64],
+        activations = [nn.ReLU(), nn.ReLU(), nn.ReLU()],
+    )
+
+    # 2
+    # net_cfg = LstmCfg(
+    #     input_size = env.observation_space,
+    #     num_envs = env.num_envs,
+    #     num_layers = 1,
+    #     hidden_size = 32,
+    #     sequence_length = 16,
+    # )
+
+    # 3
+    # net_cfg = RnnMlpCfg(
+    #     input_size = env.observation_space,
+    #     rnn = LstmCfg(
+    #         num_envs = env.num_envs,
+    #         num_layers = 1,
+    #         hidden_size = 32,
+    #         sequence_length = 16,
+    #     ),
+    #     mlp = MlpCfg(
+    #         hidden_units = [64, 64, 64],
+    #         activations = [nn.ReLU(), nn.ReLU(), nn.ReLU()],
+    #     ),
+    # )
+
+    # 3.5 
+    # net_cfg = RnnMlpCfg(
+    #     input_size = env.observation_space,
+    #     module = RnnMlpWithForwardedInput,
+    #     rnn = LstmCfg(
+    #         num_envs = env.num_envs,
+    #         num_layers = 1,
+    #         hidden_size = 32,
+    #         sequence_length = 16,
+    #     ),
+    #     mlp = MlpCfg(
+    #         hidden_units = [64, 64, 64],
+    #         activations = [nn.ReLU(), nn.ReLU(), nn.ReLU()],
+    #     ),
+    # )
+
+    # 4
+    # net_cfg = NetworkCfg( input_size = env.observation_space,
+    #                       module = example_module)
+
+    # variant IV - all config
+    # net also config (optional)?
+    model = build_model( 
+        SharedRLModelCfg(
+            network= net_cfg,
+            device= device,
+            policy_output_layer= GaussianLayerCfg(
+                    output_size=env.action_space,
+                    min_log_std=-1.2,
+                    max_log_std=2,
+                    initial_log_std=0.2,
+                ),
+            value_output_layer= DeterministicLayerCfg(),
+        )
+    )
+    # RLCfg(
+    #     Model
+    #     AgentCfg
+    #     MemoryCfg
+    # )
+
+    # build_model(cfg.model)
+
+    # cfg.agent.algoclass(
+    #     cfg.agent.algo_config
+    #     cfg.memory
+    # )
+
+    # SharedRLModelCfg(RLModelCfg){
+    #     class = SharedRLModel
+
+    # }
+
+    # net
+    # output layers.class()
+
+    # cfg.model.class(  net, output_layers )
+
+    #configclass -> dataclass - easy (Ron said)
+
+
+    # variant V:
+    # ideas?
+
+
+    return {'policy': model, 'value': model}
+
+# TODO Update separate model to configclass
+# def get_separate_model(env):
+#     # instantiate the agent's models (function approximators).
+#     params = {'input_size': env.observation_space, 
+#               'hidden_units': [64, 64, 64], 
+#               'activations': ['relu', 'relu', 'relu']
+#               }
+#     net = MLP(params)
+
+#     policy_model = RLModel(
+#         network=net,
+#         device= device,
+#         output_layer = GaussianLayer(
+#                 input_size=64,  # TODO
+#                 output_size=env.action_space,
+#                 output_activation=nn.Tanh(),
+#                 clip_actions=False,
+#                 clip_log_std=True,
+#                 min_log_std=-1.2,
+#                 max_log_std=2,
+#                 initial_log_std=0.2,
+#             ),
+#     )
+
+#     value_model = RLModel(
+#         network=net,
+#         device= device,
+#         output_layer = DeterministicLayer(
+#                 input_size=64,
+#                 output_size=1,
+#                 output_activation=nn.Identity(),
+#             ),
+#     )
+
+#     return {'policy': policy_model, 'value': value_model}
+
+
+# load and wrap the gymnasium environment.
+# note: the environment version may change depending on the gymnasium version
+try:
+    env = gym.vector.make("Pendulum-v1", num_envs=4, asynchronous=False)
+except (gym.error.DeprecatedEnv, gym.error.VersionNotFound) as e:
+    env_id = [spec for spec in gym.envs.registry if spec.startswith("Pendulum-v")][0]
+    print("Pendulum-v1 not found. Trying {}".format(env_id))
+    env = gym.vector.make(env_id, num_envs=4, asynchronous=False)
+
+env = GymnasiumWrapper(env)
+
+device = env.device
+
+
+# instantiate a memory as rollout buffer (any memory can be used for this)
+memory = RandomMemory(memory_size=1024, num_envs=env.num_envs, device=device)
+
+models = get_shared_model(env)
+
+print(models)
+
+
+# configure and instantiate the agent (visit its documentation to see all the options)
+# https://skrl.readthedocs.io/en/latest/api/agents/ppo.html#configuration-and-hyperparameters
+cfg = PPO_DEFAULT_CONFIG.copy()
+cfg["rollouts"] = 1024  # memory_size
+cfg["learning_epochs"] = 10
+cfg["mini_batches"] = 32
+cfg["discount_factor"] = 0.9
+cfg["lambda"] = 0.95
+cfg["learning_rate"] = 1e-3
+cfg["learning_rate_scheduler"] = KLAdaptiveRL
+cfg["learning_rate_scheduler_kwargs"] = {"kl_threshold": 0.008}
+cfg["grad_norm_clip"] = 0.5
+cfg["ratio_clip"] = 0.2
+cfg["value_clip"] = 0.2
+cfg["clip_predicted_values"] = False
+cfg["entropy_loss_scale"] = 0.0
+cfg["value_loss_scale"] = 0.5
+cfg["kl_threshold"] = 0
+cfg["state_preprocessor"] = RunningStandardScaler
+cfg["state_preprocessor_kwargs"] = {"size": env.observation_space, "device": device}
+cfg["value_preprocessor"] = RunningStandardScaler
+cfg["value_preprocessor_kwargs"] = {"size": 1, "device": device}
+# logging to TensorBoard and write checkpoints (in timesteps)
+cfg["experiment"]["write_interval"] = 500
+cfg["experiment"]["checkpoint_interval"] = 5000
+cfg["experiment"]["directory"] = "runs/torch/Pendulum"
+
+# TODO(nicer)
+if models['policy']._rnn:
+    agent = PPO_RNN(models=models,
+                memory=memory,
+                cfg=cfg,
+                observation_space=env.observation_space,
+                action_space=env.action_space,
+                device=device)
+else:
+    agent = PPO(models=models,
+                memory=memory,
+                cfg=cfg,
+                observation_space=env.observation_space,
+                action_space=env.action_space,
+                device=device)
+
+# configure and instantiate the RL trainer
+cfg_trainer = {"timesteps": 100000, "headless": True}
+trainer = SequentialTrainer(cfg=cfg_trainer, env=env, agents=[agent])
+
+# start training
+trainer.train()

{rlmodule-0.0.5 → rlmodule-0.1.0}/rlmodule/source/deprecated.py

@@ -72,13 +72,14 @@ class GaussianMixin:
               )
             )
         """
-        self._clip_actions = clip_actions and (issubclass(type(self.action_space), gym.Space) or \
-            issubclass(type(self.action_space), gymnasium.Space))
+        # TODO return this functionality
+        # self._clip_actions = clip_actions and (issubclass(type(self.action_space), gym.Space) or \
+        #     issubclass(type(self.action_space), gymnasium.Space))
 
-        if self._clip_actions:
-            # TODO(ll) the code assume it will be inherited ... self.device is not set anywhere.  
-            self._clip_actions_min = torch.tensor(self.action_space.low, device=self.device, dtype=torch.float32)
-            self._clip_actions_max = torch.tensor(self.action_space.high, device=self.device, dtype=torch.float32)
+        # if self._clip_actions:
+        #     # TODO(ll) the code assume it will be inherited ... self.device is not set anywhere.  
+        #     self._clip_actions_min = torch.tensor(self.action_space.low, device=self.device, dtype=torch.float32)
+        #     self._clip_actions_max = torch.tensor(self.action_space.high, device=self.device, dtype=torch.float32)
 
         self._clip_log_std = clip_log_std
         self._log_std_min = min_log_std
@@ -194,8 +195,9 @@ class GaussianMixin:
         actions = self._distribution.rsample()
 
         # clip actions
-        if self._clip_actions:
-            actions = torch.clamp(actions, min=self._clip_actions_min, max=self._clip_actions_max)
+        # TODO enable
+        # if self._clip_actions:
+        #     actions = torch.clamp(actions, min=self._clip_actions_min, max=self._clip_actions_max)
         
         if taken_actions is None:
             taken_actions = actions
@@ -309,12 +311,14 @@ class DeterministicMixin:
               )
             )
         """
-        self._clip_actions = clip_actions and (issubclass(type(self.action_space), gym.Space) or \
-            issubclass(type(self.action_space), gymnasium.Space))
+        pass
+        # TODO return this functionality
+        # self._clip_actions = clip_actions and (issubclass(type(self.action_space), gym.Space) or \
+        #     issubclass(type(self.action_space), gymnasium.Space))
 
-        if self._clip_actions:
-            self._clip_actions_min = torch.tensor(self.action_space.low, device=self.device, dtype=torch.float32)
-            self._clip_actions_max = torch.tensor(self.action_space.high, device=self.device, dtype=torch.float32)
+        # if self._clip_actions:
+        #     self._clip_actions_min = torch.tensor(self.action_space.low, device=self.device, dtype=torch.float32)
+        #     self._clip_actions_max = torch.tensor(self.action_space.high, device=self.device, dtype=torch.float32)
 
     def act(self,
             inputs: Mapping[str, Union[torch.Tensor, Any]],
@@ -344,8 +348,9 @@ class DeterministicMixin:
         actions, outputs = self.compute(inputs, role)
 
         # clip actions
-        if self._clip_actions:
-            actions = torch.clamp(actions, min=self._clip_actions_min, max=self._clip_actions_max)
+        # TODO enable
+        # if self._clip_actions:
+        #     actions = torch.clamp(actions, min=self._clip_actions_min, max=self._clip_actions_max)
 
         return actions, None, outputs
     
@@ -371,8 +376,8 @@ class DeterministicMixin:
             >>> print(actions.shape, outputs)
             torch.Size([4096, 1]) {}
         """
-        # clip actions
-        if self._clip_actions:
-            actions = torch.clamp(actions, min=self._clip_actions_min, max=self._clip_actions_max)
+        # clip actions # TODO enable
+        # if self._clip_actions:
+        #     actions = torch.clamp(actions, min=self._clip_actions_min, max=self._clip_actions_max)
 
         return actions, None, outputs