rlmodule 0.1.5__tar.gz → 0.1.6__tar.gz

{rlmodule-0.1.5 → rlmodule-0.1.6}/LICENSE

@@ -27,4 +27,4 @@ ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

{rlmodule-0.1.5 → rlmodule-0.1.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rlmodule
-Version: 0.1.5
+Version: 0.1.6
 Summary: Flexible reinforcement learning models instantiators library
 Author: Lopatovsky
 Author-email: lopatovsky@gmail.com
@@ -23,7 +23,7 @@ Requires-Python: >=3.6
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: gym
-Requires-Dist: gymnasium
+Requires-Dist: gymnasium>=1.0.0
 Requires-Dist: torch>=1.9
 Requires-Dist: skrl>=1.3.0
 Provides-Extra: skrl
@@ -34,19 +34,41 @@ Requires-Dist: skrl>=1.3.0; extra == "all"
 # rlmodule
 Flexible reinforcement learning models instantiators library
 
+Function approximators simple, but still strong. RNN - GRU - LSTM / SAC
+
+Now it only supports skrl, but is intended to be library agnostic - in later expansion
+
+try other algos
+shared separate model
+
+
+<!--
+# todo
+# some envs have by default range on which they work, like pendulum -2, 2, need to propaget that in.
+# cnn
+# WRITE README tutorial
+# Run & fix pre-commit
+# annotate cfgs in modules - why doesn't work TYPE_CHECKING
+# extensive comments
+# Launch new version to pip
+# Import new version in Isaac-lab
+# lazy linear? what is it ?
+# random model run add function back
+-->
+
 
 ## How to run
 
 ### Install rlmodule from local code
 
-- Make sure you are in base rlmodule dit.
+- Make sure you are in base rlmodule dict.
 
 - Start virtual env.
 ```
 python3 -m venv venv
 source venv/bin/activate
 ```
-- Install library from local code 
+- Install library from local code
 ```
 pip install -e .
 ```
@@ -54,14 +76,24 @@ Note: sometimes installation may fail, if there is a run/ dir present, you may n
 ```
 rm -rf runs
 ```
+
 ### Run chosen example
 ```
-python3 rlmodule/skrl/examples/gymnasium/skrl_gymnasium_pendulum.py
+python3 rlmodule/rlmodule/skrl/torch/gymnasium/pendulum_ppo_mlp_separate_model.py
 ```
 
+Optional: observe run results in Tensorboard
+
+```
+tensorboard --logdir=runs/
+```
+
+
 
 ## Update new version to PIP
 
+Change version name in pyproject.toml
+
 ```
 pip install build twine
 ```
@@ -74,6 +106,3 @@ python -m build
 ```
 twine upload dist/*
 ```
-
-
-

rlmodule-0.1.6/README.md

@@ -0,0 +1,75 @@
+# rlmodule
+Flexible reinforcement learning models instantiators library
+
+Function approximators simple, but still strong. RNN - GRU - LSTM / SAC
+
+Now it only supports skrl, but is intended to be library agnostic - in later expansion
+
+try other algos
+shared separate model
+
+
+<!--
+# todo
+# some envs have by default range on which they work, like pendulum -2, 2, need to propaget that in.
+# cnn
+# WRITE README tutorial
+# Run & fix pre-commit
+# annotate cfgs in modules - why doesn't work TYPE_CHECKING
+# extensive comments
+# Launch new version to pip
+# Import new version in Isaac-lab
+# lazy linear? what is it ?
+# random model run add function back
+-->
+
+
+## How to run
+
+### Install rlmodule from local code
+
+- Make sure you are in base rlmodule dict.
+
+- Start virtual env.
+```
+python3 -m venv venv
+source venv/bin/activate
+```
+- Install library from local code
+```
+pip install -e .
+```
+Note: sometimes installation may fail, if there is a run/ dir present, you may need to remove it (TODO: fix)
+```
+rm -rf runs
+```
+
+### Run chosen example
+```
+python3 rlmodule/rlmodule/skrl/torch/gymnasium/pendulum_ppo_mlp_separate_model.py
+```
+
+Optional: observe run results in Tensorboard
+
+```
+tensorboard --logdir=runs/
+```
+
+
+
+## Update new version to PIP
+
+Change version name in pyproject.toml
+
+```
+pip install build twine
+```
+
+```
+rm -rf runs
+python -m build
+```
+
+```
+twine upload dist/*
+```

{rlmodule-0.1.5 → rlmodule-0.1.6}/pyproject.toml

@@ -1,12 +1,12 @@
 [project]
 name = "rlmodule"
-version = "0.1.5"
+version = "0.1.6"
 description = "Flexible reinforcement learning models instantiators library"
 readme = "README.md"
 requires-python = ">=3.6"
 license = {text = "MIT License"}
 authors = [
-  {name = "Lopatovsky"}, 
+  {name = "Lopatovsky"},
   {email = "lopatovsky@gmail.com"},
 ]
 maintainers = [
@@ -24,7 +24,7 @@ classifiers = [
 ]
 dependencies = [
   "gym",
-  "gymnasium",
+  "gymnasium>=1.0.0",
   "torch>=1.9",
   "skrl>=1.3.0",
 ]

rlmodule-0.1.6/rlmodule/__init__.py

@@ -0,0 +1,28 @@
+import logging
+
+
+__all__ = ["logger"]
+
+
+# logger with format
+class _Formatter(logging.Formatter):
+    _format = "[%(name)s:%(levelname)s] %(message)s"
+    _formats = {
+        logging.DEBUG: f"\x1b[38;20m{_format}\x1b[0m",
+        logging.INFO: f"\x1b[38;20m{_format}\x1b[0m",
+        logging.WARNING: f"\x1b[33;20m{_format}\x1b[0m",
+        logging.ERROR: f"\x1b[31;20m{_format}\x1b[0m",
+        logging.CRITICAL: f"\x1b[31;1m{_format}\x1b[0m",
+    }
+
+    def format(self, record):
+        return logging.Formatter(self._formats.get(record.levelno)).format(record)
+
+
+_handler = logging.StreamHandler()
+_handler.setLevel(logging.DEBUG)
+_handler.setFormatter(_Formatter())
+
+logger = logging.getLogger("rlmodule")
+logger.setLevel(logging.DEBUG)
+logger.addHandler(_handler)

rlmodule-0.1.6/rlmodule/skrl/torch/__init__.py

@@ -0,0 +1,3 @@
+__all__ = ["build_modelBaseRLCfg", "RLModelCfg", "SeparatedRLModelCfg", "SharedRLModelCfg"]
+from rlmodule.source.build import build_model  # noqa: F401
+from rlmodule.source.rlmodel_cfg import BaseRLCfg, RLModelCfg, SeparatedRLModelCfg, SharedRLModelCfg  # noqa: F401

rlmodule-0.1.6/rlmodule/skrl/torch/gymnasium/pendulum_ppo_mlp_separate_model.py

@@ -0,0 +1,124 @@
+import os
+from datetime import datetime
+import gymnasium as gym
+
+# import the skrl components to build the RL system
+from skrl.agents.torch.ppo import PPO, PPO_DEFAULT_CONFIG, PPO_RNN
+from skrl.envs.wrappers.torch.gymnasium_envs import GymnasiumWrapper
+from skrl.memories.torch import RandomMemory
+from skrl.resources.preprocessors.torch import RunningStandardScaler
+from skrl.resources.schedulers.torch import KLAdaptiveRL
+from skrl.trainers.torch import SequentialTrainer
+from skrl.utils import set_seed
+
+import torch.nn as nn
+
+from rlmodule.skrl.torch import RLModelCfg, build_model
+from rlmodule.skrl.torch.network import MlpCfg
+from rlmodule.skrl.torch.output_layer import DeterministicLayerCfg, GaussianLayerCfg
+
+
+def get_model(env):
+    """Instantiate the agent's models (function approximators)."""
+
+    net_cfg = MlpCfg(
+        input_states=env.observation_space,
+        hidden_units=[400, 300],
+        activation=nn.ReLU,
+    )
+
+    print(env.action_space)
+    print(env.observation_space)
+
+    policy_model = build_model(
+        RLModelCfg(
+            network=net_cfg,
+            device=device,
+            output_layer=GaussianLayerCfg(
+                output_size=env.action_space,
+                output_scale=2.0,
+                min_log_std=-1.2,
+                max_log_std=2,
+                initial_log_std=0.0,
+            ),
+        )
+    )
+
+    value_model = build_model(
+        RLModelCfg(
+            network=net_cfg,
+            device=device,
+            output_layer=DeterministicLayerCfg(),
+        )
+    )
+
+    models = {"policy": policy_model, "value": value_model}
+    print(models)
+    return models
+
+
+# set seed for reproducibility
+seed = 42
+set_seed(seed)
+
+# load and wrap the gymnasium environment.
+env = gym.make_vec("Pendulum-v1", num_envs=4, vectorization_mode="sync")
+env.reset(seed=seed)
+env = GymnasiumWrapper(env)
+device = env.device
+
+# instantiate a memory as rollout buffer (any memory can be used for this)
+memory = RandomMemory(memory_size=1024, num_envs=env.num_envs, device=device)
+
+models = get_model(env)
+
+# configure and instantiate the agent (visit its documentation to see all the options)
+# https://skrl.readthedocs.io/en/latest/api/agents/ppo.html#configuration-and-hyperparameters
+cfg = PPO_DEFAULT_CONFIG.copy()
+cfg["rollouts"] = 1024  # memory_size
+cfg["learning_epochs"] = 10
+cfg["mini_batches"] = 32
+cfg["discount_factor"] = 0.9
+cfg["lambda"] = 0.95
+cfg["learning_rate"] = 1e-3
+cfg["learning_rate_scheduler"] = KLAdaptiveRL
+cfg["learning_rate_scheduler_kwargs"] = {"kl_threshold": 0.008}
+cfg["grad_norm_clip"] = 0.5
+cfg["ratio_clip"] = 0.2
+cfg["value_clip"] = 0.2
+cfg["clip_predicted_values"] = False
+cfg["entropy_loss_scale"] = 0.0
+cfg["value_loss_scale"] = 0.5
+cfg["kl_threshold"] = 0
+cfg["state_preprocessor"] = RunningStandardScaler
+cfg["state_preprocessor_kwargs"] = {"size": env.observation_space, "device": device}
+cfg["value_preprocessor"] = RunningStandardScaler
+cfg["value_preprocessor_kwargs"] = {"size": 1, "device": device}
+# logging to TensorBoard and write checkpoints (in timesteps)
+cfg["experiment"]["write_interval"] = 500
+cfg["experiment"]["checkpoint_interval"] = 5000
+cfg["experiment"]["directory"] = "runs/skrl/torch/"
+cfg["experiment"]["experiment_name"] = (
+    os.path.splitext(os.path.basename(__file__))[0] + "_" + datetime.now().strftime("%Y-%m-%d_%H:%M:%S")
+)
+
+params = {
+    "models": models,
+    "memory": memory,
+    "cfg": cfg,
+    "observation_space": env.observation_space,
+    "action_space": env.action_space,
+    "device": device,
+}
+
+if models["policy"].is_rnn:
+    agent = PPO_RNN(**params)
+else:
+    agent = PPO(**params)
+
+# configure and instantiate the RL trainer
+cfg_trainer = {"timesteps": 100000, "headless": True}
+trainer = SequentialTrainer(cfg=cfg_trainer, env=env, agents=[agent])
+
+# start training
+trainer.train()

rlmodule-0.1.6/rlmodule/skrl/torch/gymnasium/pendulum_ppo_mlp_shared_model.py

@@ -0,0 +1,112 @@
+import os
+from datetime import datetime
+import gymnasium as gym
+
+# import the skrl components to build the RL system
+from skrl.agents.torch.ppo import PPO, PPO_DEFAULT_CONFIG, PPO_RNN
+from skrl.envs.wrappers.torch.gymnasium_envs import GymnasiumWrapper
+from skrl.memories.torch import RandomMemory
+from skrl.resources.preprocessors.torch import RunningStandardScaler
+from skrl.resources.schedulers.torch import KLAdaptiveRL
+from skrl.trainers.torch import SequentialTrainer
+from skrl.utils import set_seed
+
+import torch.nn as nn
+
+from rlmodule.skrl.torch import SharedRLModelCfg, build_model
+from rlmodule.skrl.torch.network import MlpCfg
+from rlmodule.skrl.torch.output_layer import DeterministicLayerCfg, GaussianLayerCfg
+
+
+def get_model(env):
+    """Instantiate the agent's models (function approximators)."""
+
+    net_cfg = MlpCfg(
+        input_states=env.observation_space,
+        hidden_units=[400, 300],
+        activation=nn.ReLU,
+    )
+
+    model = build_model(
+        SharedRLModelCfg(
+            network=net_cfg,
+            device=device,
+            policy_output_layer=GaussianLayerCfg(
+                output_size=env.action_space,
+                min_log_std=-1.2,
+                max_log_std=2,
+                initial_log_std=0.0,
+            ),
+            value_output_layer=DeterministicLayerCfg(),
+        )
+    )
+
+    print(model)
+    return {"policy": model, "value": model}
+
+
+# set seed for reproducibility
+seed = 42
+set_seed(seed)
+
+# load and wrap the gymnasium environment.
+env = gym.make_vec("Pendulum-v1", num_envs=4, vectorization_mode="sync")
+env.reset(seed=seed)
+env = GymnasiumWrapper(env)
+device = env.device
+
+# instantiate a memory as rollout buffer (any memory can be used for this)
+memory = RandomMemory(memory_size=1024, num_envs=env.num_envs, device=device)
+
+models = get_model(env)
+
+# configure and instantiate the agent (visit its documentation to see all the options)
+# https://skrl.readthedocs.io/en/latest/api/agents/ppo.html#configuration-and-hyperparameters
+cfg = PPO_DEFAULT_CONFIG.copy()
+cfg["rollouts"] = 1024  # memory_size
+cfg["learning_epochs"] = 10
+cfg["mini_batches"] = 32
+cfg["discount_factor"] = 0.9
+cfg["lambda"] = 0.95
+cfg["learning_rate"] = 1e-3
+cfg["learning_rate_scheduler"] = KLAdaptiveRL
+cfg["learning_rate_scheduler_kwargs"] = {"kl_threshold": 0.008}
+cfg["grad_norm_clip"] = 0.5
+cfg["ratio_clip"] = 0.2
+cfg["value_clip"] = 0.2
+cfg["clip_predicted_values"] = False
+cfg["entropy_loss_scale"] = 0.0
+cfg["value_loss_scale"] = 0.5
+cfg["kl_threshold"] = 0
+cfg["state_preprocessor"] = RunningStandardScaler
+cfg["state_preprocessor_kwargs"] = {"size": env.observation_space, "device": device}
+cfg["value_preprocessor"] = RunningStandardScaler
+cfg["value_preprocessor_kwargs"] = {"size": 1, "device": device}
+# logging to TensorBoard and write checkpoints (in timesteps)
+cfg["experiment"]["write_interval"] = 500
+cfg["experiment"]["checkpoint_interval"] = 5000
+cfg["experiment"]["directory"] = "runs/skrl/torch/"
+cfg["experiment"]["experiment_name"] = (
+    os.path.splitext(os.path.basename(__file__))[0] + "_" + datetime.now().strftime("%Y-%m-%d_%H:%M:%S")
+)
+
+params = {
+    "models": models,
+    "memory": memory,
+    "cfg": cfg,
+    "observation_space": env.observation_space,
+    "action_space": env.action_space,
+    "device": device,
+}
+
+if models["policy"].is_rnn:
+    agent = PPO_RNN(**params)
+else:
+    agent = PPO(**params)
+
+# configure and instantiate the RL trainer
+cfg_trainer = {"timesteps": 100000, "headless": True}
+trainer = SequentialTrainer(cfg=cfg_trainer, env=env, agents=[agent])
+
+# start training
+trainer.train()

rlmodule-0.1.6/rlmodule/skrl/torch/gymnasium/pendulum_ppo_rnn_forwarded_input_shared_model.py

@@ -0,0 +1,123 @@
+import os
+from datetime import datetime
+import gymnasium as gym
+
+# import the skrl components to build the RL system
+from skrl.agents.torch.ppo import PPO, PPO_DEFAULT_CONFIG, PPO_RNN
+from skrl.envs.wrappers.torch.gymnasium_envs import GymnasiumWrapper
+from skrl.memories.torch import RandomMemory
+from skrl.resources.preprocessors.torch import RunningStandardScaler
+from skrl.resources.schedulers.torch import KLAdaptiveRL
+from skrl.trainers.torch import SequentialTrainer
+from skrl.utils import set_seed
+
+import torch.nn as nn
+
+from rlmodule.skrl.torch import SharedRLModelCfg, build_model
+from rlmodule.skrl.torch.network import MlpCfg
+from rlmodule.skrl.torch.output_layer import DeterministicLayerCfg, GaussianLayerCfg
+from rlmodule.source.network import RnnMlpWithForwardedInput
+from rlmodule.source.network_cfg import RnnCfg, RnnMlpCfg
+
+
+def get_model(env):
+    """Instantiate the agent's models (function approximators)."""
+
+    net_cfg = RnnMlpCfg(
+        input_states=env.observation_space,
+        module=RnnMlpWithForwardedInput,
+        rnn=RnnCfg(
+            num_envs=env.num_envs,
+            num_layers=1,
+            hidden_size=32,
+            sequence_length=16,
+        ),
+        mlp=MlpCfg(
+            hidden_units=[64, 64],
+            activation=nn.ReLU,
+        ),
+    )
+
+    model = build_model(
+        SharedRLModelCfg(
+            network=net_cfg,
+            device=device,
+            policy_output_layer=GaussianLayerCfg(
+                output_size=env.action_space,
+                min_log_std=-1.2,
+                max_log_std=2,
+                initial_log_std=0.0,
+            ),
+            value_output_layer=DeterministicLayerCfg(),
+        )
+    )
+
+    print(model)
+    return {"policy": model, "value": model}
+
+
+# set seed for reproducibility
+seed = 42
+set_seed(seed)
+
+# load and wrap the gymnasium environment.
+env = gym.make_vec("Pendulum-v1", num_envs=4, vectorization_mode="sync")
+env.reset(seed=seed)
+env = GymnasiumWrapper(env)
+device = env.device
+
+# instantiate a memory as rollout buffer (any memory can be used for this)
+memory = RandomMemory(memory_size=1024, num_envs=env.num_envs, device=device)
+
+models = get_model(env)
+
+# configure and instantiate the agent (visit its documentation to see all the options)
+# https://skrl.readthedocs.io/en/latest/api/agents/ppo.html#configuration-and-hyperparameters
+cfg = PPO_DEFAULT_CONFIG.copy()
+cfg["rollouts"] = 1024  # memory_size
+cfg["learning_epochs"] = 10
+cfg["mini_batches"] = 32
+cfg["discount_factor"] = 0.9
+cfg["lambda"] = 0.95
+cfg["learning_rate"] = 3e-4
+cfg["learning_rate_scheduler"] = KLAdaptiveRL
+cfg["learning_rate_scheduler_kwargs"] = {"kl_threshold": 0.008}
+cfg["grad_norm_clip"] = 0.5
+cfg["ratio_clip"] = 0.2
+cfg["value_clip"] = 0.2
+cfg["clip_predicted_values"] = False
+cfg["entropy_loss_scale"] = 0.0
+cfg["value_loss_scale"] = 0.5
+cfg["kl_threshold"] = 0
+cfg["state_preprocessor"] = RunningStandardScaler
+cfg["state_preprocessor_kwargs"] = {"size": env.observation_space, "device": device}
+cfg["value_preprocessor"] = RunningStandardScaler
+cfg["value_preprocessor_kwargs"] = {"size": 1, "device": device}
+# logging to TensorBoard and write checkpoints (in timesteps)
+cfg["experiment"]["write_interval"] = 500
+cfg["experiment"]["checkpoint_interval"] = 5000
+cfg["experiment"]["directory"] = "runs/skrl/torch/"
+cfg["experiment"]["experiment_name"] = (
+    os.path.splitext(os.path.basename(__file__))[0] + "_" + datetime.now().strftime("%Y-%m-%d_%H:%M:%S")
+)
+
+params = {
+    "models": models,
+    "memory": memory,
+    "cfg": cfg,
+    "observation_space": env.observation_space,
+    "action_space": env.action_space,
+    "device": device,
+}
+
+if models["policy"].is_rnn:
+    agent = PPO_RNN(**params)
+else:
+    agent = PPO(**params)
+
+# configure and instantiate the RL trainer
+cfg_trainer = {"timesteps": 100000, "headless": True}
+trainer = SequentialTrainer(cfg=cfg_trainer, env=env, agents=[agent])
+
+# start training
+trainer.train()