rlmodule 0.0.5__tar.gz

rlmodule-0.0.5/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 FabricaAI
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

rlmodule-0.0.5/PKG-INFO

@@ -0,0 +1,34 @@
+Metadata-Version: 2.1
+Name: rlmodule
+Version: 0.0.5
+Summary: Flexible reinforcement learning models instantiators library
+Author: Lopatovsky
+Maintainer: Lopatovsky
+License: MIT License
+Project-URL: Homepage, https://github.com/FabricaAI/rlmodule
+Project-URL: Documentation, https://github.com/FabricaAI/rlmodule
+Project-URL: Discussions, https://github.com/FabricaAI/rlmodule/discussions
+Project-URL: Bug Reports, https://github.com/FabricaAI/rlmodule/issues
+Project-URL: Say Thanks!, https://github.com/lopatovsky
+Project-URL: Source, https://github.com/FabricaAI/rlmodule
+Keywords: reinforcement-learning,machine-learning,reinforcement,machine,learning,rl,rl-models,rl-modules,models,modules
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Intended Audience :: Science/Research
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Programming Language :: Python :: 3
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.6
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: gym
+Requires-Dist: gymnasium
+Requires-Dist: torch>=1.9
+Requires-Dist: skrl>=1.2.0
+Provides-Extra: skrl
+Requires-Dist: skrl>=1.2.0; extra == "skrl"
+Provides-Extra: all
+Requires-Dist: skrl>=1.2.0; extra == "all"
+
+# rlmodule
+Flexible reinforcement learning models instantiators library

rlmodule-0.0.5/README.md

@@ -0,0 +1,2 @@
+# rlmodule
+Flexible reinforcement learning models instantiators library

rlmodule-0.0.5/pyproject.toml

@@ -0,0 +1,94 @@
+[project]
+name = "rlmodule"
+version = "0.0.5"
+description = "Flexible reinforcement learning models instantiators library"
+readme = "README.md"
+requires-python = ">=3.6"
+license = {text = "MIT License"}
+authors = [
+  {name = "Lopatovsky"},
+]
+maintainers = [
+  {name = "Lopatovsky"},
+]
+keywords = ["reinforcement-learning", "machine-learning", "reinforcement", "machine", "learning", "rl", "rl-models", "rl-modules", "models", "modules"]
+classifiers = [
+  "License :: OSI Approved :: MIT License",
+  "Intended Audience :: Science/Research",
+  "Topic :: Scientific/Engineering",
+  "Topic :: Scientific/Engineering :: Artificial Intelligence",
+  "Programming Language :: Python :: 3",
+  "Operating System :: OS Independent",
+]
+dependencies = [
+  "gym",
+  "gymnasium",
+  "torch>=1.9",
+  "skrl>=1.2.0",
+]
+
+[project.optional-dependencies]
+
+# TODO(ll) only examples should depend on skrl, not the main code.
+skrl = [
+  "skrl>=1.2.0",
+]
+
+all = [
+  "skrl>=1.2.0",
+]
+
+# urls
+[project.urls]
+"Homepage" = "https://github.com/FabricaAI/rlmodule"
+"Documentation" = "https://github.com/FabricaAI/rlmodule"
+"Discussions" = "https://github.com/FabricaAI/rlmodule/discussions"
+"Bug Reports" = "https://github.com/FabricaAI/rlmodule/issues"
+"Say Thanks!" = "https://github.com/lopatovsky"
+"Source" = "https://github.com/FabricaAI/rlmodule"
+
+[tool.yapf]
+# run: yapf -p -m -i -r <folder>
+based_on_style = "pep8"
+blank_line_before_nested_class_or_def = false
+blank_lines_between_top_level_imports_and_variables = 2
+column_limit = 120
+join_multiple_lines = false
+space_between_ending_comma_and_closing_bracket = false
+spaces_around_power_operator = true
+split_all_top_level_comma_separated_values = true
+split_before_arithmetic_operator = true
+split_before_dict_set_generator = false
+split_before_dot = true
+split_complex_comprehension = true
+coalesce_brackets = true
+
+
+[tool.codespell]
+# run: codespell <folder>
+skip = "./docs/_build,./docs/source/_static"
+quiet-level = 3
+count = ""
+
+
+[tool.isort]
+use_parentheses = false
+line_length = 120
+multi_line_output = 3
+lines_after_imports = 2
+known_annotation = ["typing"]
+known_framework = [
+  "torch",
+  "numpy",
+]
+sections = [
+  "FUTURE",
+  "ANNOTATION",
+  "STDLIB",
+  "THIRDPARTY",
+  "FRAMEWORK",
+  "FIRSTPARTY",
+  "LOCALFOLDER",
+]
+no_lines_before = "THIRDPARTY"
+skip = ["docs"]

rlmodule-0.0.5/rlmodule/__init__.py

@@ -0,0 +1,3 @@
+# TODO this is not very good way
+from .source.rlmodel import RLModel, SharedRLModel, GaussianLayer, DeterministicLayer # noqa: F401, F403
+from .source.modules import MLP # noqa: F401, F403

rlmodule-0.0.5/rlmodule/skrl/examples/gymnasium/skrl_gymnasium_pendulum.py

@@ -0,0 +1,120 @@
+import gymnasium as gym
+
+import torch
+import torch.nn as nn
+
+from rlmodule import SharedRLModel, GaussianLayer, DeterministicLayer, MLP
+
+# import the skrl components to build the RL system
+from skrl.agents.torch.ppo import PPO, PPO_DEFAULT_CONFIG
+
+from skrl.envs.wrappers.torch.gymnasium_envs import GymnasiumWrapper
+#from skrl.envs.wrappers.torch import wrap_env
+from skrl.memories.torch import RandomMemory
+from skrl.resources.preprocessors.torch import RunningStandardScaler
+from skrl.resources.schedulers.torch import KLAdaptiveRL
+from skrl.trainers.torch import SequentialTrainer
+from skrl.utils import set_seed
+from skrl.envs.wrappers.torch.gymnasium_envs import GymnasiumWrapper
+
+# todo set seed
+# seed for reproducibility
+set_seed()  # e.g. `set_seed(42)` for fixed seed
+
+def get_shared_model(env):
+    # instantiate the agent's models (function approximators).
+    params = {'input_size': env.observation_space.shape[0], 'hidden_units': [64, 64, 64], 'activations': ['relu', 'relu', 'relu']}
+    net = MLP(params)
+    
+    model = SharedRLModel(
+        observation_space=env.observation_space.shape[0], # TODO not do shape[0] but process box normally
+        action_space=env.observation_space.shape[0],
+        device=device,
+        network=net,
+        output_layer={
+            'policy': GaussianLayer(
+                input_size=64,  # TODO
+                output_size=env.action_space.shape[0],
+                device=device,
+                output_activation=nn.Tanh(),
+                clip_actions=False,
+                clip_log_std=True,
+                min_log_std=-1.2,
+                max_log_std=2,
+                initial_log_std=0.2,
+            ),
+            'value': DeterministicLayer(
+                input_size=64,
+                output_size=1,
+                device=device,
+                output_activation=nn.Identity(),
+            ),
+        },
+    ).to(device) # TODO careful about device
+
+    return {'policy': model, 'value': model}
+
+
+# load and wrap the gymnasium environment.
+# note: the environment version may change depending on the gymnasium version
+try:
+    env = gym.vector.make("Pendulum-v1", num_envs=4, asynchronous=False)
+except (gym.error.DeprecatedEnv, gym.error.VersionNotFound) as e:
+    env_id = [spec for spec in gym.envs.registry if spec.startswith("Pendulum-v")][0]
+    print("Pendulum-v1 not found. Trying {}".format(env_id))
+    env = gym.vector.make(env_id, num_envs=4, asynchronous=False)
+
+env = GymnasiumWrapper(env)
+
+device = env.device
+
+
+# instantiate a memory as rollout buffer (any memory can be used for this)
+memory = RandomMemory(memory_size=1024, num_envs=env.num_envs, device=device)
+
+models = get_shared_model(env)
+
+print(models)
+
+
+# configure and instantiate the agent (visit its documentation to see all the options)
+# https://skrl.readthedocs.io/en/latest/api/agents/ppo.html#configuration-and-hyperparameters
+cfg = PPO_DEFAULT_CONFIG.copy()
+cfg["rollouts"] = 1024  # memory_size
+cfg["learning_epochs"] = 10
+cfg["mini_batches"] = 32
+cfg["discount_factor"] = 0.9
+cfg["lambda"] = 0.95
+cfg["learning_rate"] = 1e-3
+cfg["learning_rate_scheduler"] = KLAdaptiveRL
+cfg["learning_rate_scheduler_kwargs"] = {"kl_threshold": 0.008}
+cfg["grad_norm_clip"] = 0.5
+cfg["ratio_clip"] = 0.2
+cfg["value_clip"] = 0.2
+cfg["clip_predicted_values"] = False
+cfg["entropy_loss_scale"] = 0.0
+cfg["value_loss_scale"] = 0.5
+cfg["kl_threshold"] = 0
+cfg["state_preprocessor"] = RunningStandardScaler
+cfg["state_preprocessor_kwargs"] = {"size": env.observation_space, "device": device}
+cfg["value_preprocessor"] = RunningStandardScaler
+cfg["value_preprocessor_kwargs"] = {"size": 1, "device": device}
+# logging to TensorBoard and write checkpoints (in timesteps)
+cfg["experiment"]["write_interval"] = 500
+cfg["experiment"]["checkpoint_interval"] = 5000
+cfg["experiment"]["directory"] = "runs/torch/Pendulum"
+
+agent = PPO(models=models,
+            memory=memory,
+            cfg=cfg,
+            observation_space=env.observation_space,
+            action_space=env.action_space,
+            device=device)
+
+
+# configure and instantiate the RL trainer
+cfg_trainer = {"timesteps": 100000, "headless": True}
+trainer = SequentialTrainer(cfg=cfg_trainer, env=env, agents=[agent])
+
+# start training
+trainer.train()

rlmodule-0.0.5/rlmodule/skrl/examples/gymnasium/skrl_gymnasium_pendulum_no_module.py

@@ -0,0 +1,123 @@
+import gymnasium as gym
+
+import torch
+import torch.nn as nn
+
+# import the skrl components to build the RL system
+from skrl.agents.torch.ppo import PPO, PPO_DEFAULT_CONFIG
+
+from skrl.envs.wrappers.torch.gymnasium_envs import GymnasiumWrapper
+# from skrl.envs.wrappers.torch import wrap_env
+from skrl.memories.torch import RandomMemory
+from skrl.models.torch import DeterministicMixin, GaussianMixin, Model
+from skrl.resources.preprocessors.torch import RunningStandardScaler
+from skrl.resources.schedulers.torch import KLAdaptiveRL
+from skrl.trainers.torch import SequentialTrainer
+from skrl.utils import set_seed
+
+
+# seed for reproducibility
+set_seed()  # e.g. `set_seed(42)` for fixed seed
+
+
+# define models (stochastic and deterministic models) using mixins
+class Policy(GaussianMixin, Model):
+    def __init__(self, observation_space, action_space, device, clip_actions=False,
+                 clip_log_std=True, min_log_std=-20, max_log_std=2, reduction="sum"):
+        Model.__init__(self, observation_space, action_space, device)
+        GaussianMixin.__init__(self, clip_actions, clip_log_std, min_log_std, max_log_std, reduction)
+
+        self.net = nn.Sequential(nn.Linear(self.num_observations, 64),
+                                 nn.ReLU(),
+                                 nn.Linear(64, 64),
+                                 nn.ReLU(),
+                                 nn.Linear(64, self.num_actions))
+        self.log_std_parameter = nn.Parameter(torch.zeros(self.num_actions))
+
+    def compute(self, inputs, role):
+        # Pendulum-v1 action_space is -2 to 2
+        return 2 * torch.tanh(self.net(inputs["states"])), self.log_std_parameter, {}
+
+class Value(DeterministicMixin, Model):
+    def __init__(self, observation_space, action_space, device, clip_actions=False):
+        Model.__init__(self, observation_space, action_space, device)
+        DeterministicMixin.__init__(self, clip_actions)
+
+        self.net = nn.Sequential(nn.Linear(self.num_observations, 64),
+                                 nn.ReLU(),
+                                 nn.Linear(64, 64),
+                                 nn.ReLU(),
+                                 nn.Linear(64, 1))
+
+    def compute(self, inputs, role):
+        return self.net(inputs["states"]), {}
+
+
+# load and wrap the gymnasium environment.
+# note: the environment version may change depending on the gymnasium version
+try:
+    env = gym.vector.make("Pendulum-v1", num_envs=4, asynchronous=False)
+except (gym.error.DeprecatedEnv, gym.error.VersionNotFound) as e:
+    env_id = [spec for spec in gym.envs.registry if spec.startswith("Pendulum-v")][0]
+    print("Pendulum-v1 not found. Trying {}".format(env_id))
+    env = gym.vector.make(env_id, num_envs=4, asynchronous=False)
+
+env = GymnasiumWrapper(env)
+
+device = env.device
+
+
+# instantiate a memory as rollout buffer (any memory can be used for this)
+memory = RandomMemory(memory_size=1024, num_envs=env.num_envs, device=device)
+
+
+# instantiate the agent's models (function approximators).
+# PPO requires 2 models, visit its documentation for more details
+# https://skrl.readthedocs.io/en/latest/api/agents/ppo.html#models
+models = {}
+models["policy"] = Policy(env.observation_space, env.action_space, device, clip_actions=True)
+models["value"] = Value(env.observation_space, env.action_space, device)
+
+print(models)
+
+# configure and instantiate the agent (visit its documentation to see all the options)
+# https://skrl.readthedocs.io/en/latest/api/agents/ppo.html#configuration-and-hyperparameters
+cfg = PPO_DEFAULT_CONFIG.copy()
+cfg["rollouts"] = 1024  # memory_size
+cfg["learning_epochs"] = 10
+cfg["mini_batches"] = 32
+cfg["discount_factor"] = 0.9
+cfg["lambda"] = 0.95
+cfg["learning_rate"] = 1e-3
+cfg["learning_rate_scheduler"] = KLAdaptiveRL
+cfg["learning_rate_scheduler_kwargs"] = {"kl_threshold": 0.008}
+cfg["grad_norm_clip"] = 0.5
+cfg["ratio_clip"] = 0.2
+cfg["value_clip"] = 0.2
+cfg["clip_predicted_values"] = False
+cfg["entropy_loss_scale"] = 0.0
+cfg["value_loss_scale"] = 0.5
+cfg["kl_threshold"] = 0
+cfg["state_preprocessor"] = RunningStandardScaler
+cfg["state_preprocessor_kwargs"] = {"size": env.observation_space, "device": device}
+cfg["value_preprocessor"] = RunningStandardScaler
+cfg["value_preprocessor_kwargs"] = {"size": 1, "device": device}
+# logging to TensorBoard and write checkpoints (in timesteps)
+cfg["experiment"]["write_interval"] = 500
+cfg["experiment"]["checkpoint_interval"] = 5000
+cfg["experiment"]["directory"] = "runs/torch/Pendulum"
+
+agent = PPO(models=models,
+            memory=memory,
+            cfg=cfg,
+            observation_space=env.observation_space,
+            action_space=env.action_space,
+            device=device)
+
+
+# configure and instantiate the RL trainer
+cfg_trainer = {"timesteps": 100000, "headless": True}
+trainer = SequentialTrainer(cfg=cfg_trainer, env=env, agents=[agent])
+
+# start training
+trainer.train()