rl-language 0.1.20__py3-none-win_amd64.whl

command_line/__init__.py

@@ -0,0 +1,15 @@
+#
+# This file is part of the RLC project.
+#
+# RLC is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License version 2 as published by the Free Software Foundation.
+#
+# RLC is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along with RLC. If not, see <https://www.gnu.org/licenses/>.
+#
+from .utils import (
+    make_rlc_argparse,
+    load_simulation_from_args,
+    load_dataset,
+    load_network,
+)

command_line/utils.py

@@ -0,0 +1,81 @@
+#
+# This file is part of the RLC project.
+#
+# RLC is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License version 2 as published by the Free Software Foundation.
+#
+# RLC is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along with RLC. If not, see <https://www.gnu.org/licenses/>.
+#
+from shutil import which
+import argparse
+from loader import Simulation, compile
+import pickle
+from ml import RLCTransformer
+import torch
+
+
+def load_network(path, ntokens, device):
+    if path == "":
+        return RLCTransformer(ntokens=ntokens, device=device)
+    return torch.load(path)
+
+
+def load_dataset(path):
+    to_return = []
+    with open(path, "rb") as f:
+        while True:
+            try:
+                loaded = pickle.load(f)
+                for state, actions_and_scores in loaded:
+                    to_return.append((state, actions_and_scores))
+            except Exception as e:
+                break
+    return to_return
+
+
+def make_rlc_argparse(name, description):
+    parser = argparse.ArgumentParser(name, description=description)
+    parser.add_argument(
+        "source_file",
+        type=str,
+        help="path to .rl source file, or to a python wrapper obtained with rlc --python",
+    )
+    parser.add_argument(
+        "--include",
+        "-i",
+        type=str,
+        nargs="*",
+        help="path to folder where rl files can be found",
+        default=[],
+    )
+    parser.add_argument(
+        "--rlc",
+        "-c",
+        type=str,
+        nargs="?",
+        help="path to rlc compiler",
+        default="rlc",
+    )
+    parser.add_argument(
+        "--runtime",
+        "-rt",
+        type=str,
+        default="",
+        nargs="?",
+        help="path to runtime library",
+    )
+    return parser
+
+
+def load_simulation_from_args(args, optimize=False):
+    assert (
+        which(args.rlc) is not None
+    ), "could not find executable {}, use --rlc <path_to_rlc> to configure it".format(
+        args.rlc
+    )
+
+    if args.source_file.endswith(".py"):
+      return Simulation(args.source_file)
+    else:
+      return compile(args.source_file, args.rlc, args.include, args.runtime, optimized=optimize)

loader/__init__.py

@@ -0,0 +1,10 @@
+#
+# This file is part of the RLC project.
+#
+# RLC is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License version 2 as published by the Free Software Foundation.
+#
+# RLC is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along with RLC. If not, see <https://www.gnu.org/licenses/>.
+#
+from .simulation import Simulation, Action, State, compile

loader/simulation.py

@@ -0,0 +1,291 @@
+#
+# Copyright 2024 Massimo Fioravanti
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from importlib import import_module, machinery, util
+import os
+import inspect
+from math import log2, ceil, floor
+from pathlib import Path
+from collections import defaultdict
+from tempfile import TemporaryDirectory
+from subprocess import run
+from ctypes import Structure, Array
+from sys import stdout, stderr
+
+loaded_libs = {}
+
+def import_file(name, file_path):
+    if name in loaded_libs:
+        return loaded_libs[name]
+    loader = machinery.SourceFileLoader(name, file_path)
+    spec = util.spec_from_loader(name, loader)
+    mod = util.module_from_spec(spec)
+    loader.exec_module(mod)
+    loaded_libs[name] = mod
+    return mod
+
+
+class Action:
+    def __init__(self, action, simulation):
+        self.action = action
+        self.simulation = simulation
+        self.functions = simulation.module.functions
+
+    def __str__(self) -> str:
+        return self.simulation.to_python_string(self.functions.to_string(self.action))
+
+    def to_bytes(self) -> bytes:
+        result = self.functions.as_byte_vector(self.action)
+        real_content = []
+        for i in range(getattr(result, "__size")):
+            real_content.append(getattr(result, "__data")[i] + 128)
+        return bytes(real_content)
+
+    def can_run(self, state) -> bool:
+        return self.functions.can_apply(self.action, state.state).value
+
+    def run(self, state):
+        self.functions.apply(self.action, state.state)
+        return state
+
+    def copy(self):
+        return Action(self.state.copy(), self.simulation)
+
+
+class ActionType:
+    def __init__(self, action_type, action_id, simulation):
+        self.simulation = simulation
+        self.action_type = action_type
+        self.action_id = action_id
+
+    def get_arg_min_maxs(self):
+        return [(0, 3), (0, 3)]
+
+    def materialize(self, *args):
+        empty_action = Action(self.simulation.any_action(), self.simulation)
+        current_action = self.action_type()
+        for (name, type), value in zip(getattr(current_action, "_fields_"), args):
+            setattr(current_action, name, value)
+
+        self.simulation.module.functions.assign(empty_action.action, current_action)
+        return empty_action
+
+
+class State:
+    def __init__(self, simulation, state):
+        self.simulation = simulation
+        self.state = state
+
+    def copy(self):
+        return State(self.simulation, self.state.copy())
+
+    def is_done(self) -> bool:
+        return self.state.resume_index == -1
+
+    def __str__(self) -> str:
+        return self.simulation.to_python_string(
+            self.simulation.module.functions.to_string(self.state)
+        )
+
+    def from_string(self, string: str) -> bool:
+        rl_string = self.simulation.to_rl_string(str)
+        return self.simulation.module.functions.from_string(self.state, rl_string)
+
+    @property
+    def actions(self) -> [Action]:
+        return self.simulation.get_actions()
+
+    def as_byte_vector(self):
+        result = self.simulation.module.functions.as_byte_vector(self.state)
+        real_content = []
+        for i in range(getattr(result, "__size")):
+            real_content.append(getattr(result, "__data")[i] + 128)
+        return bytes(real_content)
+
+    def from_byte_vector(self, byte_vector):
+        vector = self.simulation.module.VectorTint8_tT()
+        for byte in byte_vector:
+            self.simulation.module.functions.append(vector, byte - 128)
+        self.simulation.module.functions.from_byte_vector(self.state, vector)
+
+    def write_binary(self, path: str):
+        with open(path, mode="wb") as file:
+            file.write(self.as_byte_vector())
+            file.flush()
+
+    def load_binary(self, path: str):
+        with open(path, mode="rb") as file:
+            bytes = file.read()
+            self.from_byte_vector(bytes)
+
+    def load(self, path: str) -> bool:
+        with open(path, mode="r") as file:
+            bytes = file.read()
+            return self.from_string(bytes)
+
+    def score(self):
+        return self.state.score
+
+
+class Simulation:
+    def __init__(self, wrapper: str, tmp_dir=None):
+        self.wrapper_path = wrapper
+        self.module = import_file("sim", wrapper)
+        self.tmp_dir = tmp_dir
+
+        if "play" in self.module.actionToAnyFunctionType:
+            self.any_action = self.module.actionToAnyFunctionType["play"]
+            any_action_union = getattr(self.any_action(), "content")
+            alternatives = getattr(any_action_union, "_fields_")
+
+            self.actions = [
+                ActionType(action_type, i, self)
+                for (i, (_, action_type)) in enumerate(alternatives)
+            ]
+
+    def get_class(self, name):
+        return getattr(self.module, name)
+
+    def get_all_actions(self):
+        action = self.module.AnyGameAction()
+        actions = self.module.functions.enumerate(action)
+        result = []
+        for i in range(self.module.functions.size(actions)):
+            copy = self.module.AnyGameAction()
+            self.module.functions.assign(copy, self.module.functions.get(actions, i).contents)
+            result.append(copy)
+
+
+        return result
+
+    def valid_actions(self, state):
+        # Convert NumPy arrays to nested tuples to make them hashable.
+        x = []
+        for action in self.get_all_actions():
+            if self.module.functions.can_apply_impl(action, state):
+                x.append(action)
+        return x
+
+    def can_apply_action(self, action, state):
+        return self.module.functions.can_apply_impl(action, state)
+
+
+    def start(self, name: str, *args):
+        return State(self, getattr(self.module.functions, name)(*args))
+
+    def parse_action(self, action: str) -> Action:
+        any_action = self.module.actionToAnyFunctionType["play"]()
+        rl_string = self.to_rl_string(action)
+        if self.module.functions.from_string(any_action, rl_string):
+            return Action(any_action, self)
+        else:
+            return None
+
+    def parse_actions_from_binary_buffer(self, actions: bytes) -> [list]:
+        vector = self._bytes_to_byte_vector(actions)
+        any_action = self.module.actionToAnyFunctionType["play"]()
+        vec = self.module.functions.parse_actions(any_action, vector)
+        to_return = []
+        for i in range(getattr(vec, "__size")):
+            to_return.append(getattr(vec, "__data")[i])
+        return to_return
+
+    def _bytes_to_byte_vector(self, byte_vector: bytes):
+        vector = self.module.VectorTint8_tT()
+        for byte in byte_vector:
+            self.module.functions.append(vector, byte)
+        return vector
+
+    @property
+    def action_names(self) -> [str]:
+        return [name for name in self.module.actions.keys()]
+
+    def dump(self):
+        for name in self.action_names:
+            print(name)
+
+    def to_rl_string(self, string):
+        return self.module.rl_s__strlit_r_String(string)
+
+    def to_python_string(self, string):
+        first_character = getattr(getattr(string, "__data"), "__data")
+        return self.module.cast(first_character, self.module.c_char_p).value.decode(
+            "utf-8"
+        )
+
+    def action_from_string(self, string: str) -> Action:
+        rl_string = self.to_rl_string(str)
+        action = self.any_action()
+        if not self.module.functions.from_string(action, rl_string):
+            return None
+        return Action(action, self)
+
+    def action_from_byte_vector(self, byte_vector) -> Action:
+        action = self.any_action()
+        vector = self.module.VectorTint8_tT()
+        for byte in byte_vector:
+            self.module.functions.append(vector, byte - 128)
+        if not self.module.functions.parse_action_optimized(action, vector, 0):
+            return None
+        return Action(action, self)
+
+    def __enter__(self):
+        return self
+
+    def cleanup(self):
+        self.__exit__()
+
+    def __exit__(self, *args):
+        import ctypes
+        import _ctypes
+        libHandle = self.module.lib._handle
+
+        if self.tmp_dir is not None:
+          if os.name == "nt":
+            del self.module
+            _ctypes.FreeLibrary(libHandle)
+          self.tmp_dir.cleanup()
+
+    @property
+    def functions(self):
+        return self.module.functions
+
+
+def compile(source, rlc_compiler="rlc", rlc_includes=[], rlc_runtime_lib="", optimized=True):
+    include_args = []
+    for arg in rlc_includes:
+        include_args.append("-i")
+        include_args.append(arg)
+    tmp_dir = TemporaryDirectory(ignore_cleanup_errors=True)
+    assert (
+        run(
+            [
+                rlc_compiler,
+                source,
+                "--python",
+                "-o",
+                Path(tmp_dir.name) / Path("wrapper.py"),
+                "-O2" if optimized else "",
+            ]
+            + include_args
+        ).returncode
+        == 0
+    )
+    lib_name = "lib.dll" if os.name == "nt" else "lib.so"
+    args = [rlc_compiler, source, "--shared", "-o", Path(tmp_dir.name) / Path(lib_name), "-O2" if optimized else ""]
+    if rlc_runtime_lib != "":
+        args = args + ["--runtime-lib", rlc_runtime_lib]
+    assert run(args + include_args).returncode == 0
+    return Simulation(str(Path(tmp_dir.name) / Path("wrapper.py")), tmp_dir)

ml/__init__.py

@@ -0,0 +1,12 @@
+#
+# This file is part of the RLC project.
+#
+# RLC is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License version 2 as published by the Free Software Foundation.
+#
+# RLC is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along with RLC. If not, see <https://www.gnu.org/licenses/>.
+#
+from .training_data import TrainingData
+from .transformer import RLCTransformer, run_once, next_probabilities
+from .action_oracle import ActionOracle

ml/action_oracle.py

@@ -0,0 +1,61 @@
+#
+# Copyright 2024 Massimo Fioravanti
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from loader import State
+import torch
+from ml import RLCTransformer, run_once
+
+
+class ActionOracle:
+    def __init__(self, transformer: RLCTransformer, action_table):
+        self.transformer = transformer
+        self.action_table = action_table
+
+    def __call__(self, states: State):
+        non_null_states = [state for state in states if state is not None]
+        tensor = (
+            torch.IntTensor(
+                [[int(x) for x in state.as_byte_vector()] for state in non_null_states]
+            )
+            .t()
+            .to(self.transformer.device)
+        )
+        result_bytes = run_once(
+            self.transformer.model,
+            tensor,
+            True,
+            3,
+            self.transformer.device,
+            self.transformer.ntokens,
+        )
+
+        result = []
+        index = 0
+        for state in states:
+            if state is None:
+                result.append(None)
+            else:
+                num = result_bytes[0, index]
+                num = num % len(self.action_table)
+                if num >= len(self.action_table) or num < 0:
+                    result.append(None)
+                else:
+                    action = self.action_table[num]
+                    if not action.can_run(state):
+                        result.append(None)
+                    else:
+                        result.append(action)
+                index = index + 1
+        return result

ml/raylib/action_mask.py

@@ -0,0 +1,201 @@
+from ray.rllib.core.rl_module.rl_module import RLModule, RLModuleConfig
+from ray.rllib.policy.sample_batch import SampleBatch
+import torch
+from ray.rllib.utils.torch_utils import FLOAT_MIN
+from ray.rllib.algorithms.ppo.torch.ppo_torch_rl_module import PPOTorchRLModule
+import gymnasium as gym
+from ray.rllib.utils.annotations import override
+from ray.rllib.core.columns import Columns
+from ray.rllib.utils.typing import TensorType
+
+from typing import Dict, Optional, Tuple
+
+
+class ActionMaskRLMBase(RLModule):
+    @override(RLModule)
+    def __init__(self, config: RLModuleConfig):
+        # If observation space is not of type `Dict` raise an error.
+        if not isinstance(config.observation_space, gym.spaces.dict.Dict):
+            raise ValueError(
+                "This RLModule requires the environment to provide a "
+                "`gym.spaces.Dict` observation space of the form: \n"
+                " {'action_mask': Box(0.0, 1.0, shape=(self.action_space.n,)),"
+                "  'observation_space': self.observation_space}"
+            )
+
+        # While the environment holds an observation space that contains, both,
+        # the action mask and the original observation space, the 'RLModule'
+        # receives only the `"observation"` element of the space, but not the
+        # action mask.
+        self.observation_space_with_mask = config.observation_space
+        config.observation_space = config.observation_space["observations"]
+
+        # Keeps track if observation specs have been checked already.
+        self._checked_observations = False
+
+        # The PPORLModule, in its constructor will build networks for the original
+        # observation space (i.e. without the action mask).
+        super().__init__(config)
+
+
+
+def _check_batch(batch):
+    """Check whether the batch contains the required keys."""
+    if "action_mask" not in batch[SampleBatch.OBS]:
+        raise ValueError(
+            "Action mask not found in observation. This model requires "
+            "the environment to provide observations that include an "
+            "action mask (i.e. an observation space of the Dict space "
+            "type that looks as follows: \n"
+            "{'action_mask': Box(0.0, 1.0, shape=(self.action_space.n,)),"
+            "'observations': <observation_space>}"
+        )
+    if "observations" not in batch[SampleBatch.OBS]:
+        raise ValueError(
+            "Observations not found in observation.This model requires "
+            "the environment to provide observations that include a "
+            " (i.e. an observation space of the Dict space "
+            "type that looks as follows: \n"
+            "{'action_mask': Box(0.0, 1.0, shape=(self.action_space.n,)),"
+            "'observations': <observation_space>}"
+        )
+
+
+class TorchActionMaskRLM(ActionMaskRLMBase, PPOTorchRLModule):
+    @override(PPOTorchRLModule)
+    def setup(self):
+        super().setup()
+        # We need to reset here the observation space such that the
+        # super`s (`PPOTorchRLModule`) observation space is the
+        # original space (i.e. without the action mask) and `self`'s
+        # observation space contains the action mask.
+        self.config.observation_space = self.observation_space_with_mask
+
+    @override(PPOTorchRLModule)
+    def setup(self):
+        super().setup()
+        # We need to reset here the observation space such that the
+        # super`s (`PPOTorchRLModule`) observation space is the
+        # original space (i.e. without the action mask) and `self`'s
+        # observation space contains the action mask.
+        self.config.observation_space = self.observation_space_with_mask
+
+    @override(PPOTorchRLModule)
+    def _forward_inference(
+        self, batch: Dict[str, TensorType], **kwargs
+    ) -> Dict[str, TensorType]:
+        # Preprocess the original batch to extract the action mask.
+        action_mask, batch = self._preprocess_batch(batch)
+        # Run the forward pass.
+        outs = super()._forward_inference(batch, **kwargs)
+        # Mask the action logits and return.
+        return self._mask_action_logits(outs, action_mask)
+
+    @override(PPOTorchRLModule)
+    def _forward_exploration(
+        self, batch: Dict[str, TensorType], **kwargs
+    ) -> Dict[str, TensorType]:
+        # Preprocess the original batch to extract the action mask.
+        action_mask, batch = self._preprocess_batch(batch)
+        # Run the forward pass.
+        outs = super()._forward_exploration(batch, **kwargs)
+        # Mask the action logits and return.
+        return self._mask_action_logits(outs, action_mask)
+
+    @override(PPOTorchRLModule)
+    def _forward_train(
+        self, batch: Dict[str, TensorType], **kwargs
+    ) -> Dict[str, TensorType]:
+        # Preprocess the original batch to extract the action mask.
+        action_mask, batch = self._preprocess_batch(batch)
+        # Run the forward pass.
+        outs = super()._forward_train(batch, **kwargs)
+        # Mask the action logits and return.
+        return self._mask_action_logits(outs, action_mask)
+
+    @override(PPOTorchRLModule)
+    def _compute_values(self, batch: Dict[str, TensorType], device):
+        # Preprocess the batch to extract the `observations` to `Columns.OBS`.
+        _, batch = self._preprocess_batch(batch)
+        # Call the super's method to compute values for GAE.
+        return super()._compute_values(batch, device)
+
+    def _preprocess_batch(
+        self, batch: Dict[str, TensorType], **kwargs
+    ) -> Tuple[TensorType, Dict[str, TensorType]]:
+        """Extracts observations and action mask from the batch
+
+        Args:
+            batch: A dictionary containing tensors (at least `Columns.OBS`)
+
+        Returns:
+            A tuple with the action mask tensor and the modified batch containing
+                the original observations.
+        """
+        # Check observation specs for action mask and observation keys.
+        self._check_batch(batch)
+
+        # Extract the available actions tensor from the observation.
+        action_mask = batch[Columns.OBS].pop("action_mask")
+
+        # Modify the batch for the `PPORLModule`'s `forward` method, i.e.
+        # pass only `"obs"` into the `forward` method.
+        batch[Columns.OBS] = batch[Columns.OBS].pop("observations")
+
+        # Return the extracted action mask and the modified batch.
+        return action_mask, batch
+
+    def _check_batch(self, batch: Dict[str, TensorType]) -> Optional[ValueError]:
+        """Assert that the batch includes action mask and observations.
+
+        Args:
+            batch: A dicitonary containing tensors (at least `Columns.OBS`) to be
+                checked.
+
+        Raises:
+            `ValueError` if the column `Columns.OBS`  does not contain observations
+                and action mask.
+        """
+        if not self._checked_observations:
+            if "action_mask" not in batch[Columns.OBS]:
+                raise ValueError(
+                    "No action mask found in observation. This `RLModule` requires "
+                    "the environment to provide observations that include an "
+                    "action mask (i.e. an observation space of the Dict space "
+                    "type that looks as follows: \n"
+                    "{'action_mask': Box(0.0, 1.0, shape=(self.action_space.n,)),"
+                    "'observations': self.observation_space}"
+                )
+            if "observations" not in batch[Columns.OBS]:
+                raise ValueError(
+                    "No observations found in observation. This 'RLModule` requires "
+                    "the environment to provide observations that include the original "
+                    "observations under a key `'observations'` in a dict (i.e. an "
+                    "observation space of the Dict space type that looks as follows: \n"
+                    "{'action_mask': Box(0.0, 1.0, shape=(self.action_space.n,)),"
+                    "'observations': <observation_space>}"
+                )
+            self._checked_observations = True
+
+    def _mask_action_logits(
+        self, batch: Dict[str, TensorType], action_mask: TensorType
+    ) -> Dict[str, TensorType]:
+        """Masks the action logits for the output of `forward` methods
+
+        Args:
+            batch: A dictionary containing tensors (at least action logits).
+            action_mask: A tensor containing the action mask for the current
+                observations.
+
+        Returns:
+            A modified batch with masked action logits for the action distribution
+            inputs.
+        """
+        # Convert action mask into an `[0.0][-inf]`-type mask.
+        inf_mask = torch.clamp(torch.log(action_mask), min=FLOAT_MIN)
+
+        # Mask the logits.
+        batch[Columns.ACTION_DIST_INPUTS] += inf_mask
+
+        # Return the batch with the masked action logits.
+        return batch