rl-language 0.1.0__py3-none-any.whl

command_line/__init__.py

@@ -0,0 +1,15 @@
+#
+# This file is part of the RLC project.
+#
+# RLC is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License version 2 as published by the Free Software Foundation.
+#
+# RLC is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along with RLC. If not, see <https://www.gnu.org/licenses/>.
+#
+from .utils import (
+    make_rlc_argparse,
+    load_simulation_from_args,
+    load_dataset,
+    load_network,
+)

command_line/utils.py

@@ -0,0 +1,93 @@
+#
+# This file is part of the RLC project.
+#
+# RLC is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License version 2 as published by the Free Software Foundation.
+#
+# RLC is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along with RLC. If not, see <https://www.gnu.org/licenses/>.
+#
+from shutil import which
+import argparse
+from loader import Simulation, compile
+import pickle
+from ml import RLCTransformer
+import torch
+
+
+def load_network(path, ntokens, device):
+    if path == "":
+        return RLCTransformer(ntokens=ntokens, device=device)
+    return torch.load(path)
+
+
+def load_dataset(path):
+    to_return = []
+    with open(path, "rb") as f:
+        while True:
+            try:
+                loaded = pickle.load(f)
+                for (state, actions_and_scores) in loaded:
+                    to_return.append((state, actions_and_scores))
+            except Exception as e:
+                break
+    return to_return
+
+
+def make_rlc_argparse(name, description):
+    parser = argparse.ArgumentParser(name, description=description)
+    parser.add_argument(
+        "--wrapper",
+        "-w",
+        type=str,
+        nargs="?",
+        help="path to wrapper to load",
+        default="",
+    )
+    parser.add_argument(
+        "--include",
+        "-i",
+        type=str,
+        nargs="*",
+        help="path to folder where rl files can be found",
+        default=[],
+    )
+    parser.add_argument(
+        "--rlc",
+        "-c",
+        type=str,
+        nargs="?",
+        help="path to rlc compiler",
+        default="rlc",
+    )
+    parser.add_argument(
+        "--source",
+        "-rl",
+        type=str,
+        nargs="?",
+        help="path to .rl source file",
+    )
+    parser.add_argument(
+        "--runtime",
+        "-rt",
+        type=str,
+        default="",
+        nargs="?",
+        help="path to runtime library",
+    )
+    return parser
+
+
+def load_simulation_from_args(args):
+    assert (
+        which(args.rlc) is not None
+    ), "could not find executable {}, use --rlc <path_to_rlc> to configure it".format(
+        args.rlc
+    )
+
+    sim = (
+        Simulation(args.wrapper)
+        if args.wrapper != ""
+        else compile(args.source, args.rlc, args.include, args.runtime)
+    )
+    return sim

loader/__init__.py

@@ -0,0 +1,10 @@
+#
+# This file is part of the RLC project.
+#
+# RLC is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License version 2 as published by the Free Software Foundation.
+#
+# RLC is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along with RLC. If not, see <https://www.gnu.org/licenses/>.
+#
+from .simulation import Simulation, Action, State, compile

loader/simulation.py

@@ -0,0 +1,237 @@
+#
+# Copyright 2024 Massimo Fioravanti
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from importlib import import_module, machinery, util
+import inspect
+from math import log2, ceil, floor
+from collections import defaultdict
+from tempfile import TemporaryDirectory
+from subprocess import run
+from ctypes import Structure, Array
+from sys import stdout, stderr
+
+
+def import_file(name, file_path):
+    loader = machinery.SourceFileLoader(name, file_path)
+    spec = util.spec_from_loader(name, loader)
+    mod = util.module_from_spec(spec)
+    loader.exec_module(mod)
+    return mod
+
+
+class Action:
+    def __init__(self, action, simulation):
+        self.action = action
+        self.simulation = simulation
+        self.functions = simulation.module.functions
+
+    def __str__(self) -> str:
+        return self.simulation.to_python_string(self.functions.to_string(self.action))
+
+    def to_bytes(self) -> bytes:
+        result = self.functions.as_byte_vector(self.action)
+        real_content = []
+        for i in range(getattr(result, "__size")):
+            real_content.append(getattr(result, "__data")[i] + 128)
+        return bytes(real_content)
+
+    def can_run(self, state) -> bool:
+        return self.functions.can_apply_impl(self.action, state.state)
+
+    def run(self, state):
+        self.functions.apply(self.action, state.state)
+        return state
+
+    def copy(self):
+        return Action(self.state.copy(), self.simulation)
+
+
+class ActionType:
+    def __init__(self, action_type, action_id, simulation):
+        self.simulation = simulation
+        self.action_type = action_type
+        self.action_id = action_id
+
+    def get_arg_min_maxs(self):
+        return [(0, 3), (0, 3)]
+
+    def materialize(self, *args):
+        empty_action = Action(self.simulation.any_action(), self.simulation)
+        current_action = self.action_type()
+        for ((name, type), value) in zip(getattr(current_action, "_fields_"), args):
+            setattr(current_action, name, value)
+
+        self.simulation.module.functions.assign(empty_action.action, current_action)
+        return empty_action
+
+
+class State:
+    def __init__(self, simulation, state):
+        self.simulation = simulation
+        self.state = state
+
+    def copy(self):
+        return State(self.simulation, self.state.copy())
+
+    def is_done(self) -> bool:
+        return self.state.resume_index == -1
+
+    def __str__(self) -> str:
+        return self.simulation.to_python_string(
+            self.simulation.module.functions.to_string(self.state)
+        )
+
+    def from_string(self, string: str) -> bool:
+        rl_string = self.simulation.to_rl_string(str)
+        return self.simulation.module.functions.from_string(self.state, rl_string)
+
+    @property
+    def actions(self) -> [Action]:
+        return self.simulation.get_actions()
+
+    def as_byte_vector(self):
+        result = self.simulation.module.functions.as_byte_vector(self.state)
+        real_content = []
+        for i in range(getattr(result, "__size")):
+            real_content.append(getattr(result, "__data")[i] + 128)
+        return bytes(real_content)
+
+    def from_byte_vector(self, byte_vector):
+        vector = self.simulation.module.VectorTint8_tT()
+        for byte in byte_vector:
+            self.simulation.module.functions.append(vector, byte - 128)
+        self.simulation.module.functions.from_byte_vector(self.state, vector)
+
+    def write_binary(self, path: str):
+        with open(path, mode="wb") as file:
+            file.write(self.as_byte_vector())
+            file.flush()
+
+    def load_binary(self, path: str):
+        with open(path, mode="rb") as file:
+            bytes = file.read()
+            self.from_byte_vector(bytes)
+
+    def load(self, path: str) -> bool:
+        with open(path, mode="r") as file:
+            bytes = file.read()
+            return self.from_string(bytes)
+
+    def score(self):
+        return self.state.score
+
+
+class Simulation:
+    def __init__(self, wrapper: str):
+        self.wrapper_path = wrapper
+        self.module = import_file("sim", wrapper)
+
+        if "play" in self.module.actionToAnyFunctionType:
+            self.any_action = self.module.actionToAnyFunctionType["play"]
+            any_action_union = getattr(self.any_action(), "content")
+            alternatives = getattr(any_action_union, "_fields_")
+
+            self.actions = [
+                ActionType(action_type, i, self)
+                for (i, (_, action_type)) in enumerate(alternatives)
+            ]
+
+    def get_actions(self) -> [Action]:
+        return self.actions
+
+    def start(self, name: str, *args):
+        return State(self, getattr(self.module.functions, name)(*args))
+
+    def parse_action(self, action: str) -> Action:
+        any_action = self.module.actionToAnyFunctionType["play"]()
+        rl_string = self.to_rl_string(action)
+        if self.module.functions.from_string(any_action, rl_string):
+            return Action(any_action, self)
+        else:
+            return None
+
+    def parse_actions_from_binary_buffer(self, actions: bytes) -> [list]:
+        vector = self._bytes_to_byte_vector(actions)
+        any_action = self.module.actionToAnyFunctionType["play"]()
+        vec = self.module.functions.parse_actions(any_action, vector)
+        to_return = []
+        for i in range(getattr(vec, "__size")):
+            to_return.append(getattr(vec, "__data")[i])
+        return to_return
+
+    def _bytes_to_byte_vector(self, byte_vector: bytes):
+        vector = self.module.VectorTint8_tT()
+        for byte in byte_vector:
+            self.module.functions.append(vector, byte)
+        return vector
+
+    @property
+    def action_names(self) -> [str]:
+        return [name for name in self.module.actions.keys()]
+
+    def dump(self):
+        for name in self.action_names:
+            print(name)
+
+    def to_rl_string(self, string):
+        return self.module.rl_s__strlit_r_String(string)
+
+    def to_python_string(self, string):
+        first_character = getattr(getattr(string, "__data"), "__data")
+        return self.module.cast(first_character, self.module.c_char_p).value.decode(
+            "utf-8"
+        )
+
+    def action_from_string(self, string: str) -> Action:
+        rl_string = self.to_rl_string(str)
+        action = self.any_action()
+        if not self.module.functions.from_string(action, rl_string):
+            return None
+        return Action(action, self)
+
+    def action_from_byte_vector(self, byte_vector) -> Action:
+        action = self.any_action()
+        vector = self.module.VectorTint8_tT()
+        for byte in byte_vector:
+            self.module.functions.append(vector, byte - 128)
+        if not self.module.functions.parse_action_optimized(action, vector, 0):
+            return None
+        return Action(action, self)
+
+
+def compile(source, rlc_compiler="rlc", rlc_includes=[], rlc_runtime_lib=""):
+    include_args = []
+    for arg in rlc_includes:
+        include_args.append("-i")
+        include_args.append(arg)
+    with TemporaryDirectory() as tmp_dir:
+        assert (
+            run(
+                [
+                    rlc_compiler,
+                    source,
+                    "--python",
+                    "-o",
+                    "{}/wrapper.py".format(tmp_dir),
+                ]
+                + include_args
+            ).returncode
+            == 0
+        )
+        args = [rlc_compiler, source, "--shared", "-o", "{}/lib.so".format(tmp_dir)]
+        if rlc_runtime_lib != "":
+            args = args + ["--runtime-lib", rlc_runtime_lib]
+        assert run(args + include_args).returncode == 0
+        return Simulation(tmp_dir + "/wrapper.py")

ml/__init__.py

@@ -0,0 +1,12 @@
+#
+# This file is part of the RLC project.
+#
+# RLC is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License version 2 as published by the Free Software Foundation.
+#
+# RLC is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along with RLC. If not, see <https://www.gnu.org/licenses/>.
+#
+from .training_data import TrainingData
+from .transformer import RLCTransformer, run_once, next_probabilities
+from .action_oracle import ActionOracle

ml/action_oracle.py

@@ -0,0 +1,61 @@
+#
+# Copyright 2024 Massimo Fioravanti
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from loader import State
+import torch
+from ml import RLCTransformer, run_once
+
+
+class ActionOracle:
+    def __init__(self, transformer: RLCTransformer, action_table):
+        self.transformer = transformer
+        self.action_table = action_table
+
+    def __call__(self, states: State):
+        non_null_states = [state for state in states if state is not None]
+        tensor = (
+            torch.IntTensor(
+                [[int(x) for x in state.as_byte_vector()] for state in non_null_states]
+            )
+            .t()
+            .to(self.transformer.device)
+        )
+        result_bytes = run_once(
+            self.transformer.model,
+            tensor,
+            True,
+            3,
+            self.transformer.device,
+            self.transformer.ntokens,
+        )
+
+        result = []
+        index = 0
+        for state in states:
+            if state is None:
+                result.append(None)
+            else:
+                num = result_bytes[0, index]
+                num = num % len(self.action_table)
+                if num >= len(self.action_table) or num < 0:
+                    result.append(None)
+                else:
+                    action = self.action_table[num]
+                    if not action.can_run(state):
+                        result.append(None)
+                    else:
+                        result.append(action)
+                index = index + 1
+        return result

ml/raylib/action_mask.py

@@ -0,0 +1,87 @@
+from ray.rllib.core.rl_module.rl_module import RLModule, RLModuleConfig
+from ray.rllib.policy.sample_batch import SampleBatch
+from ray.rllib.algorithms.ppo.torch.ppo_torch_rl_module import PPOTorchRLModule
+
+class ActionMaskRLMBase(RLModule):
+    def __init__(self, config: RLModuleConfig):
+        if not isinstance(config.observation_space, gym.spaces.Dict):
+            super().__init__(config)
+            return
+        # We need to adjust the observation space for this RL Module so that, when
+        # building the default models, the RLModule does not "see" the action mask but
+        # only the original observation space without the action mask. This tricks it
+        # into building models that are compatible with the original observation space.
+        config.observation_space = config.observation_space["observations"]
+
+        # The PPORLModule, in its constructor, will build models for the modified
+        # observation space.
+        super().__init__(config)
+
+def _check_batch(batch):
+    """Check whether the batch contains the required keys."""
+    if "action_mask" not in batch[SampleBatch.OBS]:
+        raise ValueError(
+            "Action mask not found in observation. This model requires "
+            "the environment to provide observations that include an "
+            "action mask (i.e. an observation space of the Dict space "
+            "type that looks as follows: \n"
+            "{'action_mask': Box(0.0, 1.0, shape=(self.action_space.n,)),"
+            "'observations': <observation_space>}"
+        )
+    if "observations" not in batch[SampleBatch.OBS]:
+        raise ValueError(
+            "Observations not found in observation.This model requires "
+            "the environment to provide observations that include a "
+            " (i.e. an observation space of the Dict space "
+            "type that looks as follows: \n"
+            "{'action_mask': Box(0.0, 1.0, shape=(self.action_space.n,)),"
+            "'observations': <observation_space>}"
+        )
+
+
+class TorchActionMaskRLM(ActionMaskRLMBase, PPOTorchRLModule):
+    def _forward_inference(self, batch, **kwargs):
+        return mask_forward_fn_torch(super()._forward_inference, batch, **kwargs)
+
+    def _forward_train(self, batch, *args, **kwargs):
+        return mask_forward_fn_torch(super()._forward_train, batch, **kwargs)
+
+    def _forward_exploration(self, batch, *args, **kwargs):
+        return mask_forward_fn_torch(super()._forward_exploration, batch, **kwargs)
+
+    def _compute_values(self, batch, device=None):
+      _check_batch(batch)
+
+      # Extract the available actions tensor from the observation.
+      action_mask = batch[SampleBatch.OBS]["action_mask"]
+
+      # Modify the incoming batch so that the default models can compute logits and
+      # values as usual.
+      batch[SampleBatch.OBS] = batch[SampleBatch.OBS]["observations"]
+      return super()._compute_values(batch, device)
+
+
+def mask_forward_fn_torch(forward_fn, batch, **kwargs):
+    _check_batch(batch)
+
+    # Extract the available actions tensor from the observation.
+    action_mask = batch[SampleBatch.OBS]["action_mask"]
+
+    # Modify the incoming batch so that the default models can compute logits and
+    # values as usual.
+    batch[SampleBatch.OBS] = batch[SampleBatch.OBS]["observations"]
+
+    outputs = forward_fn(batch, **kwargs)
+
+    # Mask logits
+    logits = outputs[SampleBatch.ACTION_DIST_INPUTS]
+    # Convert action_mask into a [0.0 || -inf]-type mask.
+    inf_mask = torch.clamp(torch.log(action_mask), min=FLOAT_MIN)
+    masked_logits = logits + inf_mask
+
+    # Replace original values with masked values.
+    outputs[SampleBatch.ACTION_DIST_INPUTS] = masked_logits
+
+    return outputs
+
+

ml/raylib/environment.py

@@ -0,0 +1,159 @@
+from ray.rllib.env.multi_agent_env_runner import MultiAgentEnvRunner
+from ray.rllib.core.rl_module.marl_module import (
+    MultiAgentRLModuleSpec,
+    MultiAgentRLModule,
+    MultiAgentRLModuleConfig,
+)
+
+class Specs:
+    def __init__(self, max_steps=100):
+        self.max_episode_steps = max_steps
+        self.id = random.randint(0, 10)
+
+def get_env(wrapper):
+    class RLCEnvironment(MultiAgentEnv):
+
+        def __init__(self, render_mode=None, size=5):
+            action = wrapper.AnyGameAction()
+            self._actions = wrapper.functions.enumerate(action)
+            self.state_size = (
+                wrapper.functions.observation_tensor_size(wrapper.Game()).value * 2
+            )
+            self.actions = []
+            for i in range(wrapper.functions.size(self._actions).value):
+                self.actions.append(wrapper.functions.get(self._actions, i).contents)
+
+            self.num_actions = len(self.actions)
+            self.unwrapper_space = spaces.Dict(
+                {
+                    "observations": spaces.Box(0, 1, shape=(self.state_size,), dtype=int),
+                    "action_mask": spaces.Box(0, 1, shape=(self.num_actions,), dtype=int),
+                }
+            )
+            self.observation_space = spaces.Dict(
+                {i: self.unwrapper_space for i in range(2)}
+            )
+            self.action_space = spaces.Dict(
+                {i: spaces.Discrete(self.num_actions) for i in range(2)}
+            )
+
+            self.spec = Specs(10000)
+
+            self.state = wrapper.functions.play()
+            self.current_score = [
+                wrapper.functions.score(self.state, 0),
+                wrapper.functions.score(self.state, 1),
+            ]
+            self.last_score = self.current_score
+            self._agent_ids = [0, 1]
+            self.num_agents = 2
+            self._skip_env_checking = True
+            super().__init__()
+            self._obs_space_in_preferred_format = True
+            self._action_space_in_preferred_format = True
+
+        @property
+        def legal_actions(self):
+            # Convert NumPy arrays to nested tuples to make them hashable.
+            x = []
+            for i, action in enumerate(self.actions):
+                if wrapper.functions.can_apply_impl(action, self.state):
+                    x.append(1)
+                else:
+                    x.append(0)
+            return np.array(x, dtype=np.int8)
+
+        def legal_actions_list(self):
+            # Convert NumPy arrays to nested tuples to make them hashable.
+            x = []
+            for action in self.actions:
+                if wrapper.functions.can_apply_impl(action, self.state):
+                    x.append(action)
+            return x
+
+        def _get_done_winner(self):
+            # if self.state.resume_index == -1:
+            # if wrapper.functions.three_in_a_line_player(self.state.board, 1):
+            # return (True, 0.0)
+            # if wrapper.functions.three_in_a_line_player(self.state.board, 2):
+            # return (True, 0.0)
+            # return (True, 1.0)
+            is_done = {i: self.state.resume_index == -1 for i in range(2)}
+            is_done["__all__"] = self.state.resume_index == -1
+            scores = {
+                i: (self.current_score[i] - self.last_score[i]) for i in range(2)
+            }
+
+            return is_done, scores         # return (False, 0.0)
+
+        def _get_info(self):
+            done, reward = self._get_done_winner()
+            return {"reward": reward}
+
+        def reset(self, seed=None, options=None):
+            # print("RESET")
+            self.state = wrapper.functions.play()
+            observation = self._current_state()
+            info = self._get_info()
+            self.current_score = [
+                wrapper.functions.score(self.state, 0),
+                wrapper.functions.score(self.state, 1),
+            ]
+            self.last_score = self.current_score
+
+            return observation, info
+
+        def step(self, action):
+            to_apply = action[self.current_player()]
+            # wrapper.functions.print(self.actions[to_apply])
+            if not wrapper.functions.can_apply_impl(self.actions[to_apply], self.state):
+                wrapper.functions.apply(random.choice(self.legal_actions_list()), self.state)
+            else:
+                wrapper.functions.apply(self.actions[to_apply], self.state)
+
+            while self.current_player() == -1:  # random player
+                action = random.choice(self.legal_actions_list())
+                # wrapper.functions.print(action)
+                assert wrapper.functions.can_apply_impl(action, self.state)
+                wrapper.functions.apply(action, self.state)
+
+            self.last_score = self.current_score
+            self.current_score = [
+                wrapper.functions.score(self.state, 0),
+                wrapper.functions.score(self.state, 1),
+            ]
+
+            done, reward = self._get_done_winner()
+            observation = self._current_state()
+            info = self._get_info()
+
+            truncated = {i: False for i in range(2)}
+            truncated["__all__"] = False
+            info["current_player"] = self.current_player
+            return observation, reward, done, truncated, info
+
+        def current_player(self):
+            return wrapper.functions.get_current_player(self.state).value
+
+        def _current_state(self):
+
+            serialized = wrapper.VectorTdoubleT()
+            wrapper.functions.resize(serialized, self.state_size)
+            # seriazed2 = wrapper.functions.as_byte_vector(self.state)
+            wrapper.functions.to_observation_tensor(self.state, serialized)
+
+            vec = np.rint(
+                np.ctypeslib.as_array(
+                    wrapper.functions.get(serialized, 0), shape=(self.state_size,)
+                )
+            ).astype(int)
+            return {
+                i: {
+                    "observations": vec,
+                    "action_mask": self.legal_actions,
+                }
+                for i in range(2)
+            }
+
+    return RLCEnvironment
+