gr-libs 0.2.2__py3-none-any.whl → 0.2.5__py3-none-any.whl

gr_libs/_evaluation/_generate_experiments_results.py

@@ -1,141 +0,0 @@
-import copy
-import os
-
-import dill
-import matplotlib.pyplot as plt
-import numpy as np
-
-from gr_libs.ml.utils.storage import (
-    get_experiment_results_path,
-    set_global_storage_configs,
-)
-
-
-def gen_graph(
-    graph_name,
-    x_label_str,
-    tasks,
-    panda_env,
-    minigrid_env,
-    parking_env,
-    maze_env,
-    percentage,
-):
-
-    fragmented_accuracies = {
-        "graml": {
-            #'panda': [],
-            #'minigrid': [],
-            #'point_maze': [],
-            "parking": []
-        },
-        "graql": {
-            #'panda': [],
-            #'minigrid': [],
-            #'point_maze': [],
-            "parking": []
-        },
-    }
-
-    continuing_accuracies = copy.deepcopy(fragmented_accuracies)
-
-    # domains_envs = [('minigrid', minigrid_env), ('point_maze', maze_env), ('parking', parking_env)]
-    domains_envs = [("parking", parking_env)]
-
-    for partial_obs_type, accuracies, is_same_learn in zip(
-        ["fragmented", "continuing"],
-        [fragmented_accuracies, continuing_accuracies],
-        [False, True],
-    ):
-        for domain, env in domains_envs:
-            for task in tasks:
-                set_global_storage_configs(
-                    recognizer_str="graml",
-                    is_fragmented=partial_obs_type,
-                    is_inference_same_length_sequences=True,
-                    is_learn_same_length_sequences=is_same_learn,
-                )
-                graml_res_file_path = (
-                    f"{get_experiment_results_path(domain, env, task)}.pkl"
-                )
-                set_global_storage_configs(
-                    recognizer_str="graql", is_fragmented=partial_obs_type
-                )
-                graql_res_file_path = (
-                    f"{get_experiment_results_path(domain, env, task)}.pkl"
-                )
-                if os.path.exists(graml_res_file_path):
-                    with open(graml_res_file_path, "rb") as results_file:
-                        results = dill.load(results_file)
-                        accuracies["graml"][domain].append(
-                            results[percentage]["accuracy"]
-                        )
-                else:
-                    assert False, f"no file for {graml_res_file_path}"
-                if os.path.exists(graql_res_file_path):
-                    with open(graql_res_file_path, "rb") as results_file:
-                        results = dill.load(results_file)
-                        accuracies["graql"][domain].append(
-                            results[percentage]["accuracy"]
-                        )
-                else:
-                    assert False, f"no file for {graql_res_file_path}"
-
-    def plot_accuracies(accuracies, partial_obs_type):
-        plt.figure(figsize=(10, 6))
-        colors = plt.cm.get_cmap(
-            "tab10", len(accuracies["graml"]) * len(accuracies["graml"]["parking"])
-        )
-
-        # Define different line styles for each algorithm
-        line_styles = {"graml": "-", "graql": "--"}
-        x_vals = np.arange(3, 8)
-        plt.xticks(x_vals)
-        plt.yticks(np.linspace(0, 1, 6))
-        plt.ylim([0, 1])
-        # Plot each domain-env pair's accuracies with different line styles for each algorithm
-        for alg in ["graml", "graql"]:
-            for idx, (domain, acc_values) in enumerate(accuracies[alg].items()):
-                if acc_values and len(acc_values) > 0:  # Only plot if there are values
-                    x_values = np.arange(3, len(acc_values) + 3)
-                    plt.plot(
-                        x_values,
-                        acc_values,
-                        marker="o",
-                        linestyle=line_styles[alg],
-                        color=colors(idx),
-                        label=f"{alg}-{domain}-{partial_obs_type}-{percentage}",
-                    )
-
-        # Set labels, title, and grid
-        plt.xlabel(x_label_str)
-        plt.ylabel("Accuracy")
-        plt.grid(True)
-
-        # Add legend to differentiate between domain-env pairs
-        plt.legend()
-
-        # Save the figure
-        fig_path = os.path.join(f"{graph_name}_{partial_obs_type}.png")
-        plt.savefig(fig_path)
-        print(f"Accuracies figure saved at: {fig_path}")
-
-    print(f"fragmented_accuracies: {fragmented_accuracies}")
-    plot_accuracies(fragmented_accuracies, "fragmented")
-    print(f"continuing_accuracies: {continuing_accuracies}")
-    plot_accuracies(continuing_accuracies, "continuing")
-
-
-if __name__ == "__main__":
-    # gen_graph("increasing_base_goals", "Number of base goals", ['L1', 'L2', 'L3', 'L4', 'L5'], panda_env='gd_agent', minigrid_env='obstacles', parking_env='gd_agent', maze_env='obstacles')
-    # gen_graph("increasing_dynamic_goals", "Number of dynamic goals", ['L1', 'L2', 'L3', 'L4', 'L5'], panda_env='gc_agent', minigrid_env='lava_crossing', parking_env='gc_agent', maze_env='four_rooms')
-    gen_graph(
-        "base_problems",
-        "Number of goals",
-        ["L111", "L222", "L333", "L444", "L555"],
-        panda_env="gd_agent",
-        minigrid_env="obstacles",
-        parking_env="gc_agent",
-        maze_env="obstacles",
-        percentage="0.7",
-    )

gr_libs/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.2.2'
-__version_tuple__ = version_tuple = (0, 2, 2)
+__version__ = version = '0.2.5'
+__version_tuple__ = version_tuple = (0, 2, 5)

gr_libs/all_experiments.py

@@ -1,67 +1,43 @@
 """ executes odgr_executor parallely on a set of problems defined in consts.py """
 
+import argparse
 import concurrent.futures
 import os
 import subprocess
 import sys
-import threading
 
 import dill
 import numpy as np
 
 from gr_libs.ml.utils.storage import get_experiment_results_path
 
-# Define the lists
-# domains = ['minigrid', 'point_maze', 'parking', 'panda']
-# envs = {
-#     'minigrid': ['obstacles', 'lava_crossing'],
-#     'point_maze': ['four_rooms', 'lava_crossing'],
-#     'parking': ['gc_agent', 'gd_agent'],
-#     'panda': ['gc_agent', 'gd_agent']
-# }
-# tasks = {
-#     'minigrid': ['L111', 'L222', 'L333', 'L444', 'L555'],
-#     'point_maze': ['L111', 'L222', 'L333', 'L444', 'L555'],
-#     'parking': ['L111', 'L222', 'L333', 'L444', 'L555'],
-#     'panda': ['L111', 'L222', 'L333', 'L444', 'L555']
-# }
-configs = {
-    "minigrid": {
-        "MiniGrid-SimpleCrossingS13N4": ["L1", "L2", "L3", "L4", "L5"],
-        "MiniGrid-LavaCrossingS9N2": ["L1", "L2", "L3", "L4", "L5"],
-    }
-    # 'point_maze': {
-    #     'PointMaze-FourRoomsEnvDense-11x11': ['L1', 'L2', 'L3', 'L4', 'L5'],
-    #     'PointMaze-ObstaclesEnvDense-11x11': ['L1', 'L2', 'L3', 'L4', 'L5']
-    # }
-    # 'parking': {
-    #     'Parking-S-14-PC-': ['L1', 'L2', 'L3', 'L4', 'L5'],
-    #     'Parking-S-14-PC-': ['L1', 'L2', 'L3', 'L4', 'L5']
-    # }
-    # 'panda': {
-    #     'PandaMyReachDense': ['L1', 'L2', 'L3', 'L4', 'L5'],
-    #     'PandaMyReachDense': ['L1', 'L2', 'L3', 'L4', 'L5']
-    # }
-}
-# for minigrid:
-# TODO assert these instead i the beggingning of the code before beginning
-# with the actual threading
-recognizers = ["ExpertBasedGraml", "Graql"]
-# recognizers = ['Graql']
-
-# for point_maze:
-# recognizers = ['ExpertBasedGraml']
-# recognizers = ['Draco']
-
-# for parking:
-# recognizers = ['GCGraml']
-# recognizers = ['GCDraco']
+parser = argparse.ArgumentParser()
+parser.add_argument("--domains", nargs="+", required=True, help="List of domains")
+parser.add_argument(
+    "--envs",
+    nargs="+",
+    required=True,
+    help="List of environments (same order as domains)",
+)
+parser.add_argument(
+    "--tasks", nargs="+", required=True, help="List of tasks (e.g. L1 L2 L3 L4 L5)"
+)
+parser.add_argument(
+    "--recognizers", nargs="+", required=True, help="List of recognizers"
+)
+parser.add_argument(
+    "--n", type=int, default=5, help="Number of times to execute each task"
+)
+args = parser.parse_args()
 
-# for panda:
-# recognizers = ['GCGraml']
-# recognizers = ['GCDraco']
+# Build configs dynamically
+configs = {}
+for domain, env in zip(args.domains, args.envs):
+    configs.setdefault(domain, {})
+    configs[domain][env] = args.tasks
 
-n = 5  # Number of times to execute each task
+recognizers = args.recognizers
+n = args.n
 
 
 # Function to read results from the result file
@@ -97,40 +73,31 @@ def run_experiment(domain, env, task, recognizer, i, generate_new=False):
     Returns:
         tuple: A tuple containing the experiment details and the results.
     """
-    cmd = f"python gr_libs/odgr_executor.py --domain {domain} --recognizer \
-          {recognizer} --env_name {env} --task {task} --collect_stats"
-    print(f"Starting execution: {cmd}")
+    cmd = f"python gr_libs/odgr_executor.py --domain {domain} --recognizer {recognizer} --env_name {env} --task {task} --collect_stats --experiment_num {i}"
     try:
         res_file_path = get_experiment_results_path(domain, env, task, recognizer)
-        res_file_path_txt = os.path.join(res_file_path, "res.txt")
-        i_res_file_path_txt = os.path.join(res_file_path, f"res_{i}.txt")
-        res_file_path_pkl = os.path.join(res_file_path, "res.pkl")
         i_res_file_path_pkl = os.path.join(res_file_path, f"res_{i}.pkl")
+        i_res_file_path_txt = os.path.join(res_file_path, f"res_{i}.txt")
         if generate_new or (
             not os.path.exists(i_res_file_path_txt)
             or not os.path.exists(i_res_file_path_pkl)
         ):
-            if os.path.exists(i_res_file_path_txt) or os.path.exists(
-                i_res_file_path_pkl
-            ):
-                i_res_file_path_txt = i_res_file_path_txt.replace(f"_{i}", f"_{i}_new")
-                i_res_file_path_pkl = i_res_file_path_pkl.replace(f"_{i}", f"_{i}_new")
-            process = subprocess.Popen(cmd, shell=True)
-            process.wait()
+            process = subprocess.Popen(
+                cmd,
+                shell=True,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                text=True,
+            )
+            stdout, stderr = process.communicate()
             if process.returncode != 0:
-                print(f"Execution failed: {cmd}")
-                print(f"Error: {result.stderr}")
+                print(f"Execution failed: {cmd}\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}")
                 return None
             else:
                 print(f"Finished execution successfully: {cmd}")
-            file_lock = threading.Lock()
-            with file_lock:
-                os.rename(res_file_path_pkl, i_res_file_path_pkl)
-                os.rename(res_file_path_txt, i_res_file_path_txt)
         else:
             print(
-                f"File {i_res_file_path_txt} already exists. Skipping execution \
-                 of {cmd}"
+                f"File {i_res_file_path_txt} already exists. Skipping execution of {cmd}"
             )
         return ((domain, env, task, recognizer), read_results(i_res_file_path_pkl))
     except Exception as e:
@@ -252,43 +219,42 @@ for key, percentage_dict in compiled_accuracies.items():
             std_dev = np.std(accuracies)
             compiled_summary[key][percentage][is_cons] = (avg_accuracy, std_dev)
 
-# Write different summary results to different files
+# Write different summary results to different files, one per recognizer
 if not os.path.exists(os.path.join("outputs", "summaries")):
     os.makedirs(os.path.join("outputs", "summaries"))
-detailed_summary_file_path = os.path.join(
-    "outputs",
-    "summaries",
-    f"detailed_summary_{''.join(configs.keys())}_{recognizers[0]}.txt",
-)
-compiled_summary_file_path = os.path.join(
-    "outputs",
-    "summaries",
-    f"compiled_summary_{''.join(configs.keys())}_{recognizers[0]}.txt",
-)
-with open(detailed_summary_file_path, "w") as f:
-    for key, percentage_dict in detailed_summary.items():
-        domain, env, task, recognizer = key
-        f.write(f"{domain}\t{env}\t{task}\t{recognizer}\n")
-        for percentage, cons_info in percentage_dict.items():
-            for is_cons, (avg_accuracy, std_dev) in cons_info.items():
-                f.write(
-                    f"\t\t{percentage}\t{is_cons}\t{avg_accuracy:.4f}\t{std_dev:.4f}\n"
-                )
 
-with open(compiled_summary_file_path, "w") as f:
-    for key, percentage_dict in compiled_summary.items():
-        for percentage, cons_info in percentage_dict.items():
-            for is_cons, (avg_accuracy, std_dev) in cons_info.items():
-                f.write(
-                    f"{key[0]}\t{key[1]}\t{percentage}\t{is_cons}\t{avg_accuracy:.4f}\t{std_dev:.4f}\n"
-                )
-        domain, recognizer = key
-        f.write(f"{domain}\t{recognizer}\n")
-        for percentage, cons_info in percentage_dict.items():
-            for is_cons, (avg_accuracy, std_dev) in cons_info.items():
-                f.write(
-                    f"\t\t{percentage}\t{is_cons}\t{avg_accuracy:.4f}\t{std_dev:.4f}\n"
-                )
+for recognizer in recognizers:
+    compiled_summary_file_path = os.path.join(
+        "outputs",
+        "summaries",
+        f"compiled_summary_{''.join(configs.keys())}_{recognizer}.txt",
+    )
+    with open(compiled_summary_file_path, "w") as f:
+        for key, percentage_dict in compiled_summary.items():
+            domain, recog = key
+            if recog != recognizer:
+                continue  # Only write results for this recognizer
+            for percentage, cons_info in percentage_dict.items():
+                for is_cons, (avg_accuracy, std_dev) in cons_info.items():
+                    f.write(
+                        f"{domain}\t{recog}\t{percentage}\t{is_cons}\t{avg_accuracy:.4f}\t{std_dev:.4f}\n"
+                    )
+    print(f"Compiled summary results written to {compiled_summary_file_path}")
 
-print(f"Detailed summary results written to {detailed_summary_file_path}")
-print(f"Compiled summary results written to {compiled_summary_file_path}")
+    detailed_summary_file_path = os.path.join(
+        "outputs",
+        "summaries",
+        f"detailed_summary_{''.join(configs.keys())}_{recognizer}.txt",
+    )
+    with open(detailed_summary_file_path, "w") as f:
+        for key, percentage_dict in detailed_summary.items():
+            domain, env, task, recog = key
+            if recog != recognizer:
+                continue  # Only write results for this recognizer
+            f.write(f"{domain}\t{env}\t{task}\t{recog}\n")
+            for percentage, cons_info in percentage_dict.items():
+                for is_cons, (avg_accuracy, std_dev) in cons_info.items():
+                    f.write(
+                        f"\t\t{percentage}\t{is_cons}\t{avg_accuracy:.4f}\t{std_dev:.4f}\n"
+                    )
+    print(f"Detailed summary results written to {detailed_summary_file_path}")

gr_libs/environment/environment.py

@@ -1,8 +1,10 @@
 """ environment.py """
 
 import os
+import sys
 from abc import abstractmethod
 from collections import namedtuple
+from contextlib import contextmanager
 
 import gymnasium as gym
 import numpy as np
@@ -23,6 +25,23 @@ LSTMProperties = namedtuple(
 )
 
 
+@contextmanager
+def suppress_output():
+    """
+    Context manager to suppress stdout and stderr (including C/C++ prints).
+    """
+    with open(os.devnull, "w") as devnull:
+        old_stdout = sys.stdout
+        old_stderr = sys.stderr
+        sys.stdout = devnull
+        sys.stderr = devnull
+        try:
+            yield
+        finally:
+            sys.stdout = old_stdout
+            sys.stderr = old_stderr
+
+
 class EnvProperty:
     """
     Base class for environment properties.
@@ -135,9 +154,10 @@ class EnvProperty:
 
     def create_vec_env(self, kwargs):
         """
-        Create a vectorized environment.
+        Create a vectorized environment, suppressing prints from gym/pybullet/panda-gym.
         """
-        env = gym.make(**kwargs)
+        with suppress_output():
+            env = gym.make(**kwargs)
         return DummyVecEnv([lambda: env])
 
     @abstractmethod

gr_libs/evaluation/generate_experiments_results.py

@@ -0,0 +1,100 @@
+import argparse
+import os
+
+import dill
+import matplotlib.pyplot as plt
+import numpy as np
+
+from gr_libs.ml.utils.storage import get_experiment_results_path
+
+
+def load_results(domain, env, task, recognizer, n_runs, percentage, cons_type):
+    # Collect accuracy for a single task and recognizer
+    accs = []
+    res_dir = get_experiment_results_path(domain, env, task, recognizer)
+    if not os.path.exists(res_dir):
+        return accs
+    for i in range(n_runs):
+        res_file = os.path.join(res_dir, f"res_{i}.pkl")
+        if not os.path.exists(res_file):
+            continue
+        with open(res_file, "rb") as f:
+            results = dill.load(f)
+        if percentage in results and cons_type in results[percentage]:
+            acc = results[percentage][cons_type].get("accuracy")
+            if acc is not None:
+                accs.append(acc)
+    return accs
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--domain", required=True)
+    parser.add_argument("--env", required=True)
+    parser.add_argument("--tasks", nargs="+", required=True)
+    parser.add_argument("--recognizers", nargs="+", required=True)
+    parser.add_argument("--n_runs", type=int, default=5)
+    parser.add_argument("--percentage", required=True)
+    parser.add_argument(
+        "--cons_type", choices=["consecutive", "non_consecutive"], required=True
+    )
+    parser.add_argument("--graph_name", type=str, default="experiment_results")
+    args = parser.parse_args()
+
+    plt.figure(figsize=(7, 5))
+    has_data = False
+    missing_recognizers = []
+
+    for recognizer in args.recognizers:
+        x_vals = []
+        y_means = []
+        y_sems = []
+        for task in args.tasks:
+            accs = load_results(
+                args.domain,
+                args.env,
+                task,
+                recognizer,
+                args.n_runs,
+                args.percentage,
+                args.cons_type,
+            )
+            if accs:
+                x_vals.append(task)
+                y_means.append(np.mean(accs))
+                y_sems.append(np.std(accs) / np.sqrt(len(accs)))
+        if x_vals:
+            has_data = True
+            x_ticks = np.arange(len(x_vals))
+            plt.plot(x_ticks, y_means, marker="o", label=recognizer)
+            plt.fill_between(
+                x_ticks,
+                np.array(y_means) - np.array(y_sems),
+                np.array(y_means) + np.array(y_sems),
+                alpha=0.2,
+            )
+            plt.xticks(x_ticks, x_vals)
+        else:
+            print(
+                f"Warning: No data found for recognizer '{recognizer}' in {args.domain} / {args.env} / {args.percentage} / {args.cons_type}"
+            )
+            missing_recognizers.append(recognizer)
+
+    if not has_data:
+        raise RuntimeError(
+            f"No data found for any recognizer in {args.domain} / {args.env} / {args.percentage} / {args.cons_type}. "
+            f"Missing recognizers: {', '.join(missing_recognizers)}"
+        )
+
+    plt.xlabel("Task")
+    plt.ylabel("Accuracy")
+    plt.title(f"{args.domain} - {args.env} ({args.percentage}, {args.cons_type})")
+    plt.legend()
+    plt.grid(True)
+    fig_path = f"{args.graph_name}_{'_'.join(args.recognizers)}_{args.domain}_{args.env}_{args.percentage}_{args.cons_type}.png"
+    plt.savefig(fig_path)
+    print(f"Figure saved at: {fig_path}")
+
+
+if __name__ == "__main__":
+    main()

gr_libs/ml/neural/deep_rl_learner.py

@@ -5,7 +5,7 @@ from types import MethodType
 import cv2
 import numpy as np
 
-from gr_libs.environment.environment import EnvProperty
+from gr_libs.environment.environment import EnvProperty, suppress_output
 
 if __name__ != "__main__":
     from gr_libs.ml.utils.storage import get_agent_model_dir
@@ -184,12 +184,7 @@ class DeepRLAgent:
         """
         fourcc = cv2.VideoWriter_fourcc("m", "p", "4", "v")
         fps = 30.0
-        # if is_gc:
-        # 	assert goal_idx is not None
-        # 	self.reset_with_goal_idx(goal_idx)
-        # else:
-        # 	assert goal_idx is None
-        self.env.reset()
+        self.safe_env_reset()
         frame_size = (
             self.env.render(mode="rgb_array").shape[1],
             self.env.render(mode="rgb_array").shape[0],
@@ -198,7 +193,7 @@ class DeepRLAgent:
         video_writer = cv2.VideoWriter(video_path, fourcc, fps, frame_size)
         general_done, success_done = False, False
         gc.collect()
-        obs = self.env.reset()
+        obs = self.safe_env_reset()
         self.env_prop.change_goal_to_specific_desired(obs, desired)
         counter = 0
         while not (general_done or success_done):
@@ -209,17 +204,11 @@ class DeepRLAgent:
                 general_done = general_done[0]
             self.env_prop.change_goal_to_specific_desired(obs, desired)
             if "success" in info[0].keys():
-                success_done = info[0][
-                    "success"
-                ]  # make sure the agent actually reached the goal within the max time
+                success_done = info[0]["success"]
             elif "is_success" in info[0].keys():
-                success_done = info[0][
-                    "is_success"
-                ]  # make sure the agent actually reached the goal within the max time
+                success_done = info[0]["is_success"]
             elif "step_task_completions" in info[0].keys():
-                success_done = (
-                    len(info[0]["step_task_completions"]) == 1
-                )  # bug of dummyVecEnv, it removes the episode_task_completions from the info dict.
+                success_done = len(info[0]["step_task_completions"]) == 1
             else:
                 raise NotImplementedError(
                     "no other option for any of the environments."
@@ -270,17 +259,17 @@ class DeepRLAgent:
 
     def safe_env_reset(self):
         """
-        Reset the environment safely.
+        Reset the environment safely, suppressing output.
 
         Returns:
             The initial observation.
         """
         try:
-            obs = self.env.reset()
+            obs = suppress_env_reset(self.env)
         except Exception:
             kwargs = {"id": self.problem_name, "render_mode": "rgb_array"}
             self.env = self.env_prop.create_vec_env(kwargs)
-            obs = self.env.reset()
+            obs = suppress_env_reset(self.env)
         return obs
 
     def get_mean_and_std_dev(self, observation):
@@ -632,3 +621,11 @@ class GCDeepRLAgent(DeepRLAgent):
                 desired=goal_directed_goal,
             )
         return observations
+
+
+def suppress_env_reset(env):
+    """
+    Utility function to suppress prints during env.reset().
+    """
+    with suppress_output():
+        return env.reset()