omniopt2 8471__py3-none-any.whl → 9171__py3-none-any.whl

{omniopt2-8471.data → omniopt2-9171.data}/data/bin/.omniopt.py

@@ -2,7 +2,7 @@
 
 #from mayhemmonkey import MayhemMonkey
 #mayhemmonkey = MayhemMonkey()
-#mayhemmonkey.set_function_fail_after_count("open", 201)
+#mayhemmonkey.set_function_fail_after_count("open", 10)
 #mayhemmonkey.set_function_error_rate("open", 0.1)
 #mayhemmonkey.set_function_group_error_rate(["io", "math"], 0.8)
 #mayhemmonkey.install_faulty()
@@ -27,11 +27,15 @@ import inspect
 import tracemalloc
 import resource
 from urllib.parse import urlencode
-
 import psutil
 
 FORCE_EXIT: bool = False
 
+LAST_LOG_TIME: int = 0
+last_msg_progressbar = ""
+last_msg_raw = None
+last_lock_print_debug = threading.Lock()
+
 def force_exit(signal_number: Any, frame: Any) -> Any:
     global FORCE_EXIT
 
@@ -84,6 +88,7 @@ log_nr_gen_jobs: list[int] = []
 generation_strategy_human_readable: str = ""
 oo_call: str = "./omniopt"
 progress_bar_length: int = 0
+worker_usage_file = 'worker_usage.csv'
 
 if os.environ.get("CUSTOM_VIRTUAL_ENV") == "1":
     oo_call = "omniopt"
@@ -99,7 +104,7 @@ joined_valid_occ_types: str = ", ".join(valid_occ_types)
 SUPPORTED_MODELS: list = ["SOBOL", "FACTORIAL", "SAASBO", "BOTORCH_MODULAR", "UNIFORM", "BO_MIXED", "RANDOMFOREST", "EXTERNAL_GENERATOR", "PSEUDORANDOM", "TPE"]
 joined_supported_models: str = ", ".join(SUPPORTED_MODELS)
 
-special_col_names: list = ["arm_name", "generation_method", "trial_index", "trial_status", "generation_node", "idxs", "start_time", "end_time", "run_time", "exit_code", "program_string", "signal", "hostname", "submit_time", "queue_time", "metric_name", "mean", "sem", "worker_generator_uuid"]
+special_col_names: list = ["arm_name", "generation_method", "trial_index", "trial_status", "generation_node", "idxs", "start_time", "end_time", "run_time", "exit_code", "program_string", "signal", "hostname", "submit_time", "queue_time", "metric_name", "mean", "sem", "worker_generator_uuid", "runtime", "status"]
 
 IGNORABLE_COLUMNS: list = ["start_time", "end_time", "hostname", "signal", "exit_code", "run_time", "program_string"] + special_col_names
 
@@ -153,12 +158,6 @@ try:
             message="Ax currently requires a sqlalchemy version below 2.0.*",
         )
 
-        warnings.filterwarnings(
-            "ignore",
-            category=RuntimeWarning,
-            message="coroutine 'start_logging_daemon' was never awaited"
-        )
-
     with spinner("Importing argparse..."):
         import argparse
 
@@ -204,6 +203,9 @@ try:
     with spinner("Importing rich.pretty..."):
         from rich.pretty import pprint
 
+    with spinner("Importing pformat..."):
+        from pprint import pformat
+
     with spinner("Importing rich.prompt..."):
         from rich.prompt import Prompt, FloatPrompt, IntPrompt
 
@@ -237,9 +239,6 @@ try:
     with spinner("Importing uuid..."):
         import uuid
 
-    #with spinner("Importing qrcode..."):
-    #    import qrcode
-
     with spinner("Importing cowsay..."):
         import cowsay
 
@@ -270,6 +269,9 @@ try:
     with spinner("Importing beartype..."):
         from beartype import beartype
 
+    with spinner("Importing rendering stuff..."):
+        from ax.plot.base import AxPlotConfig
+
     with spinner("Importing statistics..."):
         import statistics
 
@@ -381,16 +383,9 @@ def _record_stats(func_name: str, elapsed: float, mem_diff: float, mem_after: fl
         call_path_str = " -> ".join(short_stack)
         _func_call_paths[func_name][call_path_str] += 1
 
-        print(
-            f"Function '{func_name}' took {elapsed:.4f}s "
-            f"(total {percent_if_added:.1f}% of tracked time)"
-        )
-        print(
-            f"Memory before: {mem_after - mem_diff:.2f} MB, after: {mem_after:.2f} MB, "
-            f"diff: {mem_diff:+.2f} MB, peak during call: {mem_peak:.2f} MB"
-        )
+        print(f"Function '{func_name}' took {elapsed:.4f}s (total {percent_if_added:.1f}% of tracked time)")
+        print(f"Memory before: {mem_after - mem_diff:.2f} MB, after: {mem_after:.2f} MB, diff: {mem_diff:+.2f} MB, peak during call: {mem_peak:.2f} MB")
 
-        # NEU: Runtime Stats
         runtime_stats = collect_runtime_stats()
         print("=== Runtime Stats ===")
         print(f"RSS: {runtime_stats['rss_MB']:.2f} MB, VMS: {runtime_stats['vms_MB']:.2f} MB")
@@ -502,6 +497,24 @@ try:
         dier: FunctionType = helpers.dier
         is_equal: FunctionType = helpers.is_equal
         is_not_equal: FunctionType = helpers.is_not_equal
+    with spinner("Importing pareto..."):
+        pareto_file: str = f"{script_dir}/.pareto.py"
+        spec = importlib.util.spec_from_file_location(
+            name="pareto",
+            location=pareto_file,
+        )
+        if spec is not None and spec.loader is not None:
+            pareto = importlib.util.module_from_spec(spec)
+            spec.loader.exec_module(pareto)
+        else:
+            raise ImportError(f"Could not load module from {pareto_file}")
+
+        pareto_front_table_filter_rows: FunctionType = pareto.pareto_front_table_filter_rows
+        pareto_front_table_add_headers: FunctionType = pareto.pareto_front_table_add_headers
+        pareto_front_table_add_rows: FunctionType = pareto.pareto_front_table_add_rows
+        pareto_front_filter_complete_points: FunctionType = pareto.pareto_front_filter_complete_points
+        pareto_front_select_pareto_points: FunctionType = pareto.pareto_front_select_pareto_points
+
 except KeyboardInterrupt:
     print("You pressed CTRL-c while importing the helpers file")
     sys.exit(0)
@@ -544,6 +557,17 @@ logfile_worker_creation_logs: str = f'{log_uuid_dir}_worker_creation_logs'
 logfile_trial_index_to_param_logs: str = f'{log_uuid_dir}_trial_index_to_param_logs'
 LOGFILE_DEBUG_GET_NEXT_TRIALS: Union[str, None] = None
 
+def error_without_print(text: str) -> None:
+    print_debug(text)
+
+    if get_current_run_folder():
+        try:
+            with open(get_current_run_folder("oo_errors.txt"), mode="a", encoding="utf-8") as myfile:
+                myfile.write(text + "\n\n")
+        except (OSError, FileNotFoundError) as e:
+            helpers.print_color("red", f"Error: {e}. This may mean that the {get_current_run_folder()} was deleted during the run. Could not write '{text} to {get_current_run_folder()}/oo_errors.txt'")
+            sys.exit(99)
+
 def print_red(text: str) -> None:
     helpers.print_color("red", text)
 
@@ -802,6 +826,7 @@ class ConfigLoader:
     parameter: Optional[List[str]]
     experiment_constraints: Optional[List[str]]
     main_process_gb: int
+    beartype: bool
     worker_timeout: int
     slurm_signal_delay_s: int
     gridsearch: bool
@@ -822,6 +847,7 @@ class ConfigLoader:
     run_program_once: str
     mem_gb: int
     flame_graph: bool
+    memray: bool
     continue_previous_job: Optional[str]
     calculate_pareto_front_of_job: Optional[List[str]]
     revert_to_random_when_seemingly_exhausted: bool
@@ -983,6 +1009,7 @@ class ConfigLoader:
         debug.add_argument('--verbose_break_run_search_table', help='Verbose logging for break_run_search', action='store_true', default=False)
         debug.add_argument('--debug', help='Enable debugging', action='store_true', default=False)
         debug.add_argument('--flame_graph', help='Enable flame-graphing. Makes everything slower, but creates a flame graph', action='store_true', default=False)
+        debug.add_argument('--memray', help='Use memray to show memory usage', action='store_true', default=False)
         debug.add_argument('--no_sleep', help='Disables sleeping for fast job generation (not to be used on HPC)', action='store_true', default=False)
         debug.add_argument('--tests', help='Run simple internal tests', action='store_true', default=False)
         debug.add_argument('--show_worker_percentage_table_at_end', help='Show a table of percentage of usage of max worker over time', action='store_true', default=False)
@@ -996,8 +1023,8 @@ class ConfigLoader:
         debug.add_argument('--runtime_debug', help='Logs which functions use most of the time', action='store_true', default=False)
         debug.add_argument('--debug_stack_regex', help='Only print debug messages if call stack matches any regex', type=str, default='')
         debug.add_argument('--debug_stack_trace_regex', help='Show compact call stack with arrows if any function in stack matches regex', type=str, default=None)
-
         debug.add_argument('--show_func_name', help='Show func name before each execution and when it is done', action='store_true', default=False)
+        debug.add_argument('--beartype', help='Use beartype', action='store_true', default=False)
 
     def load_config(self: Any, config_path: str, file_format: str) -> dict:
         if not os.path.isfile(config_path):
@@ -1235,11 +1262,15 @@ for _rn in args.result_names:
         _key = _rn
         _min_or_max = __default_min_max
 
+    _min_or_max = re.sub(r"'", "", _min_or_max)
+
     if _min_or_max not in ["min", "max"]:
         if _min_or_max:
             print_yellow(f"Value for determining whether to minimize or maximize was neither 'min' nor 'max' for key '{_key}', but '{_min_or_max}'. It will be set to the default, which is '{__default_min_max}' instead.")
         _min_or_max = __default_min_max
 
+    _key = re.sub(r"'", "", _key)
+
     if _key in arg_result_names:
         console.print(f"[red]The --result_names option '{_key}' was specified multiple times![/]")
         sys.exit(50)
@@ -1340,27 +1371,14 @@ try:
     with spinner("Importing ExternalGenerationNode..."):
         from ax.generation_strategy.external_generation_node import ExternalGenerationNode
 
-    with spinner("Importing MaxTrials..."):
-        from ax.generation_strategy.transition_criterion import MaxTrials
+    with spinner("Importing MinTrials..."):
+        from ax.generation_strategy.transition_criterion import MinTrials
 
     with spinner("Importing GeneratorSpec..."):
         from ax.generation_strategy.generator_spec import GeneratorSpec
 
-    #except Exception:
-    #    with spinner("Fallback: Importing ax.generation_strategy.generation_node..."):
-    #        import ax.generation_strategy.generation_node
-
-    #    with spinner("Fallback: Importing GenerationStep, GenerationStrategy from ax.generation_strategy..."):
-    #        from ax.generation_strategy.generation_node import GenerationNode, GenerationStep
-
-    #    with spinner("Fallback: Importing ExternalGenerationNode..."):
-    #        from ax.generation_strategy.external_generation_node import ExternalGenerationNode
-
-    #    with spinner("Fallback: Importing MaxTrials..."):
-    #        from ax.generation_strategy.transition_criterion import MaxTrials
-
-    with spinner("Importing Models from ax.generation_strategy.registry..."):
-        from ax.adapter.registry import Models
+    with spinner("Importing Generators from ax.generation_strategy.registry..."):
+        from ax.adapter.registry import Generators
 
     with spinner("Importing get_pending_observation_features..."):
         from ax.core.utils import get_pending_observation_features
@@ -1446,7 +1464,7 @@ class RandomForestGenerationNode(ExternalGenerationNode):
     def __init__(self: Any, regressor_options: Dict[str, Any] = {}, seed: Optional[int] = None, num_samples: int = 1) -> None:
         print_debug("Initializing RandomForestGenerationNode...")
         t_init_start = time.monotonic()
-        super().__init__(node_name="RANDOMFOREST")
+        super().__init__(name="RANDOMFOREST")
         self.num_samples: int = num_samples
         self.seed: int = seed
 
@@ -1466,6 +1484,9 @@ class RandomForestGenerationNode(ExternalGenerationNode):
     def update_generator_state(self: Any, experiment: Experiment, data: Data) -> None:
         search_space = experiment.search_space
         parameter_names = list(search_space.parameters.keys())
+        if experiment.optimization_config is None:
+            print_red("Error: update_generator_state is None")
+            return
         metric_names = list(experiment.optimization_config.metrics.keys())
 
         completed_trials = [
@@ -1477,7 +1498,7 @@ class RandomForestGenerationNode(ExternalGenerationNode):
         y = np.zeros([num_completed_trials, 1])
 
         for t_idx, trial in enumerate(completed_trials):
-            trial_parameters = trial.arm.parameters
+            trial_parameters = trial.arms[t_idx].parameters
             x[t_idx, :] = np.array([trial_parameters[p] for p in parameter_names])
             trial_df = data.df[data.df["trial_index"] == trial.index]
             y[t_idx, 0] = trial_df[trial_df["metric_name"] == metric_names[0]]["mean"].item()
@@ -1617,10 +1638,18 @@ class RandomForestGenerationNode(ExternalGenerationNode):
     def _format_best_sample(self: Any, best_sample: TParameterization, reverse_choice_map: dict) -> None:
         for name in best_sample.keys():
             param = self.parameters.get(name)
+            best_sample_by_name = best_sample[name]
+
             if isinstance(param, RangeParameter) and param.parameter_type == ParameterType.INT:
-                best_sample[name] = int(round(best_sample[name]))
+                if best_sample_by_name is not None:
+                    best_sample[name] = int(round(float(best_sample_by_name)))
+                else:
+                    print_debug("best_sample_by_name was empty")
             elif isinstance(param, ChoiceParameter):
-                best_sample[name] = str(reverse_choice_map.get(int(best_sample[name])))
+                if best_sample_by_name is not None:
+                    best_sample[name] = str(reverse_choice_map.get(int(best_sample_by_name)))
+                else:
+                    print_debug("best_sample_by_name was empty")
 
 decoder_registry["RandomForestGenerationNode"] = RandomForestGenerationNode
 
@@ -1668,10 +1697,10 @@ class InteractiveCLIGenerationNode(ExternalGenerationNode):
 
     def __init__(
         self: Any,
-        node_name: str = "INTERACTIVE_GENERATOR",
+        name: str = "INTERACTIVE_GENERATOR",
     ) -> None:
         t0 = time.monotonic()
-        super().__init__(node_name=node_name)
+        super().__init__(name=name)
         self.parameters = None
         self.minimize = None
         self.data = None
@@ -1827,10 +1856,10 @@ class InteractiveCLIGenerationNode(ExternalGenerationNode):
 
 @dataclass(init=False)
 class ExternalProgramGenerationNode(ExternalGenerationNode):
-    def __init__(self: Any, external_generator: str = args.external_generator, node_name: str = "EXTERNAL_GENERATOR") -> None:
+    def __init__(self: Any, external_generator: str = args.external_generator, name: str = "EXTERNAL_GENERATOR") -> None:
         print_debug("Initializing ExternalProgramGenerationNode...")
         t_init_start = time.monotonic()
-        super().__init__(node_name=node_name)
+        super().__init__(name=name)
         self.seed: int = args.seed
         self.external_generator: str = decode_if_base64(external_generator)
         self.constraints = None
@@ -2051,9 +2080,9 @@ def run_live_share_command(force: bool = False) -> Tuple[str, str]:
             return str(result.stdout), str(result.stderr)
         except subprocess.CalledProcessError as e:
             if e.stderr:
-                original_print(f"run_live_share_command: command failed with error: {e}, stderr: {e.stderr}")
+                print_debug(f"run_live_share_command: command failed with error: {e}, stderr: {e.stderr}")
             else:
-                original_print(f"run_live_share_command: command failed with error: {e}")
+                print_debug(f"run_live_share_command: command failed with error: {e}")
             return "", str(e.stderr)
         except Exception as e:
             print(f"run_live_share_command: An error occurred: {e}")
@@ -2067,6 +2096,8 @@ def force_live_share() -> bool:
     return False
 
 def live_share(force: bool = False, text_and_qr: bool = False) -> bool:
+    log_data()
+
     if not get_current_run_folder():
         print(f"live_share: get_current_run_folder was empty or false: {get_current_run_folder()}")
         return False
@@ -2080,17 +2111,23 @@ def live_share(force: bool = False, text_and_qr: bool = False) -> bool:
         if stderr:
             print_green(stderr)
         else:
-            print_red("This call should have shown the CURL, but didnt. Stderr: {stderr}, stdout: {stdout}")
+            if stderr and stdout:
+                print_red(f"This call should have shown the CURL, but didnt. Stderr: {stderr}, stdout: {stdout}")
+            elif stderr:
+                print_red(f"This call should have shown the CURL, but didnt. Stderr: {stderr}")
+            elif stdout:
+                print_red(f"This call should have shown the CURL, but didnt. Stdout: {stdout}")
+            else:
+                print_red("This call should have shown the CURL, but didnt.")
     if stdout:
         print_debug(f"live_share stdout: {stdout}")
 
     return True
 
 def init_live_share() -> bool:
-    with spinner("Initializing live share..."):
-        ret = live_share(True, True)
+    ret = live_share(True, True)
 
-        return ret
+    return ret
 
 def init_storage(db_url: str) -> None:
     init_engine_and_session_factory(url=db_url, force_init=True)
@@ -2116,7 +2153,11 @@ def try_saving_to_db() -> None:
         else:
             print_red("ax_client was not defined in try_saving_to_db")
             my_exit(101)
-        save_generation_strategy(global_gs)
+
+        if global_gs is not None:
+            save_generation_strategy(global_gs)
+        else:
+            print_red("Not saving generation strategy: global_gs was empty")
     except Exception as e:
         print_debug(f"Failed trying to save sqlite3-DB: {e}")
 
@@ -2147,6 +2188,8 @@ def merge_with_job_infos(df: pd.DataFrame) -> pd.DataFrame:
     return merged
 
 def save_results_csv() -> Optional[str]:
+    log_data()
+
     if args.dryrun:
         return None
 
@@ -2183,19 +2226,21 @@ def save_results_csv() -> Optional[str]:
 def get_results_paths() -> tuple[str, str]:
     return (get_current_run_folder(RESULTS_CSV_FILENAME), get_state_file_name('pd.json'))
 
+def ax_client_get_trials_data_frame() -> Optional[pd.DataFrame]:
+    if not ax_client:
+        my_exit(101)
+
+        return None
+
+    return ax_client.get_trials_data_frame()
+
 def fetch_and_prepare_trials() -> Optional[pd.DataFrame]:
     if not ax_client:
         return None
 
     ax_client.experiment.fetch_data()
-    df = ax_client.get_trials_data_frame()
-
-    #print("========================")
-    #print("BEFORE merge_with_job_infos:")
-    #print(df["generation_node"])
+    df = ax_client_get_trials_data_frame()
     df = merge_with_job_infos(df)
-    #print("AFTER merge_with_job_infos:")
-    #print(df["generation_node"])
 
     return df
 
@@ -2206,11 +2251,24 @@ def write_csv(df: pd.DataFrame, path: str) -> None:
         pass
     df.to_csv(path, index=False, float_format="%.30f")
 
+def ax_client_to_json_snapshot() -> Optional[dict]:
+    if not ax_client:
+        my_exit(101)
+
+        return None
+
+    json_snapshot = ax_client.to_json_snapshot()
+
+    return json_snapshot
+
 def write_json_snapshot(path: str) -> None:
     if ax_client is not None:
-        json_snapshot = ax_client.to_json_snapshot()
-        with open(path, "w", encoding="utf-8") as f:
-            json.dump(json_snapshot, f, indent=4)
+        json_snapshot = ax_client_to_json_snapshot()
+        if json_snapshot is not None:
+            with open(path, "w", encoding="utf-8") as f:
+                json.dump(json_snapshot, f, indent=4)
+        else:
+            print_debug('json_snapshot from ax_client_to_json_snapshot was None')
     else:
         print_red("write_json_snapshot: ax_client was None")
 
@@ -2410,7 +2468,7 @@ def set_nr_inserted_jobs(new_nr_inserted_jobs: int) -> None:
 
 def write_worker_usage() -> None:
     if len(WORKER_PERCENTAGE_USAGE):
-        csv_filename = get_current_run_folder('worker_usage.csv')
+        csv_filename = get_current_run_folder(worker_usage_file)
 
         csv_columns = ['time', 'num_parallel_jobs', 'nr_current_workers', 'percentage']
 
@@ -2420,35 +2478,39 @@ def write_worker_usage() -> None:
                 csv_writer.writerow(row)
     else:
         if is_slurm_job():
-            print_debug("WORKER_PERCENTAGE_USAGE seems to be empty. Not writing worker_usage.csv")
+            print_debug(f"WORKER_PERCENTAGE_USAGE seems to be empty. Not writing {worker_usage_file}")
 
 def log_system_usage() -> None:
+    global LAST_LOG_TIME
+
+    now = time.time()
+    if now - LAST_LOG_TIME < 30:
+        return
+
+    LAST_LOG_TIME = int(now)
+
     if not get_current_run_folder():
         return
 
     ram_cpu_csv_file_path = os.path.join(get_current_run_folder(), "cpu_ram_usage.csv")
-
     makedirs(os.path.dirname(ram_cpu_csv_file_path))
 
     file_exists = os.path.isfile(ram_cpu_csv_file_path)
 
-    with open(ram_cpu_csv_file_path, mode='a', newline='', encoding="utf-8") as file:
-        writer = csv.writer(file)
-
-        current_time = int(time.time())
-
-        if process is not None:
-            mem_proc = process.memory_info()
-
-            if mem_proc is not None:
-                ram_usage_mb = mem_proc.rss / (1024 * 1024)
-                cpu_usage_percent = psutil.cpu_percent(percpu=False)
+    mem_proc = process.memory_info() if process else None
+    if not mem_proc:
+        return
 
-                if ram_usage_mb > 0 and cpu_usage_percent > 0:
-                    if not file_exists:
-                        writer.writerow(["timestamp", "ram_usage_mb", "cpu_usage_percent"])
+    ram_usage_mb = mem_proc.rss / (1024 * 1024)
+    cpu_usage_percent = psutil.cpu_percent(percpu=False)
+    if ram_usage_mb <= 0 or cpu_usage_percent <= 0:
+        return
 
-                    writer.writerow([current_time, ram_usage_mb, cpu_usage_percent])
+    with open(ram_cpu_csv_file_path, mode='a', newline='', encoding="utf-8") as file:
+        writer = csv.writer(file)
+        if not file_exists:
+            writer.writerow(["timestamp", "ram_usage_mb", "cpu_usage_percent"])
+        writer.writerow([int(now), ram_usage_mb, cpu_usage_percent])
 
 def write_process_info() -> None:
     try:
@@ -2798,10 +2860,20 @@ def print_debug_get_next_trials(got: int, requested: int, _line: int) -> None:
     log_message_to_file(LOGFILE_DEBUG_GET_NEXT_TRIALS, msg, 0, "")
 
 def print_debug_progressbar(msg: str) -> None:
-    time_str = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-    msg = f"{time_str} ({worker_generator_uuid}): {msg}"
+    global last_msg_progressbar, last_msg_raw
+
+    try:
+        with last_lock_print_debug:
+            if msg != last_msg_raw:
+                time_str = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+                full_msg = f"{time_str} ({worker_generator_uuid}): {msg}"
 
-    _debug_progressbar(msg)
+                _debug_progressbar(full_msg)
+
+                last_msg_raw = msg
+                last_msg_progressbar = full_msg
+    except Exception as e:
+        print(f"Error in print_debug_progressbar: {e}", flush=True)
 
 def get_process_info(pid: Any) -> str:
     try:
@@ -3308,145 +3380,467 @@ def parse_experiment_parameters() -> None:
 
     experiment_parameters = params # type: ignore[assignment]
 
-def check_factorial_range() -> None:
-    if args.model and args.model == "FACTORIAL":
-        _fatal_error("\n⚠ --model FACTORIAL cannot be used with range parameter", 181)
+def job_calculate_pareto_front(path_to_calculate: str, disable_sixel_and_table: bool = False) -> bool:
+    pf_start_time = time.time()
 
-def check_if_range_types_are_invalid(value_type: str, valid_value_types: list) -> None:
-    if value_type not in valid_value_types:
-        valid_value_types_string = ", ".join(valid_value_types)
-        _fatal_error(f"⚠ {value_type} is not a valid value type. Valid types for range are: {valid_value_types_string}", 181)
+    if not path_to_calculate:
+        return False
 
-def check_range_params_length(this_args: Union[str, list]) -> None:
-    if len(this_args) != 5 and len(this_args) != 4 and len(this_args) != 6:
-        _fatal_error("\n⚠ --parameter for type range must have 4 (or 5, the last one being optional and float by default, or 6, while the last one is true or false) parameters: <NAME> range <START> <END> (<TYPE (int or float)>, <log_scale: bool>)", 181)
+    global CURRENT_RUN_FOLDER
+    global RESULT_CSV_FILE
+    global arg_result_names
 
-def die_if_lower_and_upper_bound_equal_zero(lower_bound: Union[int, float], upper_bound: Union[int, float]) -> None:
-    if upper_bound is None or lower_bound is None:
-        _fatal_error("die_if_lower_and_upper_bound_equal_zero: upper_bound or lower_bound is None. Cannot continue.", 91)
-    if upper_bound == lower_bound:
-        if lower_bound == 0:
-            _fatal_error(f"⚠ Lower bound and upper bound are equal: {lower_bound}, cannot automatically fix this, because they -0 = +0 (usually a quickfix would be to set lower_bound = -upper_bound)", 181)
-        print_red(f"⚠ Lower bound and upper bound are equal: {lower_bound}, setting lower_bound = -upper_bound")
-        if upper_bound is not None:
-            lower_bound = -upper_bound
+    if not path_to_calculate:
+        print_red("Can only calculate pareto front of previous job when --calculate_pareto_front_of_job is set")
+        return False
 
-def format_value(value: Any, float_format: str = '.80f') -> str:
-    try:
-        if isinstance(value, float):
-            s = format(value, float_format)
-            s = s.rstrip('0').rstrip('.') if '.' in s else s
-            return s
-        return str(value)
-    except Exception as e:
-        print_red(f"⚠ Error formatting the number {value}: {e}")
-        return str(value)
+    if not os.path.exists(path_to_calculate):
+        print_red(f"Path '{path_to_calculate}' does not exist")
+        return False
 
-def replace_parameters_in_string(
-    parameters: dict,
-    input_string: str,
-    float_format: str = '.20f',
-    additional_prefixes: list[str] = [],
-    additional_patterns: list[str] = [],
-) -> str:
-    try:
-        prefixes = ['$', '%'] + additional_prefixes
-        patterns = ['{key}', '({key})'] + additional_patterns
+    ax_client_json = f"{path_to_calculate}/state_files/ax_client.experiment.json"
 
-        for key, value in parameters.items():
-            replacement = format_value(value, float_format=float_format)
-            for prefix in prefixes:
-                for pattern in patterns:
-                    token = prefix + pattern.format(key=key)
-                    input_string = input_string.replace(token, replacement)
+    if not os.path.exists(ax_client_json):
+        print_red(f"Path '{ax_client_json}' not found")
+        return False
 
-        input_string = input_string.replace('\r', ' ').replace('\n', ' ')
-        return input_string
+    checkpoint_file: str = f"{path_to_calculate}/state_files/checkpoint.json"
+    if not os.path.exists(checkpoint_file):
+        print_red(f"The checkpoint file '{checkpoint_file}' does not exist")
+        return False
 
-    except Exception as e:
-        print_red(f"\n⚠ Error: {e}")
-        return ""
+    RESULT_CSV_FILE = f"{path_to_calculate}/{RESULTS_CSV_FILENAME}"
+    if not os.path.exists(RESULT_CSV_FILE):
+        print_red(f"{RESULT_CSV_FILE} not found")
+        return False
 
-def get_memory_usage() -> float:
-    user_uid = os.getuid()
+    res_names = []
 
-    memory_usage = float(sum(
-        p.memory_info().rss for p in psutil.process_iter(attrs=['memory_info', 'uids'])
-        if p.info['uids'].real == user_uid
-    ) / (1024 * 1024))
+    res_names_file = f"{path_to_calculate}/result_names.txt"
+    if not os.path.exists(res_names_file):
+        print_red(f"File '{res_names_file}' does not exist")
+        return False
 
-    return memory_usage
+    try:
+        with open(res_names_file, "r", encoding="utf-8") as file:
+            lines = file.readlines()
+    except Exception as e:
+        print_red(f"Error reading file '{res_names_file}': {e}")
+        return False
 
-class MonitorProcess:
-    def __init__(self: Any, pid: int, interval: float = 1.0) -> None:
-        self.pid = pid
-        self.interval = interval
-        self.running = True
-        self.thread = threading.Thread(target=self._monitor)
-        self.thread.daemon = True
+    for line in lines:
+        entry = line.strip()
+        if entry != "":
+            res_names.append(entry)
 
-        fool_linter(f"self.thread.daemon was set to {self.thread.daemon}")
+    if len(res_names) < 2:
+        print_red(f"Error: There are less than 2 result names (is: {len(res_names)}, {', '.join(res_names)}) in {path_to_calculate}. Cannot continue calculating the pareto front.")
+        return False
 
-    def _monitor(self: Any) -> None:
-        try:
-            _internal_process = psutil.Process(self.pid)
-            while self.running and _internal_process.is_running():
-                crf = get_current_run_folder()
+    load_username_to_args(path_to_calculate)
 
-                if crf and crf != "":
-                    log_file_path = os.path.join(crf, "eval_nodes_cpu_ram_logs.txt")
+    CURRENT_RUN_FOLDER = path_to_calculate
 
-                    os.makedirs(os.path.dirname(log_file_path), exist_ok=True)
+    arg_result_names = res_names
 
-                    with open(log_file_path, mode="a", encoding="utf-8") as log_file:
-                        hostname = socket.gethostname()
+    load_experiment_parameters_from_checkpoint_file(checkpoint_file, False)
 
-                        slurm_job_id = os.getenv("SLURM_JOB_ID")
+    if experiment_parameters is None:
+        return False
 
-                        if slurm_job_id:
-                            hostname += f"-SLURM-ID-{slurm_job_id}"
+    show_pareto_or_error_msg(path_to_calculate, res_names, disable_sixel_and_table)
 
-                        total_memory = psutil.virtual_memory().total / (1024 * 1024)
-                        cpu_usage = psutil.cpu_percent(interval=5)
+    pf_end_time = time.time()
 
-                        memory_usage = get_memory_usage()
+    print_debug(f"Calculating the Pareto-front took {pf_end_time - pf_start_time} seconds")
 
-                        unix_timestamp = int(time.time())
+    return True
 
-                        log_file.write(f"\nUnix-Timestamp: {unix_timestamp}, Hostname: {hostname}, CPU: {cpu_usage:.2f}%, RAM: {memory_usage:.2f} MB / {total_memory:.2f} MB\n")
-                time.sleep(self.interval)
-        except psutil.NoSuchProcess:
-            pass
+def show_pareto_or_error_msg(path_to_calculate: str, res_names: list = arg_result_names, disable_sixel_and_table: bool = False) -> None:
+    if args.dryrun:
+        print_debug("Not showing Pareto-frontier data with --dryrun")
+        return None
 
-    def __enter__(self: Any) -> None:
-        self.thread.start()
-        return self
+    if len(res_names) > 1:
+        try:
+            show_pareto_frontier_data(path_to_calculate, res_names, disable_sixel_and_table)
+        except Exception as e:
+            inner_tb = ''.join(traceback.format_exception(type(e), e, e.__traceback__))
+            print_red(f"show_pareto_frontier_data() failed with exception '{e}':\n{inner_tb}")
+    else:
+        print_debug(f"show_pareto_frontier_data will NOT be executed because len(arg_result_names) is {len(arg_result_names)}")
+    return None
 
-    def __exit__(self: Any, exc_type: Any, exc_value: Any, _traceback: Any) -> None:
-        self.running = False
-        self.thread.join()
+def get_pareto_front_data(path_to_calculate: str, res_names: list) -> dict:
+    pareto_front_data: dict = {}
 
-def execute_bash_code_log_time(code: str) -> list:
-    process_item = subprocess.Popen(code, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+    all_combinations = list(combinations(range(len(arg_result_names)), 2))
 
-    with MonitorProcess(process_item.pid):
-        try:
-            stdout, stderr = process_item.communicate()
-            result = subprocess.CompletedProcess(
-                args=code, returncode=process_item.returncode, stdout=stdout, stderr=stderr
-            )
-            return [result.stdout, result.stderr, result.returncode, None]
-        except subprocess.CalledProcessError as e:
-            real_exit_code = e.returncode
-            signal_code = None
-            if real_exit_code < 0:
-                signal_code = abs(e.returncode)
-                real_exit_code = 1
-            return [e.stdout, e.stderr, real_exit_code, signal_code]
+    skip = False
 
-def execute_bash_code(code: str) -> list:
-    try:
-        result = subprocess.run(
+    for i, j in all_combinations:
+        if not skip:
+            metric_x = arg_result_names[i]
+            metric_y = arg_result_names[j]
+
+            x_minimize = get_result_minimize_flag(path_to_calculate, metric_x)
+            y_minimize = get_result_minimize_flag(path_to_calculate, metric_y)
+
+            try:
+                if metric_x not in pareto_front_data:
+                    pareto_front_data[metric_x] = {}
+
+                pareto_front_data[metric_x][metric_y] = get_calculated_frontier(path_to_calculate, metric_x, metric_y, x_minimize, y_minimize, res_names)
+            except ax.exceptions.core.DataRequiredError as e:
+                print_red(f"Error computing Pareto frontier for {metric_x} and {metric_y}: {e}")
+            except SignalINT:
+                print_red("Calculating Pareto-fronts was cancelled by pressing CTRL-c")
+                skip = True
+
+    return pareto_front_data
+
+def pareto_front_transform_objectives(
+    points: List[Tuple[Any, float, float]],
+    primary_name: str,
+    secondary_name: str
+) -> Tuple[np.ndarray, np.ndarray]:
+    primary_idx = arg_result_names.index(primary_name)
+    secondary_idx = arg_result_names.index(secondary_name)
+
+    x = np.array([p[1] for p in points])
+    y = np.array([p[2] for p in points])
+
+    if arg_result_min_or_max[primary_idx] == "max":
+        x = -x
+    elif arg_result_min_or_max[primary_idx] != "min":
+        raise ValueError(f"Unknown mode for {primary_name}: {arg_result_min_or_max[primary_idx]}")
+
+    if arg_result_min_or_max[secondary_idx] == "max":
+        y = -y
+    elif arg_result_min_or_max[secondary_idx] != "min":
+        raise ValueError(f"Unknown mode for {secondary_name}: {arg_result_min_or_max[secondary_idx]}")
+
+    return x, y
+
+def get_pareto_frontier_points(
+    path_to_calculate: str,
+    primary_objective: str,
+    secondary_objective: str,
+    x_minimize: bool,
+    y_minimize: bool,
+    absolute_metrics: List[str],
+    num_points: int
+) -> Optional[dict]:
+    records = pareto_front_aggregate_data(path_to_calculate)
+
+    if records is None:
+        return None
+
+    points = pareto_front_filter_complete_points(path_to_calculate, records, primary_objective, secondary_objective)
+    x, y = pareto_front_transform_objectives(points, primary_objective, secondary_objective)
+    selected_points = pareto_front_select_pareto_points(x, y, x_minimize, y_minimize, points, num_points)
+    result = pareto_front_build_return_structure(path_to_calculate, selected_points, records, absolute_metrics, primary_objective, secondary_objective)
+
+    return result
+
+def pareto_front_table_read_csv() -> List[Dict[str, str]]:
+    with open(RESULT_CSV_FILE, mode="r", encoding="utf-8", newline="") as f:
+        return list(csv.DictReader(f))
+
+def create_pareto_front_table(idxs: List[int], metric_x: str, metric_y: str) -> Table:
+    table = Table(title=f"Pareto-Front for {metric_y}/{metric_x}:", show_lines=True)
+
+    rows = pareto_front_table_read_csv()
+    if not rows:
+        table.add_column("No data found")
+        return table
+
+    filtered_rows = pareto_front_table_filter_rows(rows, idxs)
+    if not filtered_rows:
+        table.add_column("No matching entries")
+        return table
+
+    param_cols, result_cols = pareto_front_table_get_columns(filtered_rows[0])
+
+    pareto_front_table_add_headers(table, param_cols, result_cols)
+    pareto_front_table_add_rows(table, filtered_rows, param_cols, result_cols)
+
+    return table
+
+def pareto_front_build_return_structure(
+    path_to_calculate: str,
+    selected_points: List[Tuple[Any, float, float]],
+    records: Dict[Tuple[int, str], Dict[str, Dict[str, float]]],
+    absolute_metrics: List[str],
+    primary_name: str,
+    secondary_name: str
+) -> dict:
+    results_csv_file = f"{path_to_calculate}/{RESULTS_CSV_FILENAME}"
+    result_names_file = f"{path_to_calculate}/result_names.txt"
+
+    with open(result_names_file, mode="r", encoding="utf-8") as f:
+        result_names = [line.strip() for line in f if line.strip()]
+
+    csv_rows = {}
+    with open(results_csv_file, mode="r", encoding="utf-8", newline='') as csvfile:
+        reader = csv.DictReader(csvfile)
+        for row in reader:
+            trial_index = int(row['trial_index'])
+            csv_rows[trial_index] = row
+
+    ignored_columns = {'trial_index', 'arm_name', 'trial_status', 'generation_node'}
+    ignored_columns.update(result_names)
+
+    param_dicts = []
+    idxs = []
+    means_dict = defaultdict(list)
+
+    for (trial_index, arm_name), _, _ in selected_points:
+        row = csv_rows.get(trial_index, {})
+        if row == {} or row is None or row['arm_name'] != arm_name:
+            continue
+
+        idxs.append(int(row["trial_index"]))
+
+        param_dict: dict[str, int | float | str] = {}
+        for key, value in row.items():
+            if key not in ignored_columns:
+                try:
+                    param_dict[key] = int(value)
+                except ValueError:
+                    try:
+                        param_dict[key] = float(value)
+                    except ValueError:
+                        param_dict[key] = value
+
+        param_dicts.append(param_dict)
+
+        for metric in absolute_metrics:
+            means_dict[metric].append(records[(trial_index, arm_name)]['means'].get(metric, float("nan")))
+
+    ret = {
+        primary_name: {
+            secondary_name: {
+                "absolute_metrics": absolute_metrics,
+                "param_dicts": param_dicts,
+                "means": dict(means_dict),
+                "idxs": idxs
+            },
+            "absolute_metrics": absolute_metrics
+        }
+    }
+
+    return ret
+
+def pareto_front_aggregate_data(path_to_calculate: str) -> Optional[Dict[Tuple[int, str], Dict[str, Dict[str, float]]]]:
+    results_csv_file = f"{path_to_calculate}/{RESULTS_CSV_FILENAME}"
+    result_names_file = f"{path_to_calculate}/result_names.txt"
+
+    if not os.path.exists(results_csv_file) or not os.path.exists(result_names_file):
+        return None
+
+    with open(result_names_file, mode="r", encoding="utf-8") as f:
+        result_names = [line.strip() for line in f if line.strip()]
+
+    records: dict = defaultdict(lambda: {'means': {}})
+
+    with open(results_csv_file, encoding="utf-8", mode="r", newline='') as csvfile:
+        reader = csv.DictReader(csvfile)
+        for row in reader:
+            trial_index = int(row['trial_index'])
+            arm_name = row['arm_name']
+            key = (trial_index, arm_name)
+
+            for metric in result_names:
+                if metric in row:
+                    try:
+                        records[key]['means'][metric] = float(row[metric])
+                    except ValueError:
+                        continue
+
+    return records
+
+def plot_pareto_frontier_sixel(data: Any, x_metric: str, y_metric: str) -> None:
+    if data is None:
+        print("[italic yellow]The data seems to be empty. Cannot plot pareto frontier.[/]")
+        return
+
+    if not supports_sixel():
+        print(f"[italic yellow]Your console does not support sixel-images. Will not print Pareto-frontier as a matplotlib-sixel-plot for {x_metric}/{y_metric}.[/]")
+        return
+
+    import matplotlib.pyplot as plt
+
+    means = data[x_metric][y_metric]["means"]
+
+    x_values = means[x_metric]
+    y_values = means[y_metric]
+
+    fig, _ax = plt.subplots()
+
+    _ax.scatter(x_values, y_values, s=50, marker='x', c='blue', label='Data Points')
+
+    _ax.set_xlabel(x_metric)
+    _ax.set_ylabel(y_metric)
+
+    _ax.set_title(f'Pareto-Front {x_metric}/{y_metric}')
+
+    _ax.ticklabel_format(style='plain', axis='both', useOffset=False)
+
+    with tempfile.NamedTemporaryFile(suffix=".png", delete=True) as tmp_file:
+        plt.savefig(tmp_file.name, dpi=300)
+
+        print_image_to_cli(tmp_file.name, 1000)
+
+    plt.close(fig)
+
+def pareto_front_table_get_columns(first_row: Dict[str, str]) -> Tuple[List[str], List[str]]:
+    all_columns = list(first_row.keys())
+    ignored_cols = set(special_col_names) - {"trial_index"}
+
+    param_cols = [col for col in all_columns if col not in ignored_cols and col not in arg_result_names and not col.startswith("OO_Info_")]
+    result_cols = [col for col in arg_result_names if col in all_columns]
+    return param_cols, result_cols
+
+def check_factorial_range() -> None:
+    if args.model and args.model == "FACTORIAL":
+        _fatal_error("\n⚠ --model FACTORIAL cannot be used with range parameter", 181)
+
+def check_if_range_types_are_invalid(value_type: str, valid_value_types: list) -> None:
+    if value_type not in valid_value_types:
+        valid_value_types_string = ", ".join(valid_value_types)
+        _fatal_error(f"⚠ {value_type} is not a valid value type. Valid types for range are: {valid_value_types_string}", 181)
+
+def check_range_params_length(this_args: Union[str, list]) -> None:
+    if len(this_args) != 5 and len(this_args) != 4 and len(this_args) != 6:
+        _fatal_error("\n⚠ --parameter for type range must have 4 (or 5, the last one being optional and float by default, or 6, while the last one is true or false) parameters: <NAME> range <START> <END> (<TYPE (int or float)>, <log_scale: bool>)", 181)
+
+def die_if_lower_and_upper_bound_equal_zero(lower_bound: Union[int, float], upper_bound: Union[int, float]) -> None:
+    if upper_bound is None or lower_bound is None:
+        _fatal_error("die_if_lower_and_upper_bound_equal_zero: upper_bound or lower_bound is None. Cannot continue.", 91)
+    if upper_bound == lower_bound:
+        if lower_bound == 0:
+            _fatal_error(f"⚠ Lower bound and upper bound are equal: {lower_bound}, cannot automatically fix this, because they -0 = +0 (usually a quickfix would be to set lower_bound = -upper_bound)", 181)
+        print_red(f"⚠ Lower bound and upper bound are equal: {lower_bound}, setting lower_bound = -upper_bound")
+        if upper_bound is not None:
+            lower_bound = -upper_bound
+
+def format_value(value: Any, float_format: str = '.80f') -> str:
+    try:
+        if isinstance(value, float):
+            s = format(value, float_format)
+            s = s.rstrip('0').rstrip('.') if '.' in s else s
+            return s
+        return str(value)
+    except Exception as e:
+        print_red(f"⚠ Error formatting the number {value}: {e}")
+        return str(value)
+
+def replace_parameters_in_string(
+    parameters: dict,
+    input_string: str,
+    float_format: str = '.20f',
+    additional_prefixes: list[str] = [],
+    additional_patterns: list[str] = [],
+) -> str:
+    try:
+        prefixes = ['$', '%'] + additional_prefixes
+        patterns = ['{' + 'key' + '}', '(' + '{' + 'key' + '}' + ')'] + additional_patterns
+
+        for key, value in parameters.items():
+            replacement = format_value(value, float_format=float_format)
+            for prefix in prefixes:
+                for pattern in patterns:
+                    token = prefix + pattern.format(key=key)
+                    input_string = input_string.replace(token, replacement)
+
+        input_string = input_string.replace('\r', ' ').replace('\n', ' ')
+        return input_string
+
+    except Exception as e:
+        print_red(f"\n⚠ Error: {e}")
+        return ""
+
+def get_memory_usage() -> float:
+    user_uid = os.getuid()
+
+    memory_usage = float(sum(
+        p.memory_info().rss for p in psutil.process_iter(attrs=['memory_info', 'uids'])
+        if p.info['uids'].real == user_uid
+    ) / (1024 * 1024))
+
+    return memory_usage
+
+class MonitorProcess:
+    def __init__(self: Any, pid: int, interval: float = 1.0) -> None:
+        self.pid = pid
+        self.interval = interval
+        self.running = True
+        self.thread = threading.Thread(target=self._monitor)
+        self.thread.daemon = True
+
+        fool_linter(f"self.thread.daemon was set to {self.thread.daemon}")
+
+    def _monitor(self: Any) -> None:
+        try:
+            _internal_process = psutil.Process(self.pid)
+            while self.running and _internal_process.is_running():
+                crf = get_current_run_folder()
+
+                if crf and crf != "":
+                    log_file_path = os.path.join(crf, "eval_nodes_cpu_ram_logs.txt")
+
+                    os.makedirs(os.path.dirname(log_file_path), exist_ok=True)
+
+                    with open(log_file_path, mode="a", encoding="utf-8") as log_file:
+                        hostname = socket.gethostname()
+
+                        slurm_job_id = os.getenv("SLURM_JOB_ID")
+
+                        if slurm_job_id:
+                            hostname += f"-SLURM-ID-{slurm_job_id}"
+
+                        total_memory = psutil.virtual_memory().total / (1024 * 1024)
+                        cpu_usage = psutil.cpu_percent(interval=5)
+
+                        memory_usage = get_memory_usage()
+
+                        unix_timestamp = int(time.time())
+
+                        log_file.write(f"\nUnix-Timestamp: {unix_timestamp}, Hostname: {hostname}, CPU: {cpu_usage:.2f}%, RAM: {memory_usage:.2f} MB / {total_memory:.2f} MB\n")
+                time.sleep(self.interval)
+        except psutil.NoSuchProcess:
+            pass
+
+    def __enter__(self: Any) -> None:
+        self.thread.start()
+        return self
+
+    def __exit__(self: Any, exc_type: Any, exc_value: Any, _traceback: Any) -> None:
+        self.running = False
+        self.thread.join()
+
+def execute_bash_code_log_time(code: str) -> list:
+    process_item = subprocess.Popen(code, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+
+    with MonitorProcess(process_item.pid):
+        try:
+            stdout, stderr = process_item.communicate()
+            result = subprocess.CompletedProcess(
+                args=code, returncode=process_item.returncode, stdout=stdout, stderr=stderr
+            )
+            return [result.stdout, result.stderr, result.returncode, None]
+        except subprocess.CalledProcessError as e:
+            real_exit_code = e.returncode
+            signal_code = None
+            if real_exit_code < 0:
+                signal_code = abs(e.returncode)
+                real_exit_code = 1
+            return [e.stdout, e.stderr, real_exit_code, signal_code]
+
+def execute_bash_code(code: str) -> list:
+    try:
+        result = subprocess.run(
             code,
             shell=True,
             check=True,
@@ -3559,7 +3953,7 @@ def _add_to_csv_acquire_lock(lockfile: str, dir_path: str) -> bool:
             time.sleep(wait_time)
             max_wait -= wait_time
         except Exception as e:
-            print("Lock error:", e)
+            print_red(f"Lock error: {e}")
             return False
     return False
 
@@ -3632,12 +4026,12 @@ def find_file_paths(_text: str) -> List[str]:
 def check_file_info(file_path: str) -> str:
     if not os.path.exists(file_path):
         if not args.tests:
-            print(f"check_file_info: The file {file_path} does not exist.")
+            print_red(f"check_file_info: The file {file_path} does not exist.")
         return ""
 
     if not os.access(file_path, os.R_OK):
         if not args.tests:
-            print(f"check_file_info: The file {file_path} is not readable.")
+            print_red(f"check_file_info: The file {file_path} is not readable.")
         return ""
 
     file_stat = os.stat(file_path)
@@ -3751,7 +4145,7 @@ def count_defective_nodes(file_path: Union[str, None] = None, entry: Any = None)
         return sorted(set(entries))
 
     except Exception as e:
-        print(f"An error has occurred: {e}")
+        print_red(f"An error has occurred: {e}")
         return []
 
 def test_gpu_before_evaluate(return_in_case_of_error: dict) -> Union[None, dict]:
@@ -3762,7 +4156,7 @@ def test_gpu_before_evaluate(return_in_case_of_error: dict) -> Union[None, dict]
 
                 fool_linter(tmp)
         except RuntimeError:
-            print(f"Node {socket.gethostname()} was detected as faulty. It should have had a GPU, but there is an error initializing the CUDA driver. Adding this node to the --exclude list.")
+            print_red(f"Node {socket.gethostname()} was detected as faulty. It should have had a GPU, but there is an error initializing the CUDA driver. Adding this node to the --exclude list.")
             count_defective_nodes(None, socket.gethostname())
             return return_in_case_of_error
         except Exception:
@@ -4233,6 +4627,8 @@ def evaluate(parameters_with_trial_index: dict) -> Optional[Union[int, float, Di
     trial_index = parameters_with_trial_index["trial_idx"]
     submit_time = parameters_with_trial_index["submit_time"]
 
+    print(f'Trial-Index: {trial_index}')
+
     queue_time = abs(int(time.time()) - int(submit_time))
 
     start_nvidia_smi_thread()
@@ -4470,7 +4866,7 @@ def replace_string_with_params(input_string: str, params: list) -> str:
         return replaced_string
     except AssertionError as e:
         error_text = f"Error in replace_string_with_params: {e}"
-        print(error_text)
+        print_red(error_text)
         raise
 
     return ""
@@ -4747,9 +5143,7 @@ def get_sixel_graphics_data(_pd_csv: str, _force: bool = False) -> list:
             _params = [_command, plot, _tmp, plot_type, tmp_file, _width]
             data.append(_params)
         except Exception as e:
-            tb = traceback.format_exc()
-            print_red(f"Error trying to print {plot_type} to CLI: {e}, {tb}")
-            print_debug(f"Error trying to print {plot_type} to CLI: {e}")
+            print_red(f"Error trying to print {plot_type} to CLI: {e}")
 
     return data
 
@@ -4940,15 +5334,17 @@ def abandon_job(job: Job, trial_index: int, reason: str) -> bool:
     if job:
         try:
             if ax_client:
-                _trial = ax_client.get_trial(trial_index)
-                _trial.mark_abandoned(reason=reason)
+                _trial = get_ax_client_trial(trial_index)
+                if _trial is None:
+                    return False
+
+                mark_abandoned(_trial, reason, trial_index)
                 print_debug(f"abandon_job: removing job {job}, trial_index: {trial_index}")
                 global_vars["jobs"].remove((job, trial_index))
             else:
                 _fatal_error("ax_client could not be found", 101)
         except Exception as e:
-            print(f"ERROR in line {get_line_info()}: {e}")
-            print_debug(f"ERROR in line {get_line_info()}: {e}")
+            print_red(f"ERROR in line {get_line_info()}: {e}")
             return False
         job.cancel()
         return True
@@ -4961,23 +5357,120 @@ def abandon_all_jobs() -> None:
         if not abandoned:
             print_debug(f"Job {job} could not be abandoned.")
 
-def show_pareto_or_error_msg(path_to_calculate: str, res_names: list = arg_result_names, disable_sixel_and_table: bool = False) -> None:
-    if args.dryrun:
-        print_debug("Not showing Pareto-frontier data with --dryrun")
+def write_result_to_trace_file(res: str) -> bool:
+    if res is None:
+        sys.stderr.write("Provided result is None, nothing to write\n")
+        return False
+
+    target_folder = get_current_run_folder()
+    target_file = os.path.join(target_folder, "optimization_trace.html")
+
+    try:
+        file_handle = open(target_file, "w", encoding="utf-8")
+    except OSError as error:
+        sys.stderr.write("Unable to open target file for writing\n")
+        sys.stderr.write(str(error) + "\n")
+        return False
+
+    try:
+        written = file_handle.write(str(res))
+        file_handle.flush()
+
+        if written == 0:
+            sys.stderr.write("No data was written to the file\n")
+            file_handle.close()
+            return False
+    except Exception as error:
+        sys.stderr.write("Error occurred while writing to file\n")
+        sys.stderr.write(str(error) + "\n")
+        file_handle.close()
+        return False
+
+    try:
+        file_handle.close()
+    except Exception as error:
+        sys.stderr.write("Failed to properly close file\n")
+        sys.stderr.write(str(error) + "\n")
+        return False
+
+    return True
+
+def render(plot_config: AxPlotConfig) -> None:
+    if plot_config is None or "data" not in plot_config:
         return None
 
-    if len(res_names) > 1:
-        try:
-            show_pareto_frontier_data(path_to_calculate, res_names, disable_sixel_and_table)
-        except Exception as e:
-            print_red(f"show_pareto_frontier_data() failed with exception '{e}'")
-    else:
-        print_debug(f"show_pareto_frontier_data will NOT be executed because len(arg_result_names) is {len(arg_result_names)}")
+    res: str = plot_config.data # type: ignore
+
+    repair_funcs = """
+function decodeBData(obj) {
+        if (!obj || typeof obj !== "object") {
+            return obj;
+        }
+
+        if (obj.bdata && obj.dtype) {
+            var binary_string = atob(obj.bdata);
+            var len = binary_string.length;
+            var bytes = new Uint8Array(len);
+
+            for (var i = 0; i < len; i++) {
+                bytes[i] = binary_string.charCodeAt(i);
+            }
+
+            switch (obj.dtype) {
+                case "i1": return Array.from(new Int8Array(bytes.buffer));
+                case "i2": return Array.from(new Int16Array(bytes.buffer));
+                case "i4": return Array.from(new Int32Array(bytes.buffer));
+                case "f4": return Array.from(new Float32Array(bytes.buffer));
+                case "f8": return Array.from(new Float64Array(bytes.buffer));
+                default:
+                    console.error("Unknown dtype:", obj.dtype);
+                    return [];
+            }
+        }
+
+        return obj;
+}
+
+function repairTraces(traces) {
+        var fixed = [];
+
+        for (var i = 0; i < traces.length; i++) {
+            var t = traces[i];
+
+            if (t.x) {
+                t.x = decodeBData(t.x);
+            }
+
+            if (t.y) {
+                t.y = decodeBData(t.y);
+            }
+
+            fixed.push(t);
+        }
+
+        return fixed;
+}
+    """
+
+    res = str(res)
+
+    res = f"<div id='plot' style='width:100%;height:600px;'></div>\n<script type='text/javascript' src='https://cdn.plot.ly/plotly-latest.min.js'></script><script>{repair_funcs}\nconst True = true;\nconst False = false;\nconst data = {res};\ndata.data = repairTraces(data.data);\nPlotly.newPlot(document.getElementById('plot'), data.data, data.layout);</script>"
+
+    write_result_to_trace_file(res)
+
     return None
 
 def end_program(_force: Optional[bool] = False, exit_code: Optional[int] = None) -> None:
     global END_PROGRAM_RAN
 
+    #dier(global_gs.current_node.generator_specs[0]._fitted_adapter.generator._surrogate.training_data[0].X)
+    #dier(global_gs.current_node.generator_specs[0]._fitted_adapter.generator._surrogate.training_data[0].Y)
+    #dier(global_gs.current_node.generator_specs[0]._fitted_adapter.generator._surrogate.outcomes)
+
+    if ax_client is not None:
+        if len(arg_result_names) == 1:
+            render(ax_client.get_optimization_trace())
+
     wait_for_jobs_to_complete()
 
     show_pareto_or_error_msg(get_current_run_folder(), arg_result_names)
@@ -5011,7 +5504,7 @@ def end_program(_force: Optional[bool] = False, exit_code: Optional[int] = None)
             _exit = new_exit
     except (SignalUSR, SignalINT, SignalCONT, KeyboardInterrupt):
         print_red("\n⚠ You pressed CTRL+C or a signal was sent. Program execution halted while ending program.")
-        print("\n⚠ KeyboardInterrupt signal was sent. Ending program will still run.")
+        print_red("\n⚠ KeyboardInterrupt signal was sent. Ending program will still run.")
         new_exit = show_end_table_and_save_end_files()
         if new_exit > 0:
             _exit = new_exit
@@ -5032,19 +5525,29 @@ def end_program(_force: Optional[bool] = False, exit_code: Optional[int] = None)
 
     my_exit(_exit)
 
+def save_ax_client_to_json_file(checkpoint_filepath: str) -> None:
+    if not ax_client:
+        my_exit(101)
+
+        return None
+
+    ax_client.save_to_json_file(checkpoint_filepath)
+
+    return None
+
 def save_checkpoint(trial_nr: int = 0, eee: Union[None, str, Exception] = None) -> None:
     if trial_nr > 3:
         if eee:
-            print(f"Error during saving checkpoint: {eee}")
+            print_red(f"Error during saving checkpoint: {eee}")
         else:
-            print("Error during saving checkpoint")
+            print_red("Error during saving checkpoint")
         return
 
     try:
         checkpoint_filepath = get_state_file_name('checkpoint.json')
 
         if ax_client:
-            ax_client.save_to_json_file(filepath=checkpoint_filepath)
+            save_ax_client_to_json_file(checkpoint_filepath)
         else:
             _fatal_error("Something went wrong using the ax_client", 101)
     except Exception as e:
@@ -5207,7 +5710,7 @@ def parse_equation_item(comparer_found: bool, item: str, parsed: list, parsed_or
         })
     elif item in [">=", "<="]:
         if comparer_found:
-            print("There is already one comparison operator! Cannot have more than one in an equation!")
+            print_red("There is already one comparison operator! Cannot have more than one in an equation!")
             return_totally = True
         comparer_found = True
 
@@ -5418,20 +5921,9 @@ def check_equation(variables: list, equation: str) -> Union[str, bool]:
 def set_objectives() -> dict:
     objectives = {}
 
-    for rn in args.result_names:
-        key, value = "", ""
-
-        if "=" in rn:
-            key, value = rn.split('=', 1)
-        else:
-            key = rn
-            value = ""
-
-        if value not in ["min", "max"]:
-            if value:
-                print_yellow(f"Value '{value}' for --result_names {rn} is not a valid value. Must be min or max. Will be set to min.")
-
-            value = "min"
+    k = 0
+    for key in arg_result_names:
+        value = arg_result_min_or_max[k]
 
         _min = True
 
@@ -5439,6 +5931,7 @@ def set_objectives() -> dict:
             _min = False
 
         objectives[key] = ObjectiveProperties(minimize=_min)
+        k = k + 1
 
     return objectives
 
@@ -5660,7 +6153,7 @@ def load_from_checkpoint(continue_previous_job: str, cli_params_experiment_param
 
     replace_parameters_for_continued_jobs(args.parameter, cli_params_experiment_parameters)
 
-    ax_client.save_to_json_file(filepath=original_ax_client_file)
+    save_ax_client_to_json_file(original_ax_client_file)
 
     load_original_generation_strategy(original_ax_client_file)
     load_ax_client_from_experiment_parameters()
@@ -5690,6 +6183,109 @@ def load_from_checkpoint(continue_previous_job: str, cli_params_experiment_param
 
     return experiment_args, gpu_string, gpu_color
 
+def get_experiment_args_import_python_script() -> str:
+
+    return """from ax.service.ax_client import AxClient, ObjectiveProperties
+from ax.adapter.registry import Generators
+import random
+
+"""
+
+def get_generate_and_test_random_function_str() -> str:
+    raw_data_entries = ",\n                ".join(
+        f'"{name}": random.uniform(0, 1)' for name in arg_result_names
+    )
+
+    return f"""
+def generate_and_test_random_parameters(n: int) -> None:
+    for _ in range(n):
+        print("======================================")
+        parameters, trial_index = ax_client.get_next_trial()
+        print("Trial Index:", trial_index)
+        print("Suggested parameters:", parameters)
+
+        ax_client.complete_trial(
+            trial_index=trial_index,
+            raw_data={{
+                {raw_data_entries}
+            }}
+        )
+
+generate_and_test_random_parameters({args.num_random_steps + 1})
+"""
+
+def get_global_gs_string() -> str:
+    seed_str = ""
+    if args.seed is not None:
+        seed_str = f"model_kwargs={{'seed': {args.seed}}},"
+
+    return f"""from ax.generation_strategy.generation_strategy import GenerationStep, GenerationStrategy
+
+global_gs = GenerationStrategy(
+    steps=[
+        GenerationStep(
+            generator=Generators.SOBOL,
+            num_trials={args.num_random_steps},
+            max_parallelism=5,
+            {seed_str}
+        ),
+        GenerationStep(
+            generator=Generators.{args.model},
+            num_trials=-1,
+            max_parallelism=5,
+        ),
+    ]
+)
+"""
+
+def get_debug_ax_client_str() -> str:
+    return """
+ax_client = AxClient(
+    verbose_logging=True,
+    enforce_sequential_optimization=False,
+    generation_strategy=global_gs
+)
+"""
+
+def write_ax_debug_python_code(experiment_args: dict) -> None:
+    if args.generation_strategy:
+        print_debug("Cannot write debug code for custom generation_strategy")
+        return None
+
+    if args.model in uncontinuable_models:
+        print_debug(f"Cannot write debug code for uncontinuable mode {args.model}")
+        return None
+
+    python_code = python_code = get_experiment_args_import_python_script() + \
+        get_global_gs_string() + \
+        get_debug_ax_client_str() + \
+        "experiment_args = " + pformat(experiment_args, width=120, compact=False) + \
+        "\nax_client.create_experiment(**experiment_args)\n" + \
+        get_generate_and_test_random_function_str()
+
+    file_path = f"{get_current_run_folder()}/debug.py"
+
+    try:
+        print_debug(python_code)
+        with open(file_path, "w", encoding="utf-8") as f:
+            f.write(python_code)
+    except Exception as e:
+        print_red(f"Error while writing {file_path}: {e}")
+
+    return None
+
+def create_ax_client_experiment(experiment_args: dict) -> None:
+    if not ax_client:
+        my_exit(101)
+
+        return None
+
+    write_ax_debug_python_code(experiment_args)
+
+    ax_client.create_experiment(**experiment_args)
+
+    return None
+
 def create_new_experiment() -> Tuple[dict, str, str]:
     if ax_client is None:
         print_red("create_new_experiment: ax_client is None")
@@ -5719,7 +6315,7 @@ def create_new_experiment() -> Tuple[dict, str, str]:
     experiment_args = set_experiment_constraints(get_constraints(), experiment_args, experiment_parameters)
 
     try:
-        ax_client.create_experiment(**experiment_args)
+        create_ax_client_experiment(experiment_args)
         new_metrics = [Metric(k) for k in arg_result_names if k not in ax_client.metric_names]
         ax_client.experiment.add_tracking_metrics(new_metrics)
     except AssertionError as error:
@@ -5733,7 +6329,7 @@ def create_new_experiment() -> Tuple[dict, str, str]:
 
     return experiment_args, gpu_string, gpu_color
 
-def get_experiment_parameters(cli_params_experiment_parameters: Optional[dict | list]) -> Optional[Tuple[AxClient, dict, str, str]]:
+def get_experiment_parameters(cli_params_experiment_parameters: Optional[dict | list]) -> Optional[Tuple[dict, str, str]]:
     continue_previous_job = args.worker_generator_path or args.continue_previous_job
 
     check_ax_client()
@@ -5743,7 +6339,7 @@ def get_experiment_parameters(cli_params_experiment_parameters: Optional[dict |
     else:
         experiment_args, gpu_string, gpu_color = create_new_experiment()
 
-    return ax_client, experiment_args, gpu_string, gpu_color
+    return experiment_args, gpu_string, gpu_color
 
 def get_type_short(typename: str) -> str:
     if typename == "RangeParameter":
@@ -5802,7 +6398,6 @@ def parse_single_experiment_parameter_table(classic_params: Optional[Union[list,
                 _upper = param["bounds"][1]
 
             _possible_int_lower = str(helpers.to_int_when_possible(_lower))
-            #print(f"name: {_name}, _possible_int_lower: {_possible_int_lower}, lower: {_lower}")
             _possible_int_upper = str(helpers.to_int_when_possible(_upper))
 
             rows.append([_name, _short_type, _possible_int_lower, _possible_int_upper, "", value_type, log_scale])
@@ -5879,19 +6474,62 @@ def print_ax_parameter_constraints_table(experiment_args: dict) -> None:
 
     return None
 
+def check_base_for_print_overview() -> Optional[bool]:
+    if args.continue_previous_job is not None and arg_result_names is not None and len(arg_result_names) != 0 and original_result_names is not None and len(original_result_names) != 0:
+        print_yellow("--result_names will be ignored in continued jobs. The result names from the previous job will be used.")
+
+    if ax_client is None:
+        print_red("ax_client was None")
+        return None
+
+    if ax_client.experiment is None:
+        print_red("ax_client.experiment was None")
+        return None
+
+    if ax_client.experiment.optimization_config is None:
+        print_red("ax_client.experiment.optimization_config was None")
+        return None
+
+    return True
+
+def get_config_objectives() -> Any:
+    if not ax_client:
+        print_red("create_new_experiment: ax_client is None")
+        my_exit(101)
+
+        return None
+
+    config_objectives = None
+
+    if ax_client.experiment and ax_client.experiment.optimization_config:
+        opt_config = ax_client.experiment.optimization_config
+        if opt_config.is_moo_problem:
+            objective = getattr(opt_config, "objective", None)
+            if objective and getattr(objective, "objectives", None) is not None:
+                config_objectives = objective.objectives
+            else:
+                print_debug("ax_client.experiment.optimization_config.objective was None")
+        else:
+            config_objectives = [opt_config.objective]
+    else:
+        print_debug("ax_client.experiment or optimization_config was None")
+
+    return config_objectives
+
 def print_result_names_overview_table() -> None:
     if not ax_client:
         _fatal_error("Tried to access ax_client in print_result_names_overview_table, but it failed, because the ax_client was not defined.", 101)
 
         return None
 
-    if args.continue_previous_job is not None and args.result_names is not None and len(args.result_names) != 0 and original_result_names is not None and len(original_result_names) != 0:
-        print_yellow("--result_names will be ignored in continued jobs. The result names from the previous job will be used.")
+    if check_base_for_print_overview() is None:
+        return None
 
-    if ax_client.experiment.optimization_config.is_moo_problem:
-        config_objectives = ax_client.experiment.optimization_config.objective.objectives
-    else:
-        config_objectives = [ax_client.experiment.optimization_config.objective]
+    config_objectives = get_config_objectives()
+
+    if config_objectives is None:
+        print_red("config_objectives not found")
+        return None
 
     res_names = []
     res_min_max = []
@@ -5986,11 +6624,13 @@ def print_overview_tables(classic_params: Optional[Union[list, dict]], experimen
     print_result_names_overview_table()
 
 def update_progress_bar(nr: int) -> None:
+    log_data()
+
     if progress_bar is not None:
         try:
             progress_bar.update(nr)
         except Exception as e:
-            print(f"Error updating progress bar: {e}")
+            print_red(f"Error updating progress bar: {e}")
     else:
         print_red("update_progress_bar: progress_bar was None")
 
@@ -5998,19 +6638,17 @@ def get_current_model_name() -> str:
     if overwritten_to_random:
         return "Random*"
 
+    gs_model = "unknown model"
+
     if ax_client:
         try:
             if args.generation_strategy:
-                idx = getattr(ax_client.generation_strategy, "current_step_index", None)
+                idx = getattr(global_gs, "current_step_index", None)
                 if isinstance(idx, int):
                     if 0 <= idx < len(generation_strategy_names):
                         gs_model = generation_strategy_names[int(idx)]
-                    else:
-                        gs_model = "unknown model"
-                else:
-                    gs_model = "unknown model"
             else:
-                gs_model = getattr(ax_client.generation_strategy, "current_node_name", "unknown model")
+                gs_model = getattr(global_gs, "current_node_name", "unknown model")
 
             if gs_model:
                 return str(gs_model)
@@ -6117,7 +6755,7 @@ def count_jobs_in_squeue() -> tuple[int, str]:
     global _last_count_time, _last_count_result
 
     now = int(time.time())
-    if _last_count_result != (0, "") and now - _last_count_time < 15:
+    if _last_count_result != (0, "") and now - _last_count_time < 5:
         return _last_count_result
 
     _len = len(global_vars["jobs"])
@@ -6139,6 +6777,7 @@ def count_jobs_in_squeue() -> tuple[int, str]:
             check=True,
             text=True
         )
+
         if "slurm_load_jobs error" in result.stderr:
             _last_count_result = (_len, "Detected slurm_load_jobs error in stderr.")
             _last_count_time = now
@@ -6179,6 +6818,8 @@ def log_worker_numbers() -> None:
         if len(WORKER_PERCENTAGE_USAGE) == 0 or WORKER_PERCENTAGE_USAGE[len(WORKER_PERCENTAGE_USAGE) - 1] != this_values:
             WORKER_PERCENTAGE_USAGE.append(this_values)
 
+        write_worker_usage()
+
 def get_slurm_in_brackets(in_brackets: list) -> list:
     if is_slurm_job():
         workers_strings = get_workers_string()
@@ -6263,6 +6904,8 @@ def progressbar_description(new_msgs: Union[str, List[str]] = []) -> None:
     global last_progress_bar_desc
     global last_progress_bar_refresh_time
 
+    log_data()
+
     if isinstance(new_msgs, str):
         new_msgs = [new_msgs]
 
@@ -6280,7 +6923,7 @@ def progressbar_description(new_msgs: Union[str, List[str]] = []) -> None:
         print_red("Cannot update progress bar! It is None.")
 
 def clean_completed_jobs() -> None:
-    job_states_to_be_removed = ["early_stopped", "abandoned", "cancelled", "timeout", "interrupted", "failed", "preempted", "node_fail", "boot_fail"]
+    job_states_to_be_removed = ["early_stopped", "abandoned", "cancelled", "timeout", "interrupted", "failed", "preempted", "node_fail", "boot_fail", "finished"]
     job_states_to_be_ignored = ["ready", "completed", "unknown", "pending", "running", "completing", "out_of_memory", "requeued", "resv_del_hold"]
 
     for job, trial_index in global_vars["jobs"][:]:
@@ -6425,7 +7068,7 @@ def get_generation_node_for_index(
 
         return generation_node
     except Exception as e:
-        print(f"Error while get_generation_node_for_index: {e}")
+        print_red(f"Error while get_generation_node_for_index: {e}")
         return "MANUAL"
 
 def _get_generation_node_for_index_index_valid(
@@ -6598,11 +7241,21 @@ def check_ax_client() -> None:
     if ax_client is None or not ax_client:
         _fatal_error("insert_job_into_ax_client: ax_client was not defined where it should have been", 101)
 
+def attach_ax_client_data(arm_params: dict) -> Optional[Tuple[Any, int]]:
+    if not ax_client:
+        my_exit(101)
+
+        return None
+
+    new_trial = ax_client.attach_trial(arm_params)
+
+    return new_trial
+
 def attach_trial(arm_params: dict) -> Tuple[Any, int]:
     if ax_client is None:
         raise RuntimeError("attach_trial: ax_client was empty")
 
-    new_trial = ax_client.attach_trial(arm_params)
+    new_trial = attach_ax_client_data(arm_params)
     if not isinstance(new_trial, tuple) or len(new_trial) < 2:
         raise RuntimeError("attach_trial didn't return the expected tuple")
     return new_trial
@@ -6633,7 +7286,7 @@ def complete_trial_if_result(trial_idx: int, result: dict, __status: Optional[An
                 is_ok = False
 
         if is_ok:
-            ax_client.complete_trial(trial_index=trial_idx, raw_data=result)
+            complete_ax_client_trial(trial_idx, result)
             update_status(__status, base_str, "Completed trial")
         else:
             print_debug("Empty job encountered")
@@ -7030,7 +7683,7 @@ def get_parameters_from_outfile(stdout_path: str) -> Union[None, dict, str]:
         if not args.tests:
             original_print(f"get_parameters_from_outfile: The file '{stdout_path}' was not found.")
     except Exception as e:
-        print(f"get_parameters_from_outfile: There was an error: {e}")
+        print_red(f"get_parameters_from_outfile: There was an error: {e}")
 
     return None
 
@@ -7048,7 +7701,7 @@ def get_hostname_from_outfile(stdout_path: Optional[str]) -> Optional[str]:
         original_print(f"The file '{stdout_path}' was not found.")
         return None
     except Exception as e:
-        print(f"There was an error: {e}")
+        print_red(f"There was an error: {e}")
         return None
 
 def add_to_global_error_list(msg: str) -> None:
@@ -7084,22 +7737,70 @@ def mark_trial_as_failed(trial_index: int, _trial: Any) -> None:
 
             return None
 
-        ax_client.log_trial_failure(trial_index=trial_index)
+        log_ax_client_trial_failure(trial_index)
         _trial.mark_failed(unsafe=True)
     except ValueError as e:
         print_debug(f"mark_trial_as_failed error: {e}")
 
     return None
 
-def check_valid_result(result: Union[None, list, int, float, tuple]) -> bool:
+def check_valid_result(result: Union[None, dict]) -> bool:
     possible_val_not_found_values = [
         VAL_IF_NOTHING_FOUND,
         -VAL_IF_NOTHING_FOUND,
         -99999999999999997168788049560464200849936328366177157906432,
         99999999999999997168788049560464200849936328366177157906432
     ]
-    values_to_check = result if isinstance(result, list) else [result]
-    return result is not None and all(r not in possible_val_not_found_values for r in values_to_check)
+
+    def flatten_values(obj: Any) -> Any:
+        values = []
+        try:
+            if isinstance(obj, dict):
+                for v in obj.values():
+                    values.extend(flatten_values(v))
+            elif isinstance(obj, (list, tuple, set)):
+                for v in obj:
+                    values.extend(flatten_values(v))
+            else:
+                values.append(obj)
+        except Exception as e:
+            print_red(f"Error while flattening values: {e}")
+        return values
+
+    if result is None:
+        return False
+
+    try:
+        all_values = flatten_values(result)
+        for val in all_values:
+            if val in possible_val_not_found_values:
+                return False
+        return True
+    except Exception as e:
+        print_red(f"Error while checking result validity: {e}")
+        return False
+
+def update_ax_client_trial(trial_idx: int, result: Union[list, dict]) -> None:
+    if not ax_client:
+        my_exit(101)
+
+        return None
+
+    trial = get_trial_by_index(trial_idx)
+
+    trial.update_trial_data(raw_data=result)
+
+    return None
+
+def complete_ax_client_trial(trial_idx: int, result: Union[list, dict]) -> None:
+    if not ax_client:
+        my_exit(101)
+
+        return None
+
+    ax_client.complete_trial(trial_index=trial_idx, raw_data=result)
+
+    return None
 
 def _finish_job_core_helper_complete_trial(trial_index: int, raw_result: dict) -> None:
     if ax_client is None:
@@ -7108,25 +7809,64 @@ def _finish_job_core_helper_complete_trial(trial_index: int, raw_result: dict) -
 
     try:
         print_debug(f"Completing trial: {trial_index} with result: {raw_result}...")
-        ax_client.complete_trial(trial_index=trial_index, raw_data=raw_result)
+        complete_ax_client_trial(trial_index, raw_result)
         print_debug(f"Completing trial: {trial_index} with result: {raw_result}... Done!")
     except ax.exceptions.core.UnsupportedError as e:
         if f"{e}":
             print_debug(f"Completing trial: {trial_index} with result: {raw_result} after failure. Trying to update trial...")
-            ax_client.update_trial_data(trial_index=trial_index, raw_data=raw_result)
+            update_ax_client_trial(trial_index, raw_result)
             print_debug(f"Completing trial: {trial_index} with result: {raw_result} after failure... Done!")
         else:
             _fatal_error(f"Error completing trial: {e}", 234)
 
     return None
 
-def _finish_job_core_helper_mark_success(_trial: ax.core.trial.Trial, result: Union[float, int, tuple]) -> None:
+def format_result_for_display(result: dict) -> str:
+    def safe_float(v: Any) -> str:
+        try:
+            if v is None:
+                return "None"
+            if isinstance(v, (int, float)):
+                if math.isnan(v):
+                    return "NaN"
+                if math.isinf(v):
+                    return "∞" if v > 0 else "-∞"
+                return f"{v:.6f}"
+            return str(v)
+        except Exception as e:
+            return f"<error: {e}>"
+
+    try:
+        if not isinstance(result, dict):
+            return safe_float(result)
+
+        parts = []
+        for key, val in result.items():
+            try:
+                if isinstance(val, (list, tuple)) and len(val) == 2:
+                    main, sem = val
+                    main_str = safe_float(main)
+                    if sem is not None:
+                        sem_str = safe_float(sem)
+                        parts.append(f"{key}: {main_str} (SEM: {sem_str})")
+                    else:
+                        parts.append(f"{key}: {main_str}")
+                else:
+                    parts.append(f"{key}: {safe_float(val)}")
+            except Exception as e:
+                parts.append(f"{key}: <error: {e}>")
+
+        return ", ".join(parts)
+    except Exception as e:
+        return f"<error formatting result: {e}>"
+
+def _finish_job_core_helper_mark_success(_trial: ax.core.trial.Trial, result: dict) -> None:
     print_debug(f"Marking trial {_trial} as completed")
     _trial.mark_completed(unsafe=True)
 
     succeeded_jobs(1)
 
-    progressbar_description(f"new result: {result}")
+    progressbar_description(f"new result: {format_result_for_display(result)}")
     update_progress_bar(1)
 
     save_results_csv()
@@ -7140,8 +7880,8 @@ def _finish_job_core_helper_mark_failure(job: Any, trial_index: int, _trial: Any
     if job:
         try:
             progressbar_description("job_failed")
-            ax_client.log_trial_failure(trial_index=trial_index)
-            _trial.mark_failed(unsafe=True)
+            log_ax_client_trial_failure(trial_index)
+            mark_trial_as_failed(trial_index, _trial)
         except Exception as e:
             print_red(f"\nERROR while trying to mark job as failure: {e}")
         job.cancel()
@@ -7158,16 +7898,16 @@ def finish_job_core(job: Any, trial_index: int, this_jobs_finished: int) -> int:
     result = job.result()
     print_debug(f"finish_job_core: trial-index: {trial_index}, job.result(): {result}, state: {state_from_job(job)}")
 
-    raw_result = result
-    result_keys = list(result.keys())
-    result = result[result_keys[0]]
     this_jobs_finished += 1
 
     if ax_client:
-        _trial = ax_client.get_trial(trial_index)
+        _trial = get_ax_client_trial(trial_index)
+
+        if _trial is None:
+            return 0
 
         if check_valid_result(result):
-            _finish_job_core_helper_complete_trial(trial_index, raw_result)
+            _finish_job_core_helper_complete_trial(trial_index, result)
 
             try:
                 _finish_job_core_helper_mark_success(_trial, result)
@@ -7175,7 +7915,7 @@ def finish_job_core(job: Any, trial_index: int, this_jobs_finished: int) -> int:
                 if len(arg_result_names) > 1 and count_done_jobs() > 1 and not job_calculate_pareto_front(get_current_run_folder(), True):
                     print_red("job_calculate_pareto_front post job failed")
             except Exception as e:
-                print(f"ERROR in line {get_line_info()}: {e}")
+                print_red(f"ERROR in line {get_line_info()}: {e}")
         else:
             _finish_job_core_helper_mark_failure(job, trial_index, _trial)
     else:
@@ -7184,6 +7924,8 @@ def finish_job_core(job: Any, trial_index: int, this_jobs_finished: int) -> int:
     print_debug(f"finish_job_core: removing job {job}, trial_index: {trial_index}")
     global_vars["jobs"].remove((job, trial_index))
 
+    log_data()
+
     force_live_share()
 
     return this_jobs_finished
@@ -7196,11 +7938,14 @@ def _finish_previous_jobs_helper_handle_failed_job(job: Any, trial_index: int) -
     if job:
         try:
             progressbar_description("job_failed")
-            _trial = ax_client.get_trial(trial_index)
-            ax_client.log_trial_failure(trial_index=trial_index)
+            _trial = get_ax_client_trial(trial_index)
+            if _trial is None:
+                return None
+
+            log_ax_client_trial_failure(trial_index)
             mark_trial_as_failed(trial_index, _trial)
         except Exception as e:
-            print(f"ERROR in line {get_line_info()}: {e}")
+            print_debug(f"ERROR in line {get_line_info()}: {e}")
         job.cancel()
         orchestrate_job(job, trial_index)
 
@@ -7215,10 +7960,12 @@ def _finish_previous_jobs_helper_handle_failed_job(job: Any, trial_index: int) -
 
 def _finish_previous_jobs_helper_handle_exception(job: Any, trial_index: int, error: Exception) -> int:
     if "None for metric" in str(error):
-        print_red(
-            f"\n⚠ It seems like the program that was about to be run didn't have 'RESULT: <FLOAT>' in it's output string."
-            f"\nError: {error}\nJob-result: {job.result()}"
-        )
+        err_msg = f"\n⚠ It seems like the program that was about to be run didn't have 'RESULT: <FLOAT>' in it's output string.\nError: {error}\nJob-result: {job.result()}"
+
+        if count_done_jobs() == 0:
+            print_red(err_msg)
+        else:
+            print_debug(err_msg)
     else:
         print_red(f"\n⚠ {error}")
 
@@ -7237,7 +7984,10 @@ def _finish_previous_jobs_helper_process_job(job: Any, trial_index: int, this_jo
         this_jobs_finished += _finish_previous_jobs_helper_handle_exception(job, trial_index, error)
     return this_jobs_finished
 
-def _finish_previous_jobs_helper_check_and_process(job: Any, trial_index: int, this_jobs_finished: int) -> int:
+def _finish_previous_jobs_helper_check_and_process(__args: Tuple[Any, int]) -> int:
+    job, trial_index = __args
+
+    this_jobs_finished = 0
     if job is None:
         print_debug(f"finish_previous_jobs: job {job} is None")
         return this_jobs_finished
@@ -7250,10 +8000,6 @@ def _finish_previous_jobs_helper_check_and_process(job: Any, trial_index: int, t
 
     return this_jobs_finished
 
-def _finish_previous_jobs_helper_wrapper(__args: Tuple[Any, int]) -> int:
-    job, trial_index = __args
-    return _finish_previous_jobs_helper_check_and_process(job, trial_index, 0)
-
 def finish_previous_jobs(new_msgs: List[str] = []) -> None:
     global JOBS_FINISHED
 
@@ -7266,13 +8012,10 @@ def finish_previous_jobs(new_msgs: List[str] = []) -> None:
 
     jobs_copy = global_vars["jobs"][:]
 
-    if len(jobs_copy) > 0:
-        print_debug(f"jobs in finish_previous_jobs: {jobs_copy}")
-
-    finishing_jobs_start_time = time.time()
+    #finishing_jobs_start_time = time.time()
 
     with ThreadPoolExecutor() as finish_job_executor:
-        futures = [finish_job_executor.submit(_finish_previous_jobs_helper_wrapper, (job, trial_index)) for job, trial_index in jobs_copy]
+        futures = [finish_job_executor.submit(_finish_previous_jobs_helper_check_and_process, (job, trial_index)) for job, trial_index in jobs_copy]
 
         for future in as_completed(futures):
             try:
@@ -7280,11 +8023,11 @@ def finish_previous_jobs(new_msgs: List[str] = []) -> None:
             except Exception as e:
                 print_red(f"⚠ Exception in parallel job handling: {e}")
 
-    finishing_jobs_end_time = time.time()
+    #finishing_jobs_end_time = time.time()
 
-    finishing_jobs_runtime = finishing_jobs_end_time - finishing_jobs_start_time
+    #finishing_jobs_runtime = finishing_jobs_end_time - finishing_jobs_start_time
 
-    print_debug(f"Finishing jobs took {finishing_jobs_runtime} second(s)")
+    #print_debug(f"Finishing jobs took {finishing_jobs_runtime} second(s)")
 
     if this_jobs_finished > 0:
         save_results_csv()
@@ -7450,24 +8193,43 @@ def submit_new_job(parameters: Union[dict, str], trial_index: int) -> Any:
 
     print_debug(f"Done submitting new job, took {elapsed} seconds")
 
+    log_data()
+
     return new_job
 
+def get_ax_client_trial(trial_index: int) -> Optional[ax.core.trial.Trial]:
+    if not ax_client:
+        my_exit(101)
+
+        return None
+
+    try:
+        log_data()
+
+        return ax_client.get_trial(trial_index)
+    except KeyError:
+        error_without_print(f"get_ax_client_trial: trial_index {trial_index} failed")
+        return None
+
 def orchestrator_start_trial(parameters: Union[dict, str], trial_index: int) -> None:
     if submitit_executor and ax_client:
         new_job = submit_new_job(parameters, trial_index)
         if new_job:
             submitted_jobs(1)
 
-            _trial = ax_client.get_trial(trial_index)
+            _trial = get_ax_client_trial(trial_index)
 
-            try:
-                _trial.mark_staged(unsafe=True)
-            except Exception as e:
-                print_debug(f"orchestrator_start_trial: error {e}")
-            _trial.mark_running(unsafe=True, no_runner_required=True)
+            if _trial is not None:
+                try:
+                    _trial.mark_staged(unsafe=True)
+                except Exception as e:
+                    print_debug(f"orchestrator_start_trial: error {e}")
+                _trial.mark_running(unsafe=True, no_runner_required=True)
 
-            print_debug(f"orchestrator_start_trial: appending job {new_job} to global_vars['jobs'], trial_index: {trial_index}")
-            global_vars["jobs"].append((new_job, trial_index))
+                print_debug(f"orchestrator_start_trial: appending job {new_job} to global_vars['jobs'], trial_index: {trial_index}")
+                global_vars["jobs"].append((new_job, trial_index))
+            else:
+                print_red("Trial was none in orchestrator_start_trial")
         else:
             print_red("orchestrator_start_trial: Failed to start new job")
     elif ax_client:
@@ -7492,7 +8254,7 @@ def handle_restart(stdout_path: str, trial_index: int) -> None:
     if parameters:
         orchestrator_start_trial(parameters, trial_index)
     else:
-        print(f"Could not determine parameters from outfile {stdout_path} for restarting job")
+        print_red(f"Could not determine parameters from outfile {stdout_path} for restarting job")
 
 def check_alternate_path(path: str) -> str:
     if os.path.exists(path):
@@ -7588,7 +8350,11 @@ def execute_evaluation(_params: list) -> Optional[int]:
 
         return None
 
-    _trial = ax_client.get_trial(trial_index)
+    _trial = get_ax_client_trial(trial_index)
+
+    if _trial is None:
+        error_without_print(f"execute_evaluation: _trial was not in execute_evaluation for params {_params}")
+        return None
 
     def mark_trial_stage(stage: str, error_msg: str) -> None:
         try:
@@ -7656,7 +8422,7 @@ def handle_failed_job(error: Union[None, Exception, str], trial_index: int, new_
             my_exit(144)
 
     if new_job is None:
-        print_red("handle_failed_job: job is None")
+        print_debug("handle_failed_job: job is None")
 
         return None
 
@@ -7667,16 +8433,24 @@ def handle_failed_job(error: Union[None, Exception, str], trial_index: int, new_
 
     return None
 
+def log_ax_client_trial_failure(trial_index: int) -> None:
+    if not ax_client:
+        my_exit(101)
+
+        return
+
+    ax_client.log_trial_failure(trial_index=trial_index)
+
 def cancel_failed_job(trial_index: int, new_job: Job) -> None:
     print_debug("Trying to cancel job that failed")
     if new_job:
         try:
             if ax_client:
-                ax_client.log_trial_failure(trial_index=trial_index)
+                log_ax_client_trial_failure(trial_index)
             else:
                 _fatal_error("ax_client not defined", 101)
         except Exception as e:
-            print(f"ERROR in line {get_line_info()}: {e}")
+            print_red(f"ERROR in line {get_line_info()}: {e}")
         new_job.cancel()
 
         print_debug(f"cancel_failed_job: removing job {new_job}, trial_index: {trial_index}")
@@ -7730,6 +8504,8 @@ def break_run_search(_name: str, _max_eval: Optional[int]) -> bool:
     _submitted_jobs = submitted_jobs()
     _failed_jobs = failed_jobs()
 
+    log_data()
+
     max_failed_jobs = max_eval
 
     if args.max_failed_jobs is not None and args.max_failed_jobs > 0:
@@ -7761,7 +8537,7 @@ def break_run_search(_name: str, _max_eval: Optional[int]) -> bool:
 
     return _ret
 
-def _calculate_nr_of_jobs_to_get(simulated_jobs: int, currently_running_jobs: int) -> int:
+def calculate_nr_of_jobs_to_get(simulated_jobs: int, currently_running_jobs: int) -> int:
     """Calculates the number of jobs to retrieve."""
     return min(
         max_eval + simulated_jobs - count_done_jobs(),
@@ -7774,7 +8550,7 @@ def remove_extra_spaces(text: str) -> str:
         raise ValueError("Input must be a string")
     return re.sub(r'\s+', ' ', text).strip()
 
-def _get_trials_message(nr_of_jobs_to_get: int, full_nr_of_jobs_to_get: int, trial_durations: List[float]) -> str:
+def get_trials_message(nr_of_jobs_to_get: int, full_nr_of_jobs_to_get: int, trial_durations: List[float]) -> str:
     """Generates the appropriate message for the number of trials being retrieved."""
     ret = ""
     if full_nr_of_jobs_to_get > 1:
@@ -7859,51 +8635,52 @@ def get_batched_arms(nr_of_jobs_to_get: int) -> list:
         print_red("get_batched_arms: ax_client was None")
         return []
 
-    # Experiment-Status laden
     load_experiment_state()
 
-    while len(batched_arms) != nr_of_jobs_to_get:
+    while len(batched_arms) < nr_of_jobs_to_get:
         if attempts > args.max_attempts_for_generation:
             print_debug(f"get_batched_arms: Stopped after {attempts} attempts: could not generate enough arms "
                         f"(got {len(batched_arms)} out of {nr_of_jobs_to_get}).")
             break
 
-        remaining = nr_of_jobs_to_get - len(batched_arms)
-        print_debug(f"get_batched_arms: Attempt {attempts + 1}: requesting {remaining} more arm(s).")
-
-        print_debug("get pending observations")
-        t0 = time.time()
-        pending_observations = get_pending_observation_features(experiment=ax_client.experiment)
-        dt = time.time() - t0
-        print_debug(f"got pending observations: {pending_observations} (took {dt:.2f} seconds)")
+        #print_debug(f"get_batched_arms: Attempt {attempts + 1}: requesting 1 more arm")
 
-        print_debug("getting global_gs.gen()")
-        batched_generator_run = global_gs.gen(
+        pending_observations = get_pending_observation_features(
             experiment=ax_client.experiment,
-            n=remaining,
-            pending_observations=pending_observations
+            include_out_of_design_points=True
         )
-        print_debug(f"got global_gs.gen(): {batched_generator_run}")
+
+        try:
+            #print_debug("getting global_gs.gen() with n=1")
+
+            batched_generator_run: Any = global_gs.gen(
+                experiment=ax_client.experiment,
+                n=1,
+                pending_observations=pending_observations,
+            )
+            print_debug(f"got global_gs.gen(): {batched_generator_run}")
+        except Exception as e:
+            print_debug(f"global_gs.gen failed: {e}")
+            traceback.print_exception(type(e), e, e.__traceback__, file=sys.stderr)
+            break
 
         depth = 0
         path = "batched_generator_run"
-        while isinstance(batched_generator_run, (list, tuple)) and len(batched_generator_run) > 0:
-            print_debug(f"Depth {depth}, path {path}, type {type(batched_generator_run).__name__}, length {len(batched_generator_run)}: {batched_generator_run}")
+        while isinstance(batched_generator_run, (list, tuple)) and len(batched_generator_run) == 1:
+            #print_debug(f"Depth {depth}, path {path}, type {type(batched_generator_run).__name__}, length {len(batched_generator_run)}: {batched_generator_run}")
             batched_generator_run = batched_generator_run[0]
             path += "[0]"
             depth += 1
 
-        print_debug(f"Final flat object at depth {depth}, path {path}: {batched_generator_run} (type {type(batched_generator_run).__name__})")
+        #print_debug(f"Final flat object at depth {depth}, path {path}: {batched_generator_run} (type {type(batched_generator_run).__name__})")
 
-        print_debug("got new arms")
-        new_arms = batched_generator_run.arms
-        print_debug(f"new_arms: {new_arms}")
+        new_arms = getattr(batched_generator_run, "arms", [])
         if not new_arms:
             print_debug("get_batched_arms: No new arms were generated in this attempt.")
         else:
-            print_debug(f"get_batched_arms: Generated {len(new_arms)} new arm(s), wanted {nr_of_jobs_to_get}.")
+            print_debug(f"get_batched_arms: Generated {len(new_arms)} new arm(s), now at {len(batched_arms) + len(new_arms)} of {nr_of_jobs_to_get}.")
+            batched_arms.extend(new_arms)
 
-        batched_arms.extend(new_arms)
         attempts += 1
 
     print_debug(f"get_batched_arms: Finished with {len(batched_arms)} arm(s) after {attempts} attempt(s).")
@@ -7942,16 +8719,14 @@ def generate_trials(n: int, recursion: bool) -> Tuple[Dict[int, Any], bool]:
                     retries += 1
                     continue
 
-                print_debug(f"Fetching trial {cnt + 1}/{n}...")
-                progressbar_description(_get_trials_message(cnt + 1, n, trial_durations))
+                progressbar_description(get_trials_message(cnt + 1, n, trial_durations))
 
                 try:
                     result = create_and_handle_trial(arm)
                     if result is not None:
                         trial_index, trial_duration, trial_successful = result
-
                 except TrialRejected as e:
-                    print_debug(f"Trial rejected: {e}")
+                    print_debug(f"generate_trials: Trial rejected, error: {e}")
                     retries += 1
                     continue
 
@@ -7961,14 +8736,23 @@ def generate_trials(n: int, recursion: bool) -> Tuple[Dict[int, Any], bool]:
                     cnt += 1
                     trials_dict[trial_index] = arm.parameters
 
-        return _finalize_generation(trials_dict, cnt, n, start_time)
+        finalized = finalize_generation(trials_dict, cnt, n, start_time)
+
+        return finalized
 
     except Exception as e:
-        return _handle_generation_failure(e, n, recursion)
+        return handle_generation_failure(e, n, recursion)
 
 class TrialRejected(Exception):
     pass
 
+def mark_abandoned(trial: Any, reason: str, trial_index: int) -> None:
+    try:
+        print_debug(f"[INFO] Marking trial {trial.index} ({trial.arm.name}) as abandoned, trial-index: {trial_index}. Reason: {reason}")
+        trial.mark_abandoned(reason)
+    except Exception as e:
+        print_red(f"[ERROR] Could not mark trial as abandoned: {e}")
+
 def create_and_handle_trial(arm: Any) -> Optional[Tuple[int, float, bool]]:
     if ax_client is None:
         print_red("ax_client is None in create_and_handle_trial")
@@ -7994,7 +8778,7 @@ def create_and_handle_trial(arm: Any) -> Optional[Tuple[int, float, bool]]:
     arm = trial.arms[0]
     if deduplicated_arm(arm):
         print_debug(f"Duplicated arm: {arm}")
-        trial.mark_abandoned(reason="Duplication detected")
+        mark_abandoned(trial, "Duplication detected", trial_index)
         raise TrialRejected("Duplicate arm.")
 
     arms_by_name_for_deduplication[arm.name] = arm
@@ -8003,7 +8787,7 @@ def create_and_handle_trial(arm: Any) -> Optional[Tuple[int, float, bool]]:
 
     if not has_no_post_generation_constraints_or_matches_constraints(post_generation_constraints, params):
         print_debug(f"Trial {trial_index} does not meet post-generation constraints. Marking abandoned. Params: {params}, constraints: {post_generation_constraints}")
-        trial.mark_abandoned(reason="Post-Generation-Constraint failed")
+        mark_abandoned(trial, "Post-Generation-Constraint failed", trial_index)
         abandoned_trial_indices.append(trial_index)
         raise TrialRejected("Post-generation constraints not met.")
 
@@ -8011,7 +8795,7 @@ def create_and_handle_trial(arm: Any) -> Optional[Tuple[int, float, bool]]:
     end = time.time()
     return trial_index, float(end - start), True
 
-def _finalize_generation(trials_dict: Dict[int, Any], cnt: int, requested: int, start_time: float) -> Tuple[Dict[int, Any], bool]:
+def finalize_generation(trials_dict: Dict[int, Any], cnt: int, requested: int, start_time: float) -> Tuple[Dict[int, Any], bool]:
     total_time = time.time() - start_time
 
     log_gen_times.append(total_time)
@@ -8022,7 +8806,7 @@ def _finalize_generation(trials_dict: Dict[int, Any], cnt: int, requested: int,
 
     return trials_dict, False
 
-def _handle_generation_failure(
+def handle_generation_failure(
     e: Exception,
     requested: int,
     recursion: bool
@@ -8038,19 +8822,19 @@ def _handle_generation_failure(
     )):
         msg = str(e)
         if msg not in error_8_saved:
-            _print_exhaustion_warning(e, recursion)
+            print_exhaustion_warning(e, recursion)
             error_8_saved.append(msg)
 
         if not recursion and args.revert_to_random_when_seemingly_exhausted:
             print_debug("Switching to random search strategy.")
-            set_global_gs_to_random()
+            set_global_gs_to_sobol()
             return fetch_next_trials(requested, True)
 
-    print_red(f"_handle_generation_failure: General Exception: {e}")
+    print_red(f"handle_generation_failure: General Exception: {e}")
 
     return {}, True
 
-def _print_exhaustion_warning(e: Exception, recursion: bool) -> None:
+def print_exhaustion_warning(e: Exception, recursion: bool) -> None:
     if not recursion and args.revert_to_random_when_seemingly_exhausted:
         print_yellow(f"\n⚠Error 8: {e} From now (done jobs: {count_done_jobs()}) on, random points will be generated.")
     else:
@@ -8095,21 +8879,24 @@ def get_model_gen_kwargs() -> dict:
         "fit_out_of_design": args.fit_out_of_design
     }
 
-def set_global_gs_to_random() -> None:
+def set_global_gs_to_sobol() -> None:
     global global_gs
     global overwritten_to_random
 
+    print("Reverting to SOBOL")
+
     global_gs = GenerationStrategy(
         name="Random*",
         nodes=[
             GenerationNode(
-                node_name="Sobol",
-                generator_specs=[
-                    GeneratorSpec(
-                        Models.SOBOL,
-                        model_gen_kwargs=get_model_gen_kwargs()
-                    )
-                ]
+                name="Sobol",
+                should_deduplicate=True,
+                generator_specs=[ # type: ignore[arg-type]
+                    GeneratorSpec( # type: ignore[arg-type]
+                        Generators.SOBOL, # type: ignore[arg-type]
+                        model_gen_kwargs=get_model_gen_kwargs() # type: ignore[arg-type]
+                    ) # type: ignore[arg-type]
+                ] # type: ignore[arg-type]
             )
         ]
     )
@@ -8494,10 +9281,10 @@ def parse_generation_strategy_string(gen_strat_str: str) -> tuple[list[dict[str,
 
     for s in splitted_by_comma:
         if "=" not in s:
-            print(f"'{s}' does not contain '='")
+            print_red(f"'{s}' does not contain '='")
             my_exit(123)
         if s.count("=") != 1:
-            print(f"There can only be one '=' in the gen_strat_str's element '{s}'")
+            print_red(f"There can only be one '=' in the gen_strat_str's element '{s}'")
             my_exit(123)
 
         model_name, nr_str = s.split("=")
@@ -8507,13 +9294,13 @@ def parse_generation_strategy_string(gen_strat_str: str) -> tuple[list[dict[str,
             _fatal_error(f"Model {matching_model} is not valid for custom generation strategy.", 56)
 
         if not matching_model:
-            print(f"'{model_name}' not found in SUPPORTED_MODELS")
+            print_red(f"'{model_name}' not found in SUPPORTED_MODELS")
             my_exit(123)
 
         try:
             nr = int(nr_str)
         except ValueError:
-            print(f"Invalid number of generations '{nr_str}' for model '{model_name}'")
+            print_red(f"Invalid number of generations '{nr_str}' for model '{model_name}'")
             my_exit(123)
 
         gen_strat_list.append({matching_model: nr})
@@ -8637,7 +9424,7 @@ def create_node(model_name: str, threshold: int, next_model_name: Optional[str])
     if model_name == "TPE":
         if len(arg_result_names) != 1:
             _fatal_error(f"Has {len(arg_result_names)} results. TPE currently only supports single-objective-optimization.", 108)
-        return ExternalProgramGenerationNode(external_generator=f"python3 {script_dir}/.tpe.py", node_name="EXTERNAL_GENERATOR")
+        return ExternalProgramGenerationNode(external_generator=f"python3 {script_dir}/.tpe.py", name="EXTERNAL_GENERATOR")
 
     external_generators = {
         "PSEUDORANDOM": f"python3 {script_dir}/.random_generator.py",
@@ -8651,10 +9438,10 @@ def create_node(model_name: str, threshold: int, next_model_name: Optional[str])
         cmd = external_generators[model_name]
         if model_name == "EXTERNAL_GENERATOR" and not cmd:
             _fatal_error("--external_generator is missing. Cannot create points for EXTERNAL_GENERATOR without it.", 204)
-        return ExternalProgramGenerationNode(external_generator=cmd, node_name="EXTERNAL_GENERATOR")
+        return ExternalProgramGenerationNode(external_generator=cmd, name="EXTERNAL_GENERATOR")
 
     trans_crit = [
-        MaxTrials(
+        MinTrials(
             threshold=threshold,
             block_transition_if_unmet=True,
             transition_to=target_model,
@@ -8670,11 +9457,12 @@ def create_node(model_name: str, threshold: int, next_model_name: Optional[str])
     if model_name.lower() != "sobol":
         kwargs["model_kwargs"] = get_model_kwargs()
 
-    model_spec = [GeneratorSpec(selected_model, **kwargs)]
+    model_spec = [GeneratorSpec(selected_model, **kwargs)] # type: ignore[arg-type]
 
     res = GenerationNode(
-        node_name=model_name,
+        name=model_name,
         generator_specs=model_spec,
+        should_deduplicate=True,
         transition_criteria=trans_crit
     )
 
@@ -8685,7 +9473,7 @@ def get_optimizer_kwargs() -> dict:
         "sequential": False
     }
 
-def create_step(model_name: str, _num_trials: int = -1, index: Optional[int] = None) -> GenerationStep:
+def create_step(model_name: str, _num_trials: int, index: int) -> GenerationStep:
     model_enum = get_model_from_name(model_name)
 
     return GenerationStep(
@@ -8700,17 +9488,16 @@ def create_step(model_name: str, _num_trials: int = -1, index: Optional[int] = N
     )
 
 def set_global_generation_strategy() -> None:
-    with spinner("Setting global generation strategy"):
-        continue_not_supported_on_custom_generation_strategy()
+    continue_not_supported_on_custom_generation_strategy()
 
-        try:
-            if args.generation_strategy is None:
-                setup_default_generation_strategy()
-            else:
-                setup_custom_generation_strategy()
-        except Exception as e:
-            print_red(f"Unexpected error in generation strategy setup: {e}")
-            my_exit(111)
+    try:
+        if args.generation_strategy is None:
+            setup_default_generation_strategy()
+        else:
+            setup_custom_generation_strategy()
+    except Exception as e:
+        print_red(f"Unexpected error in generation strategy setup: {e}")
+        my_exit(111)
 
     if global_gs is None:
         print_red("global_gs is None after setup!")
@@ -8861,10 +9648,14 @@ def execute_trials(
                 result = future.result()
                 print_debug(f"result in execute_trials: {result}")
             except Exception as exc:
-                print_red(f"execute_trials: Error at executing a trial: {exc}")
+                failed_args = future_to_args[future]
+                print_red(f"execute_trials: Error at executing a trial with args {failed_args}: {exc}")
+                traceback.print_exc()
 
     end_time = time.time()
 
+    log_data()
+
     duration = float(end_time - start_time)
     job_submit_durations.append(duration)
     job_submit_nrs.append(cnt)
@@ -8898,20 +9689,20 @@ def create_and_execute_next_runs(next_nr_steps: int, phase: Optional[str], _max_
     done_optimizing: bool = False
 
     try:
-        done_optimizing, trial_index_to_param = _create_and_execute_next_runs_run_loop(_max_eval, phase)
-        _create_and_execute_next_runs_finish(done_optimizing)
+        done_optimizing, trial_index_to_param = create_and_execute_next_runs_run_loop(_max_eval, phase)
+        create_and_execute_next_runs_finish(done_optimizing)
     except Exception as e:
         stacktrace = traceback.format_exc()
         print_debug(f"Warning: create_and_execute_next_runs encountered an exception: {e}\n{stacktrace}")
         return handle_exceptions_create_and_execute_next_runs(e)
 
-    return _create_and_execute_next_runs_return_value(trial_index_to_param)
+    return create_and_execute_next_runs_return_value(trial_index_to_param)
 
-def _create_and_execute_next_runs_run_loop(_max_eval: Optional[int], phase: Optional[str]) -> Tuple[bool, Optional[Dict]]:
+def create_and_execute_next_runs_run_loop(_max_eval: Optional[int], phase: Optional[str]) -> Tuple[bool, Optional[Dict]]:
     done_optimizing = False
     trial_index_to_param: Optional[Dict] = None
 
-    nr_of_jobs_to_get = _calculate_nr_of_jobs_to_get(get_nr_of_imported_jobs(), len(global_vars["jobs"]))
+    nr_of_jobs_to_get = calculate_nr_of_jobs_to_get(get_nr_of_imported_jobs(), len(global_vars["jobs"]))
 
     __max_eval = _max_eval if _max_eval is not None else 0
     new_nr_of_jobs_to_get = min(__max_eval - (submitted_jobs() - failed_jobs()), nr_of_jobs_to_get)
@@ -8925,6 +9716,7 @@ def _create_and_execute_next_runs_run_loop(_max_eval: Optional[int], phase: Opti
 
     for _ in range(range_nr):
         trial_index_to_param, done_optimizing = get_next_trials(get_next_trials_nr)
+        log_data()
         if done_optimizing:
             continue
 
@@ -8949,13 +9741,13 @@ def _create_and_execute_next_runs_run_loop(_max_eval: Optional[int], phase: Opti
 
     return done_optimizing, trial_index_to_param
 
-def _create_and_execute_next_runs_finish(done_optimizing: bool) -> None:
+def create_and_execute_next_runs_finish(done_optimizing: bool) -> None:
     finish_previous_jobs(["finishing jobs"])
 
     if done_optimizing:
         end_program(False, 0)
 
-def _create_and_execute_next_runs_return_value(trial_index_to_param: Optional[Dict]) -> int:
+def create_and_execute_next_runs_return_value(trial_index_to_param: Optional[Dict]) -> int:
     try:
         if trial_index_to_param:
             res = len(trial_index_to_param.keys())
@@ -9066,7 +9858,7 @@ def execute_nvidia_smi() -> None:
                 if not host:
                     print_debug("host not defined")
         except Exception as e:
-            print(f"execute_nvidia_smi: An error occurred: {e}")
+            print_red(f"execute_nvidia_smi: An error occurred: {e}")
         if is_slurm_job() and not args.force_local_execution:
             _sleep(30)
 
@@ -9104,11 +9896,6 @@ def run_search() -> bool:
 
     return False
 
-async def start_logging_daemon() -> None:
-    while True:
-        log_data()
-        time.sleep(30)
-
 def should_break_search() -> bool:
     ret = False
 
@@ -9157,10 +9944,8 @@ def check_search_space_exhaustion(nr_of_items: int) -> bool:
         print_debug(_wrn)
         progressbar_description(_wrn)
 
-        live_share()
         return True
 
-    live_share()
     return False
 
 def finalize_jobs() -> None:
@@ -9174,7 +9959,7 @@ def finalize_jobs() -> None:
         handle_slurm_execution()
 
 def go_through_jobs_that_are_not_completed_yet() -> None:
-    print_debug(f"Waiting for jobs to finish (currently, len(global_vars['jobs']) = {len(global_vars['jobs'])}")
+    #print_debug(f"Waiting for jobs to finish (currently, len(global_vars['jobs']) = {len(global_vars['jobs'])}")
 
     nr_jobs_left = len(global_vars['jobs'])
     if nr_jobs_left == 1:
@@ -9244,372 +10029,60 @@ def parse_orchestrator_file(_f: str, _test: bool = False) -> Union[dict, None]:
 
                 return data
             except Exception as e:
-                print(f"Error while parse_experiment_parameters({_f}): {e}")
+                print_red(f"Error while parse_experiment_parameters({_f}): {e}")
     else:
         print_red(f"{_f} could not be found")
 
-    return None
-
-def set_orchestrator() -> None:
-    with spinner("Setting orchestrator..."):
-        global orchestrator
-
-        if args.orchestrator_file:
-            if SYSTEM_HAS_SBATCH:
-                orchestrator = parse_orchestrator_file(args.orchestrator_file, False)
-            else:
-                print_yellow("--orchestrator_file will be ignored on non-sbatch-systems.")
-
-def check_if_has_random_steps() -> None:
-    if (not args.continue_previous_job and "--continue" not in sys.argv) and (args.num_random_steps == 0 or not args.num_random_steps) and args.model not in ["EXTERNAL_GENERATOR", "SOBOL", "PSEUDORANDOM"]:
-        _fatal_error("You have no random steps set. This is only allowed in continued jobs. To start, you need either some random steps, or a continued run.", 233)
-
-def add_exclude_to_defective_nodes() -> None:
-    with spinner("Adding excluded nodes..."):
-        if args.exclude:
-            entries = [entry.strip() for entry in args.exclude.split(',')]
-
-            for entry in entries:
-                count_defective_nodes(None, entry)
-
-def check_max_eval(_max_eval: int) -> None:
-    with spinner("Checking max_eval..."):
-        if not _max_eval:
-            _fatal_error("--max_eval needs to be set!", 19)
-
-def parse_parameters() -> Any:
-    cli_params_experiment_parameters = None
-    if args.parameter:
-        parse_experiment_parameters()
-        cli_params_experiment_parameters = experiment_parameters
-
-    return cli_params_experiment_parameters
-
-def create_pareto_front_table(idxs: List[int], metric_x: str, metric_y: str) -> Table:
-    table = Table(title=f"Pareto-Front for {metric_y}/{metric_x}:", show_lines=True)
-
-    rows = _pareto_front_table_read_csv()
-    if not rows:
-        table.add_column("No data found")
-        return table
-
-    filtered_rows = _pareto_front_table_filter_rows(rows, idxs)
-    if not filtered_rows:
-        table.add_column("No matching entries")
-        return table
-
-    param_cols, result_cols = _pareto_front_table_get_columns(filtered_rows[0])
-
-    _pareto_front_table_add_headers(table, param_cols, result_cols)
-    _pareto_front_table_add_rows(table, filtered_rows, param_cols, result_cols)
-
-    return table
-
-def _pareto_front_table_read_csv() -> List[Dict[str, str]]:
-    with open(RESULT_CSV_FILE, mode="r", encoding="utf-8", newline="") as f:
-        return list(csv.DictReader(f))
-
-def _pareto_front_table_filter_rows(rows: List[Dict[str, str]], idxs: List[int]) -> List[Dict[str, str]]:
-    result = []
-    for row in rows:
-        try:
-            trial_index = int(row["trial_index"])
-        except (KeyError, ValueError):
-            continue
-
-        if row.get("trial_status", "").strip().upper() == "COMPLETED" and trial_index in idxs:
-            result.append(row)
-    return result
-
-def _pareto_front_table_get_columns(first_row: Dict[str, str]) -> Tuple[List[str], List[str]]:
-    all_columns = list(first_row.keys())
-    ignored_cols = set(special_col_names) - {"trial_index"}
-
-    param_cols = [col for col in all_columns if col not in ignored_cols and col not in arg_result_names and not col.startswith("OO_Info_")]
-    result_cols = [col for col in arg_result_names if col in all_columns]
-    return param_cols, result_cols
-
-def _pareto_front_table_add_headers(table: Table, param_cols: List[str], result_cols: List[str]) -> None:
-    for col in param_cols:
-        table.add_column(col, justify="center")
-    for col in result_cols:
-        table.add_column(Text(f"{col}", style="cyan"), justify="center")
-
-def _pareto_front_table_add_rows(table: Table, rows: List[Dict[str, str]], param_cols: List[str], result_cols: List[str]) -> None:
-    for row in rows:
-        values = [str(helpers.to_int_when_possible(row[col])) for col in param_cols]
-        result_values = [Text(str(helpers.to_int_when_possible(row[col])), style="cyan") for col in result_cols]
-        table.add_row(*values, *result_values, style="bold green")
-
-def pareto_front_as_rich_table(idxs: list, metric_x: str, metric_y: str) -> Optional[Table]:
-    if not os.path.exists(RESULT_CSV_FILE):
-        print_debug(f"pareto_front_as_rich_table: File '{RESULT_CSV_FILE}' not found")
-        return None
-
-    return create_pareto_front_table(idxs, metric_x, metric_y)
-
-def supports_sixel() -> bool:
-    term = os.environ.get("TERM", "").lower()
-    if "xterm" in term or "mlterm" in term:
-        return True
-
-    try:
-        output = subprocess.run(["tput", "setab", "256"], capture_output=True, text=True, check=True)
-        if output.returncode == 0 and "sixel" in output.stdout.lower():
-            return True
-    except (subprocess.CalledProcessError, FileNotFoundError):
-        pass
-
-    return False
-
-def plot_pareto_frontier_sixel(data: Any, x_metric: str, y_metric: str) -> None:
-    if data is None:
-        print("[italic yellow]The data seems to be empty. Cannot plot pareto frontier.[/]")
-        return
-
-    if not supports_sixel():
-        print(f"[italic yellow]Your console does not support sixel-images. Will not print Pareto-frontier as a matplotlib-sixel-plot for {x_metric}/{y_metric}.[/]")
-        return
-
-    import matplotlib.pyplot as plt
-
-    means = data[x_metric][y_metric]["means"]
-
-    x_values = means[x_metric]
-    y_values = means[y_metric]
-
-    fig, _ax = plt.subplots()
-
-    _ax.scatter(x_values, y_values, s=50, marker='x', c='blue', label='Data Points')
-
-    _ax.set_xlabel(x_metric)
-    _ax.set_ylabel(y_metric)
-
-    _ax.set_title(f'Pareto-Front {x_metric}/{y_metric}')
-
-    _ax.ticklabel_format(style='plain', axis='both', useOffset=False)
-
-    with tempfile.NamedTemporaryFile(suffix=".png", delete=True) as tmp_file:
-        plt.savefig(tmp_file.name, dpi=300)
-
-        print_image_to_cli(tmp_file.name, 1000)
-
-    plt.close(fig)
-
-def _pareto_front_general_validate_shapes(x: np.ndarray, y: np.ndarray) -> None:
-    if x.shape != y.shape:
-        raise ValueError("Input arrays x and y must have the same shape.")
-
-def _pareto_front_general_compare(
-    xi: float, yi: float, xj: float, yj: float,
-    x_minimize: bool, y_minimize: bool
-) -> bool:
-    x_better_eq = xj <= xi if x_minimize else xj >= xi
-    y_better_eq = yj <= yi if y_minimize else yj >= yi
-    x_strictly_better = xj < xi if x_minimize else xj > xi
-    y_strictly_better = yj < yi if y_minimize else yj > yi
-
-    return bool(x_better_eq and y_better_eq and (x_strictly_better or y_strictly_better))
-
-def _pareto_front_general_find_dominated(
-    x: np.ndarray, y: np.ndarray, x_minimize: bool, y_minimize: bool
-) -> np.ndarray:
-    num_points = len(x)
-    is_dominated = np.zeros(num_points, dtype=bool)
-
-    for i in range(num_points):
-        for j in range(num_points):
-            if i == j:
-                continue
-
-            if _pareto_front_general_compare(x[i], y[i], x[j], y[j], x_minimize, y_minimize):
-                is_dominated[i] = True
-                break
-
-    return is_dominated
-
-def pareto_front_general(
-    x: np.ndarray,
-    y: np.ndarray,
-    x_minimize: bool = True,
-    y_minimize: bool = True
-) -> np.ndarray:
-    try:
-        _pareto_front_general_validate_shapes(x, y)
-        is_dominated = _pareto_front_general_find_dominated(x, y, x_minimize, y_minimize)
-        return np.where(~is_dominated)[0]
-    except Exception as e:
-        print("Error in pareto_front_general:", str(e))
-        return np.array([], dtype=int)
-
-def _pareto_front_aggregate_data(path_to_calculate: str) -> Optional[Dict[Tuple[int, str], Dict[str, Dict[str, float]]]]:
-    results_csv_file = f"{path_to_calculate}/{RESULTS_CSV_FILENAME}"
-    result_names_file = f"{path_to_calculate}/result_names.txt"
-
-    if not os.path.exists(results_csv_file) or not os.path.exists(result_names_file):
-        return None
-
-    with open(result_names_file, mode="r", encoding="utf-8") as f:
-        result_names = [line.strip() for line in f if line.strip()]
-
-    records: dict = defaultdict(lambda: {'means': {}})
-
-    with open(results_csv_file, encoding="utf-8", mode="r", newline='') as csvfile:
-        reader = csv.DictReader(csvfile)
-        for row in reader:
-            trial_index = int(row['trial_index'])
-            arm_name = row['arm_name']
-            key = (trial_index, arm_name)
-
-            for metric in result_names:
-                if metric in row:
-                    try:
-                        records[key]['means'][metric] = float(row[metric])
-                    except ValueError:
-                        continue
-
-    return records
-
-def _pareto_front_filter_complete_points(
-    path_to_calculate: str,
-    records: Dict[Tuple[int, str], Dict[str, Dict[str, float]]],
-    primary_name: str,
-    secondary_name: str
-) -> List[Tuple[Tuple[int, str], float, float]]:
-    points = []
-    for key, metrics in records.items():
-        means = metrics['means']
-        if primary_name in means and secondary_name in means:
-            x_val = means[primary_name]
-            y_val = means[secondary_name]
-            points.append((key, x_val, y_val))
-    if len(points) == 0:
-        raise ValueError(f"No full data points with both objectives found in {path_to_calculate}.")
-    return points
-
-def _pareto_front_transform_objectives(
-    points: List[Tuple[Any, float, float]],
-    primary_name: str,
-    secondary_name: str
-) -> Tuple[np.ndarray, np.ndarray]:
-    primary_idx = arg_result_names.index(primary_name)
-    secondary_idx = arg_result_names.index(secondary_name)
-
-    x = np.array([p[1] for p in points])
-    y = np.array([p[2] for p in points])
-
-    if arg_result_min_or_max[primary_idx] == "max":
-        x = -x
-    elif arg_result_min_or_max[primary_idx] != "min":
-        raise ValueError(f"Unknown mode for {primary_name}: {arg_result_min_or_max[primary_idx]}")
-
-    if arg_result_min_or_max[secondary_idx] == "max":
-        y = -y
-    elif arg_result_min_or_max[secondary_idx] != "min":
-        raise ValueError(f"Unknown mode for {secondary_name}: {arg_result_min_or_max[secondary_idx]}")
-
-    return x, y
-
-def _pareto_front_select_pareto_points(
-    x: np.ndarray,
-    y: np.ndarray,
-    x_minimize: bool,
-    y_minimize: bool,
-    points: List[Tuple[Any, float, float]],
-    num_points: int
-) -> List[Tuple[Any, float, float]]:
-    indices = pareto_front_general(x, y, x_minimize, y_minimize)
-    sorted_indices = indices[np.argsort(x[indices])]
-    sorted_indices = sorted_indices[:num_points]
-    selected_points = [points[i] for i in sorted_indices]
-    return selected_points
-
-def _pareto_front_build_return_structure(
-    path_to_calculate: str,
-    selected_points: List[Tuple[Any, float, float]],
-    records: Dict[Tuple[int, str], Dict[str, Dict[str, float]]],
-    absolute_metrics: List[str],
-    primary_name: str,
-    secondary_name: str
-) -> dict:
-    results_csv_file = f"{path_to_calculate}/{RESULTS_CSV_FILENAME}"
-    result_names_file = f"{path_to_calculate}/result_names.txt"
-
-    with open(result_names_file, mode="r", encoding="utf-8") as f:
-        result_names = [line.strip() for line in f if line.strip()]
-
-    csv_rows = {}
-    with open(results_csv_file, mode="r", encoding="utf-8", newline='') as csvfile:
-        reader = csv.DictReader(csvfile)
-        for row in reader:
-            trial_index = int(row['trial_index'])
-            csv_rows[trial_index] = row
-
-    ignored_columns = {'trial_index', 'arm_name', 'trial_status', 'generation_node'}
-    ignored_columns.update(result_names)
-
-    param_dicts = []
-    idxs = []
-    means_dict = defaultdict(list)
+    return None
 
-    for (trial_index, arm_name), _, _ in selected_points:
-        row = csv_rows.get(trial_index, {})
-        if row == {} or row is None or row['arm_name'] != arm_name:
-            print_debug(f"_pareto_front_build_return_structure: trial_index '{trial_index}' could not be found and row returned as None")
-            continue
+def set_orchestrator() -> None:
+    with spinner("Setting orchestrator..."):
+        global orchestrator
 
-        idxs.append(int(row["trial_index"]))
+        if args.orchestrator_file:
+            if SYSTEM_HAS_SBATCH:
+                orchestrator = parse_orchestrator_file(args.orchestrator_file, False)
+            else:
+                print_yellow("--orchestrator_file will be ignored on non-sbatch-systems.")
 
-        param_dict: dict[str, int | float | str] = {}
-        for key, value in row.items():
-            if key not in ignored_columns:
-                try:
-                    param_dict[key] = int(value)
-                except ValueError:
-                    try:
-                        param_dict[key] = float(value)
-                    except ValueError:
-                        param_dict[key] = value
+def check_if_has_random_steps() -> None:
+    if (not args.continue_previous_job and "--continue" not in sys.argv) and (args.num_random_steps == 0 or not args.num_random_steps) and args.model not in ["EXTERNAL_GENERATOR", "SOBOL", "PSEUDORANDOM"]:
+        _fatal_error("You have no random steps set. This is only allowed in continued jobs. To start, you need either some random steps, or a continued run.", 233)
 
-        param_dicts.append(param_dict)
+def add_exclude_to_defective_nodes() -> None:
+    with spinner("Adding excluded nodes..."):
+        if args.exclude:
+            entries = [entry.strip() for entry in args.exclude.split(',')]
 
-        for metric in absolute_metrics:
-            means_dict[metric].append(records[(trial_index, arm_name)]['means'].get(metric, float("nan")))
+            for entry in entries:
+                count_defective_nodes(None, entry)
 
-    ret = {
-        primary_name: {
-            secondary_name: {
-                "absolute_metrics": absolute_metrics,
-                "param_dicts": param_dicts,
-                "means": dict(means_dict),
-                "idxs": idxs
-            },
-            "absolute_metrics": absolute_metrics
-        }
-    }
+def check_max_eval(_max_eval: int) -> None:
+    with spinner("Checking max_eval..."):
+        if not _max_eval:
+            _fatal_error("--max_eval needs to be set!", 19)
 
-    return ret
+def parse_parameters() -> Any:
+    cli_params_experiment_parameters = None
+    if args.parameter:
+        parse_experiment_parameters()
+        cli_params_experiment_parameters = experiment_parameters
 
-def get_pareto_frontier_points(
-    path_to_calculate: str,
-    primary_objective: str,
-    secondary_objective: str,
-    x_minimize: bool,
-    y_minimize: bool,
-    absolute_metrics: List[str],
-    num_points: int
-) -> Optional[dict]:
-    records = _pareto_front_aggregate_data(path_to_calculate)
+    return cli_params_experiment_parameters
 
-    if records is None:
-        return None
+def supports_sixel() -> bool:
+    term = os.environ.get("TERM", "").lower()
+    if "xterm" in term or "mlterm" in term:
+        return True
 
-    points = _pareto_front_filter_complete_points(path_to_calculate, records, primary_objective, secondary_objective)
-    x, y = _pareto_front_transform_objectives(points, primary_objective, secondary_objective)
-    selected_points = _pareto_front_select_pareto_points(x, y, x_minimize, y_minimize, points, num_points)
-    result = _pareto_front_build_return_structure(path_to_calculate, selected_points, records, absolute_metrics, primary_objective, secondary_objective)
+    try:
+        output = subprocess.run(["tput", "setab", "256"], capture_output=True, text=True, check=True)
+        if output.returncode == 0 and "sixel" in output.stdout.lower():
+            return True
+    except (subprocess.CalledProcessError, FileNotFoundError):
+        pass
 
-    return result
+    return False
 
 def save_experiment_state() -> None:
     try:
@@ -9617,14 +10090,14 @@ def save_experiment_state() -> None:
             print_red("save_experiment_state: ax_client or ax_client.experiment is None, cannot save.")
             return
         state_path = get_current_run_folder("experiment_state.json")
-        ax_client.save_to_json_file(state_path)
+        save_ax_client_to_json_file(state_path)
     except Exception as e:
-        print(f"Error saving experiment state: {e}")
+        print_debug(f"Error saving experiment state: {e}")
 
 def wait_for_state_file(state_path: str, min_size: int = 5, max_wait_seconds: int = 60) -> bool:
     try:
         if not os.path.exists(state_path):
-            print(f"[ERROR] File '{state_path}' does not exist.")
+            print_debug(f"[ERROR] File '{state_path}' does not exist.")
             return False
 
         i = 0
@@ -9843,38 +10316,47 @@ def get_result_minimize_flag(path_to_calculate: str, resname: str) -> bool:
 
     return minmax[index] == "min"
 
-def get_pareto_front_data(path_to_calculate: str, res_names: list) -> dict:
-    pareto_front_data: dict = {}
+def post_job_calculate_pareto_front() -> None:
+    if not args.calculate_pareto_front_of_job:
+        return
 
-    all_combinations = list(combinations(range(len(arg_result_names)), 2))
+    failure = False
 
-    skip = False
+    _paths_to_calculate = []
 
-    for i, j in all_combinations:
-        if not skip:
-            metric_x = arg_result_names[i]
-            metric_y = arg_result_names[j]
+    for _path_to_calculate in list(set(args.calculate_pareto_front_of_job)):
+        try:
+            found_paths = find_results_paths(_path_to_calculate)
 
-            x_minimize = get_result_minimize_flag(path_to_calculate, metric_x)
-            y_minimize = get_result_minimize_flag(path_to_calculate, metric_y)
+            for _fp in found_paths:
+                if _fp not in _paths_to_calculate:
+                    _paths_to_calculate.append(_fp)
+        except (FileNotFoundError, NotADirectoryError) as e:
+            print_red(f"post_job_calculate_pareto_front: find_results_paths('{_path_to_calculate}') failed with {e}")
 
-            try:
-                if metric_x not in pareto_front_data:
-                    pareto_front_data[metric_x] = {}
+            failure = True
 
-                pareto_front_data[metric_x][metric_y] = get_calculated_frontier(path_to_calculate, metric_x, metric_y, x_minimize, y_minimize, res_names)
-            except ax.exceptions.core.DataRequiredError as e:
-                print_red(f"Error computing Pareto frontier for {metric_x} and {metric_y}: {e}")
-            except SignalINT:
-                print_red("Calculating Pareto-fronts was cancelled by pressing CTRL-c")
-                skip = True
+    for _path_to_calculate in _paths_to_calculate:
+        for path_to_calculate in found_paths:
+            if not job_calculate_pareto_front(path_to_calculate):
+                failure = True
 
-    return pareto_front_data
+    if failure:
+        my_exit(24)
+
+    my_exit(0)
+
+def pareto_front_as_rich_table(idxs: list, metric_x: str, metric_y: str) -> Optional[Table]:
+    if not os.path.exists(RESULT_CSV_FILE):
+        print_debug(f"pareto_front_as_rich_table: File '{RESULT_CSV_FILE}' not found")
+        return None
+
+    return create_pareto_front_table(idxs, metric_x, metric_y)
 
 def show_pareto_frontier_data(path_to_calculate: str, res_names: list, disable_sixel_and_table: bool = False) -> None:
     if len(res_names) <= 1:
         print_debug(f"--result_names (has {len(res_names)} entries) must be at least 2.")
-        return
+        return None
 
     pareto_front_data: dict = get_pareto_front_data(path_to_calculate, res_names)
 
@@ -9895,8 +10377,16 @@ def show_pareto_frontier_data(path_to_calculate: str, res_names: list, disable_s
                 else:
                     print(f"Not showing Pareto-front-sixel for {path_to_calculate}")
 
-            if len(calculated_frontier[metric_x][metric_y]["idxs"]):
-                pareto_points[metric_x][metric_y] = sorted(calculated_frontier[metric_x][metric_y]["idxs"])
+            if calculated_frontier is None:
+                print_debug("ERROR: calculated_frontier is None")
+                return None
+
+            try:
+                if len(calculated_frontier[metric_x][metric_y]["idxs"]):
+                    pareto_points[metric_x][metric_y] = sorted(calculated_frontier[metric_x][metric_y]["idxs"])
+            except AttributeError:
+                print_debug(f"ERROR: calculated_frontier structure invalid for ({metric_x}, {metric_y})")
+                return None
 
             rich_table = pareto_front_as_rich_table(
                 calculated_frontier[metric_x][metric_y]["idxs"],
@@ -9921,6 +10411,8 @@ def show_pareto_frontier_data(path_to_calculate: str, res_names: list, disable_s
 
     live_share_after_pareto()
 
+    return None
+
 def show_available_hardware_and_generation_strategy_string(gpu_string: str, gpu_color: str) -> None:
     cpu_count = os.cpu_count()
 
@@ -9932,9 +10424,11 @@ def show_available_hardware_and_generation_strategy_string(gpu_string: str, gpu_
         pass
 
     if gpu_string:
-        console.print(f"[green]You have {cpu_count} CPUs available for the main process.[/green] [{gpu_color}]{gpu_string}[/{gpu_color}] [green]{gs_string}[/green]")
+        console.print(f"[green]You have {cpu_count} CPUs available for the main process.[/green] [{gpu_color}]{gpu_string}[/{gpu_color}]")
     else:
-        print_green(f"You have {cpu_count} CPUs available for the main process. {gs_string}")
+        print_green(f"You have {cpu_count} CPUs available for the main process.")
+
+    print_green(gs_string)
 
 def write_args_overview_table() -> None:
     table = Table(title="Arguments Overview")
@@ -10201,112 +10695,6 @@ def find_results_paths(base_path: str) -> list:
 
     return list(set(found_paths))
 
-def post_job_calculate_pareto_front() -> None:
-    if not args.calculate_pareto_front_of_job:
-        return
-
-    failure = False
-
-    _paths_to_calculate = []
-
-    for _path_to_calculate in list(set(args.calculate_pareto_front_of_job)):
-        try:
-            found_paths = find_results_paths(_path_to_calculate)
-
-            for _fp in found_paths:
-                if _fp not in _paths_to_calculate:
-                    _paths_to_calculate.append(_fp)
-        except (FileNotFoundError, NotADirectoryError) as e:
-            print_red(f"post_job_calculate_pareto_front: find_results_paths('{_path_to_calculate}') failed with {e}")
-
-            failure = True
-
-    for _path_to_calculate in _paths_to_calculate:
-        for path_to_calculate in found_paths:
-            if not job_calculate_pareto_front(path_to_calculate):
-                failure = True
-
-    if failure:
-        my_exit(24)
-
-    my_exit(0)
-
-def job_calculate_pareto_front(path_to_calculate: str, disable_sixel_and_table: bool = False) -> bool:
-    pf_start_time = time.time()
-
-    if not path_to_calculate:
-        return False
-
-    global CURRENT_RUN_FOLDER
-    global RESULT_CSV_FILE
-    global arg_result_names
-
-    if not path_to_calculate:
-        print_red("Can only calculate pareto front of previous job when --calculate_pareto_front_of_job is set")
-        return False
-
-    if not os.path.exists(path_to_calculate):
-        print_red(f"Path '{path_to_calculate}' does not exist")
-        return False
-
-    ax_client_json = f"{path_to_calculate}/state_files/ax_client.experiment.json"
-
-    if not os.path.exists(ax_client_json):
-        print_red(f"Path '{ax_client_json}' not found")
-        return False
-
-    checkpoint_file: str = f"{path_to_calculate}/state_files/checkpoint.json"
-    if not os.path.exists(checkpoint_file):
-        print_red(f"The checkpoint file '{checkpoint_file}' does not exist")
-        return False
-
-    RESULT_CSV_FILE = f"{path_to_calculate}/{RESULTS_CSV_FILENAME}"
-    if not os.path.exists(RESULT_CSV_FILE):
-        print_red(f"{RESULT_CSV_FILE} not found")
-        return False
-
-    res_names = []
-
-    res_names_file = f"{path_to_calculate}/result_names.txt"
-    if not os.path.exists(res_names_file):
-        print_red(f"File '{res_names_file}' does not exist")
-        return False
-
-    try:
-        with open(res_names_file, "r", encoding="utf-8") as file:
-            lines = file.readlines()
-    except Exception as e:
-        print_red(f"Error reading file '{res_names_file}': {e}")
-        return False
-
-    for line in lines:
-        entry = line.strip()
-        if entry != "":
-            res_names.append(entry)
-
-    if len(res_names) < 2:
-        print_red(f"Error: There are less than 2 result names (is: {len(res_names)}, {', '.join(res_names)}) in {path_to_calculate}. Cannot continue calculating the pareto front.")
-        return False
-
-    load_username_to_args(path_to_calculate)
-
-    CURRENT_RUN_FOLDER = path_to_calculate
-
-    arg_result_names = res_names
-
-    load_experiment_parameters_from_checkpoint_file(checkpoint_file, False)
-
-    if experiment_parameters is None:
-        return False
-
-    show_pareto_or_error_msg(path_to_calculate, res_names, disable_sixel_and_table)
-
-    pf_end_time = time.time()
-
-    print_debug(f"Calculating the Pareto-front took {pf_end_time - pf_start_time} seconds")
-
-    return True
-
 def set_arg_states_from_continue() -> None:
     if args.continue_previous_job and not args.num_random_steps:
         num_random_steps_file = f"{args.continue_previous_job}/state_files/num_random_steps"
@@ -10349,27 +10737,27 @@ def run_program_once(params: Optional[dict] = None) -> None:
         params = {}
 
     if isinstance(args.run_program_once, str):
-        command_str = args.run_program_once
+        command_str = decode_if_base64(args.run_program_once)
         for k, v in params.items():
             placeholder = f"%({k})"
             command_str = command_str.replace(placeholder, str(v))
 
-        with spinner(f"Executing command: [cyan]{command_str}[/cyan]"):
-            result = subprocess.run(command_str, shell=True, check=True)
-            if result.returncode == 0:
-                console.log("[bold green]Setup script completed successfully ✅[/bold green]")
-            else:
-                console.log(f"[bold red]Setup script failed with exit code {result.returncode} ❌[/bold red]")
+        print(f"Executing command: [cyan]{command_str}[/cyan]")
+        result = subprocess.run(command_str, shell=True, check=True)
+        if result.returncode == 0:
+            print("[bold green]Setup script completed successfully ✅[/bold green]")
+        else:
+            print(f"[bold red]Setup script failed with exit code {result.returncode} ❌[/bold red]")
 
-                my_exit(57)
+            my_exit(57)
 
     elif isinstance(args.run_program_once, (list, tuple)):
         with spinner("run_program_once: Executing command list: [cyan]{args.run_program_once}[/cyan]"):
             result = subprocess.run(args.run_program_once, check=True)
             if result.returncode == 0:
-                console.log("[bold green]Setup script completed successfully ✅[/bold green]")
+                print("[bold green]Setup script completed successfully ✅[/bold green]")
             else:
-                console.log(f"[bold red]Setup script failed with exit code {result.returncode} ❌[/bold red]")
+                print(f"[bold red]Setup script failed with exit code {result.returncode} ❌[/bold red]")
 
                 my_exit(57)
 
@@ -10387,8 +10775,14 @@ def show_omniopt_call() -> None:
 
     original_print(oo_call + " " + cleaned)
 
+    if args.dependency is not None and args.dependency != "":
+        print(f"Dependency: {args.dependency}")
+
+    if args.ui_url is not None and args.ui_url != "":
+        print_yellow("--ui_url is deprecated. Do not use it anymore. It will be ignored and one day be removed.")
+
 def main() -> None:
-    global RESULT_CSV_FILE, ax_client, LOGFILE_DEBUG_GET_NEXT_TRIALS
+    global RESULT_CSV_FILE, LOGFILE_DEBUG_GET_NEXT_TRIALS
 
     check_if_has_random_steps()
 
@@ -10470,7 +10864,7 @@ def main() -> None:
     exp_params = get_experiment_parameters(cli_params_experiment_parameters)
 
     if exp_params is not None:
-        ax_client, experiment_args, gpu_string, gpu_color = exp_params
+        experiment_args, gpu_string, gpu_color = exp_params
         print_debug(f"experiment_parameters: {experiment_parameters}")
 
         set_orchestrator()
@@ -10491,6 +10885,8 @@ def main() -> None:
 
         write_files_and_show_overviews()
 
+        live_share()
+
         #if args.continue_previous_job:
         #    insert_jobs_from_csv(f"{args.continue_previous_job}/{RESULTS_CSV_FILENAME}")
 
@@ -10710,8 +11106,6 @@ def run_search_with_progress_bar() -> None:
     wait_for_jobs_to_complete()
 
 def complex_tests(_program_name: str, wanted_stderr: str, wanted_exit_code: int, wanted_signal: Union[int, None], res_is_none: bool = False) -> int:
-    #print_yellow(f"Test suite: {_program_name}")
-
     nr_errors: int = 0
 
     program_path: str = f"./.tests/test_wronggoing_stuff.bin/bin/{_program_name}"
@@ -10785,7 +11179,7 @@ def test_find_paths(program_code: str) -> int:
     for i in files:
         if i not in string:
             if os.path.exists(i):
-                print("Missing {i} in find_file_paths string!")
+                print(f"Missing {i} in find_file_paths string!")
                 nr_errors += 1
 
     return nr_errors
@@ -11166,17 +11560,16 @@ Exit-Code: 159
 
     my_exit(nr_errors)
 
-def main_outside() -> None:
+def main_wrapper() -> None:
     print(f"Run-UUID: {run_uuid}")
 
     auto_wrap_namespace(globals())
 
     print_logo()
 
-    start_logging_daemon() # type: ignore[unused-coroutine]
-
     fool_linter(args.num_cpus_main_job)
     fool_linter(args.flame_graph)
+    fool_linter(args.memray)
 
     with warnings.catch_warnings():
         warnings.simplefilter("ignore")
@@ -11233,6 +11626,9 @@ def stack_trace_wrapper(func: Any, regex: Any = None) -> Any:
 def auto_wrap_namespace(namespace: Any) -> Any:
     enable_beartype = any(os.getenv(v) for v in ("ENABLE_BEARTYPE", "CI"))
 
+    if args.beartype:
+        enable_beartype = True
+
     excluded_functions = {
         "log_time_and_memory_wrapper",
         "collect_runtime_stats",
@@ -11241,7 +11637,6 @@ def auto_wrap_namespace(namespace: Any) -> Any:
         "_record_stats",
         "_open",
         "_check_memory_leak",
-        "start_logging_daemon",
         "get_current_run_folder",
         "show_func_name_wrapper"
     }
@@ -11267,7 +11662,7 @@ def auto_wrap_namespace(namespace: Any) -> Any:
 
 if __name__ == "__main__":
     try:
-        main_outside()
+        main_wrapper()
     except (SignalUSR, SignalINT, SignalCONT) as e:
-        print_red(f"main_outside failed with exception {e}")
+        print_red(f"main_wrapper failed with exception {e}")
         end_program(True)