omniopt2 8366__py3-none-any.whl → 9061__py3-none-any.whl

{omniopt2-8366.data → omniopt2-9061.data}/data/bin/.omniopt.py

@@ -27,11 +27,15 @@ import inspect
 import tracemalloc
 import resource
 from urllib.parse import urlencode
-
 import psutil
 
 FORCE_EXIT: bool = False
 
+LAST_LOG_TIME: int = 0
+last_msg_progressbar = ""
+last_msg_raw = None
+last_lock_print_debug = threading.Lock()
+
 def force_exit(signal_number: Any, frame: Any) -> Any:
     global FORCE_EXIT
 
@@ -61,10 +65,10 @@ _last_count_time = 0
 _last_count_result: tuple[int, str] = (0, "")
 
 _total_time = 0.0
-_func_times = defaultdict(float)
-_func_mem = defaultdict(float)
-_func_call_paths = defaultdict(Counter)
-_last_mem = defaultdict(float)
+_func_times: defaultdict = defaultdict(float)
+_func_mem: defaultdict = defaultdict(float)
+_func_call_paths: defaultdict = defaultdict(Counter)
+_last_mem: defaultdict = defaultdict(float)
 _leak_threshold_mb = 10.0
 generation_strategy_names: list = []
 default_max_range_difference: int = 1000000
@@ -84,6 +88,7 @@ log_nr_gen_jobs: list[int] = []
 generation_strategy_human_readable: str = ""
 oo_call: str = "./omniopt"
 progress_bar_length: int = 0
+worker_usage_file = 'worker_usage.csv'
 
 if os.environ.get("CUSTOM_VIRTUAL_ENV") == "1":
     oo_call = "omniopt"
@@ -99,7 +104,7 @@ joined_valid_occ_types: str = ", ".join(valid_occ_types)
 SUPPORTED_MODELS: list = ["SOBOL", "FACTORIAL", "SAASBO", "BOTORCH_MODULAR", "UNIFORM", "BO_MIXED", "RANDOMFOREST", "EXTERNAL_GENERATOR", "PSEUDORANDOM", "TPE"]
 joined_supported_models: str = ", ".join(SUPPORTED_MODELS)
 
-special_col_names: list = ["arm_name", "generation_method", "trial_index", "trial_status", "generation_node", "idxs", "start_time", "end_time", "run_time", "exit_code", "program_string", "signal", "hostname", "submit_time", "queue_time", "metric_name", "mean", "sem", "worker_generator_uuid"]
+special_col_names: list = ["arm_name", "generation_method", "trial_index", "trial_status", "generation_node", "idxs", "start_time", "end_time", "run_time", "exit_code", "program_string", "signal", "hostname", "submit_time", "queue_time", "metric_name", "mean", "sem", "worker_generator_uuid", "runtime", "status"]
 
 IGNORABLE_COLUMNS: list = ["start_time", "end_time", "hostname", "signal", "exit_code", "run_time", "program_string"] + special_col_names
 
@@ -153,12 +158,6 @@ try:
             message="Ax currently requires a sqlalchemy version below 2.0.*",
         )
 
-        warnings.filterwarnings(
-            "ignore",
-            category=RuntimeWarning,
-            message="coroutine 'start_logging_daemon' was never awaited"
-        )
-
     with spinner("Importing argparse..."):
         import argparse
 
@@ -204,6 +203,9 @@ try:
     with spinner("Importing rich.pretty..."):
         from rich.pretty import pprint
 
+    with spinner("Importing pformat..."):
+        from pprint import pformat
+
     with spinner("Importing rich.prompt..."):
         from rich.prompt import Prompt, FloatPrompt, IntPrompt
 
@@ -237,9 +239,6 @@ try:
     with spinner("Importing uuid..."):
         import uuid
 
-    #with spinner("Importing qrcode..."):
-    #    import qrcode
-
     with spinner("Importing cowsay..."):
         import cowsay
 
@@ -333,7 +332,7 @@ def show_func_name_wrapper(func: F) -> F:
 
         return result
 
-    return wrapper  # type: ignore
+    return wrapper # type: ignore
 
 def log_time_and_memory_wrapper(func: F) -> F:
     @functools.wraps(func)
@@ -360,7 +359,7 @@ def log_time_and_memory_wrapper(func: F) -> F:
 
         return result
 
-    return wrapper  # type: ignore
+    return wrapper # type: ignore
 
 def _record_stats(func_name: str, elapsed: float, mem_diff: float, mem_after: float, mem_peak: float) -> None:
     global _total_time
@@ -381,16 +380,9 @@ def _record_stats(func_name: str, elapsed: float, mem_diff: float, mem_after: fl
         call_path_str = " -> ".join(short_stack)
         _func_call_paths[func_name][call_path_str] += 1
 
-        print(
-            f"Function '{func_name}' took {elapsed:.4f}s "
-            f"(total {percent_if_added:.1f}% of tracked time)"
-        )
-        print(
-            f"Memory before: {mem_after - mem_diff:.2f} MB, after: {mem_after:.2f} MB, "
-            f"diff: {mem_diff:+.2f} MB, peak during call: {mem_peak:.2f} MB"
-        )
+        print(f"Function '{func_name}' took {elapsed:.4f}s (total {percent_if_added:.1f}% of tracked time)")
+        print(f"Memory before: {mem_after - mem_diff:.2f} MB, after: {mem_after:.2f} MB, diff: {mem_diff:+.2f} MB, peak during call: {mem_peak:.2f} MB")
 
-        # NEU: Runtime Stats
         runtime_stats = collect_runtime_stats()
         print("=== Runtime Stats ===")
         print(f"RSS: {runtime_stats['rss_MB']:.2f} MB, VMS: {runtime_stats['vms_MB']:.2f} MB")
@@ -449,7 +441,7 @@ RESET: str = "\033[0m"
 
 uuid_regex: Pattern = re.compile(r"^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-4[a-fA-F0-9]{3}-[89aAbB][a-fA-F0-9]{3}-[a-fA-F0-9]{12}$")
 
-worker_generator_uuid: str = uuid.uuid4()
+worker_generator_uuid: str = str(uuid.uuid4())
 
 new_uuid: str = str(uuid.uuid4())
 run_uuid: str = os.getenv("RUN_UUID", new_uuid)
@@ -478,7 +470,7 @@ def get_current_run_folder(name: Optional[str] = None) -> str:
 
     return CURRENT_RUN_FOLDER
 
-def get_state_file_name(name) -> str:
+def get_state_file_name(name: str) -> str:
     state_files_folder = f"{get_current_run_folder()}/state_files/"
     makedirs(state_files_folder)
 
@@ -502,6 +494,24 @@ try:
         dier: FunctionType = helpers.dier
         is_equal: FunctionType = helpers.is_equal
         is_not_equal: FunctionType = helpers.is_not_equal
+    with spinner("Importing pareto..."):
+        pareto_file: str = f"{script_dir}/.pareto.py"
+        spec = importlib.util.spec_from_file_location(
+            name="pareto",
+            location=pareto_file,
+        )
+        if spec is not None and spec.loader is not None:
+            pareto = importlib.util.module_from_spec(spec)
+            spec.loader.exec_module(pareto)
+        else:
+            raise ImportError(f"Could not load module from {pareto_file}")
+
+        pareto_front_table_filter_rows: FunctionType = pareto.pareto_front_table_filter_rows
+        pareto_front_table_add_headers: FunctionType = pareto.pareto_front_table_add_headers
+        pareto_front_table_add_rows: FunctionType = pareto.pareto_front_table_add_rows
+        pareto_front_filter_complete_points: FunctionType = pareto.pareto_front_filter_complete_points
+        pareto_front_select_pareto_points: FunctionType = pareto.pareto_front_select_pareto_points
+
 except KeyboardInterrupt:
     print("You pressed CTRL-c while importing the helpers file")
     sys.exit(0)
@@ -525,15 +535,13 @@ def is_slurm_job() -> bool:
         return True
     return False
 
-def _sleep(t: int) -> int:
+def _sleep(t: Union[float, int]) -> None:
     if args is not None and not args.no_sleep:
         try:
             time.sleep(t)
         except KeyboardInterrupt:
             pass
 
-    return t
-
 LOG_DIR: str = "logs"
 makedirs(LOG_DIR)
 
@@ -546,6 +554,17 @@ logfile_worker_creation_logs: str = f'{log_uuid_dir}_worker_creation_logs'
 logfile_trial_index_to_param_logs: str = f'{log_uuid_dir}_trial_index_to_param_logs'
 LOGFILE_DEBUG_GET_NEXT_TRIALS: Union[str, None] = None
 
+def error_without_print(text: str) -> None:
+    print_debug(text)
+
+    if get_current_run_folder():
+        try:
+            with open(get_current_run_folder("oo_errors.txt"), mode="a", encoding="utf-8") as myfile:
+                myfile.write(text + "\n\n")
+        except (OSError, FileNotFoundError) as e:
+            helpers.print_color("red", f"Error: {e}. This may mean that the {get_current_run_folder()} was deleted during the run. Could not write '{text} to {get_current_run_folder()}/oo_errors.txt'")
+            sys.exit(99)
+
 def print_red(text: str) -> None:
     helpers.print_color("red", text)
 
@@ -579,20 +598,22 @@ def _debug(msg: str, _lvl: int = 0, eee: Union[None, str, Exception] = None) ->
 def _get_debug_json(time_str: str, msg: str) -> str:
     function_stack = []
     try:
-        frame = inspect.currentframe().f_back  # skip _get_debug_json
-        while frame:
-            func_name = _function_name_cache.get(frame.f_code)
-            if func_name is None:
-                func_name = frame.f_code.co_name
-                _function_name_cache[frame.f_code] = func_name
-
-            if func_name not in ("<module>", "print_debug", "wrapper"):
-                function_stack.append({
-                    "function": func_name,
-                    "line_number": frame.f_lineno
-                })
-
-            frame = frame.f_back
+        cf = inspect.currentframe()
+        if cf:
+            frame = cf.f_back  # skip _get_debug_json
+            while frame:
+                func_name = _function_name_cache.get(frame.f_code)
+                if func_name is None:
+                    func_name = frame.f_code.co_name
+                    _function_name_cache[frame.f_code] = func_name
+
+                if func_name not in ("<module>", "print_debug", "wrapper"):
+                    function_stack.append({
+                        "function": func_name,
+                        "line_number": frame.f_lineno
+                    })
+
+                frame = frame.f_back
     except (SignalUSR, SignalINT, SignalCONT):
         print_red("\n⚠ You pressed CTRL-C. This is ignored in _get_debug_json.")
 
@@ -694,11 +715,14 @@ def my_exit(_code: int = 0) -> None:
     if is_skip_search() and os.getenv("SKIP_SEARCH_EXIT_CODE"):
         skip_search_exit_code = os.getenv("SKIP_SEARCH_EXIT_CODE")
 
+        skip_search_exit_code_found = None
+
         try:
-            sys.exit(int(skip_search_exit_code))
+            if skip_search_exit_code_found is not None:
+                skip_search_exit_code_found = int(skip_search_exit_code)
+                sys.exit(skip_search_exit_code_found)
         except ValueError:
-            print(f"Trying to look for SKIP_SEARCH_EXIT_CODE failed. Exiting with original exit code {_code}")
-            sys.exit(_code)
+            print_debug(f"Trying to look for SKIP_SEARCH_EXIT_CODE failed. Exiting with original exit code {_code}")
 
     sys.exit(_code)
 
@@ -799,6 +823,7 @@ class ConfigLoader:
     parameter: Optional[List[str]]
     experiment_constraints: Optional[List[str]]
     main_process_gb: int
+    beartype: bool
     worker_timeout: int
     slurm_signal_delay_s: int
     gridsearch: bool
@@ -819,6 +844,7 @@ class ConfigLoader:
     run_program_once: str
     mem_gb: int
     flame_graph: bool
+    memray: bool
     continue_previous_job: Optional[str]
     calculate_pareto_front_of_job: Optional[List[str]]
     revert_to_random_when_seemingly_exhausted: bool
@@ -980,6 +1006,7 @@ class ConfigLoader:
         debug.add_argument('--verbose_break_run_search_table', help='Verbose logging for break_run_search', action='store_true', default=False)
         debug.add_argument('--debug', help='Enable debugging', action='store_true', default=False)
         debug.add_argument('--flame_graph', help='Enable flame-graphing. Makes everything slower, but creates a flame graph', action='store_true', default=False)
+        debug.add_argument('--memray', help='Use memray to show memory usage', action='store_true', default=False)
         debug.add_argument('--no_sleep', help='Disables sleeping for fast job generation (not to be used on HPC)', action='store_true', default=False)
         debug.add_argument('--tests', help='Run simple internal tests', action='store_true', default=False)
         debug.add_argument('--show_worker_percentage_table_at_end', help='Show a table of percentage of usage of max worker over time', action='store_true', default=False)
@@ -993,8 +1020,8 @@ class ConfigLoader:
         debug.add_argument('--runtime_debug', help='Logs which functions use most of the time', action='store_true', default=False)
         debug.add_argument('--debug_stack_regex', help='Only print debug messages if call stack matches any regex', type=str, default='')
         debug.add_argument('--debug_stack_trace_regex', help='Show compact call stack with arrows if any function in stack matches regex', type=str, default=None)
-
         debug.add_argument('--show_func_name', help='Show func name before each execution and when it is done', action='store_true', default=False)
+        debug.add_argument('--beartype', help='Use beartype', action='store_true', default=False)
 
     def load_config(self: Any, config_path: str, file_format: str) -> dict:
         if not os.path.isfile(config_path):
@@ -1232,11 +1259,15 @@ for _rn in args.result_names:
         _key = _rn
         _min_or_max = __default_min_max
 
+    _min_or_max = re.sub(r"'", "", _min_or_max)
+
     if _min_or_max not in ["min", "max"]:
         if _min_or_max:
             print_yellow(f"Value for determining whether to minimize or maximize was neither 'min' nor 'max' for key '{_key}', but '{_min_or_max}'. It will be set to the default, which is '{__default_min_max}' instead.")
         _min_or_max = __default_min_max
 
+    _key = re.sub(r"'", "", _key)
+
     if _key in arg_result_names:
         console.print(f"[red]The --result_names option '{_key}' was specified multiple times![/]")
         sys.exit(50)
@@ -1343,19 +1374,6 @@ try:
     with spinner("Importing GeneratorSpec..."):
         from ax.generation_strategy.generator_spec import GeneratorSpec
 
-    #except Exception:
-    #    with spinner("Fallback: Importing ax.generation_strategy.generation_node..."):
-    #        import ax.generation_strategy.generation_node
-
-    #    with spinner("Fallback: Importing GenerationStep, GenerationStrategy from ax.generation_strategy..."):
-    #        from ax.generation_strategy.generation_node import GenerationNode, GenerationStep
-
-    #    with spinner("Fallback: Importing ExternalGenerationNode..."):
-    #        from ax.generation_strategy.external_generation_node import ExternalGenerationNode
-
-    #    with spinner("Fallback: Importing MaxTrials..."):
-    #        from ax.generation_strategy.transition_criterion import MaxTrials
-
     with spinner("Importing Models from ax.generation_strategy.registry..."):
         from ax.adapter.registry import Models
 
@@ -1463,6 +1481,9 @@ class RandomForestGenerationNode(ExternalGenerationNode):
     def update_generator_state(self: Any, experiment: Experiment, data: Data) -> None:
         search_space = experiment.search_space
         parameter_names = list(search_space.parameters.keys())
+        if experiment.optimization_config is None:
+            print_red("Error: update_generator_state is None")
+            return
         metric_names = list(experiment.optimization_config.metrics.keys())
 
         completed_trials = [
@@ -1474,7 +1495,7 @@ class RandomForestGenerationNode(ExternalGenerationNode):
         y = np.zeros([num_completed_trials, 1])
 
         for t_idx, trial in enumerate(completed_trials):
-            trial_parameters = trial.arm.parameters
+            trial_parameters = trial.arms[t_idx].parameters
             x[t_idx, :] = np.array([trial_parameters[p] for p in parameter_names])
             trial_df = data.df[data.df["trial_index"] == trial.index]
             y[t_idx, 0] = trial_df[trial_df["metric_name"] == metric_names[0]]["mean"].item()
@@ -1614,10 +1635,18 @@ class RandomForestGenerationNode(ExternalGenerationNode):
     def _format_best_sample(self: Any, best_sample: TParameterization, reverse_choice_map: dict) -> None:
         for name in best_sample.keys():
             param = self.parameters.get(name)
+            best_sample_by_name = best_sample[name]
+
             if isinstance(param, RangeParameter) and param.parameter_type == ParameterType.INT:
-                best_sample[name] = int(round(best_sample[name]))
+                if best_sample_by_name is not None:
+                    best_sample[name] = int(round(float(best_sample_by_name)))
+                else:
+                    print_debug("best_sample_by_name was empty")
             elif isinstance(param, ChoiceParameter):
-                best_sample[name] = str(reverse_choice_map.get(int(best_sample[name])))
+                if best_sample_by_name is not None:
+                    best_sample[name] = str(reverse_choice_map.get(int(best_sample_by_name)))
+                else:
+                    print_debug("best_sample_by_name was empty")
 
 decoder_registry["RandomForestGenerationNode"] = RandomForestGenerationNode
 
@@ -2048,9 +2077,9 @@ def run_live_share_command(force: bool = False) -> Tuple[str, str]:
             return str(result.stdout), str(result.stderr)
         except subprocess.CalledProcessError as e:
             if e.stderr:
-                original_print(f"run_live_share_command: command failed with error: {e}, stderr: {e.stderr}")
+                print_debug(f"run_live_share_command: command failed with error: {e}, stderr: {e.stderr}")
             else:
-                original_print(f"run_live_share_command: command failed with error: {e}")
+                print_debug(f"run_live_share_command: command failed with error: {e}")
             return "", str(e.stderr)
         except Exception as e:
             print(f"run_live_share_command: An error occurred: {e}")
@@ -2064,6 +2093,8 @@ def force_live_share() -> bool:
     return False
 
 def live_share(force: bool = False, text_and_qr: bool = False) -> bool:
+    log_data()
+
     if not get_current_run_folder():
         print(f"live_share: get_current_run_folder was empty or false: {get_current_run_folder()}")
         return False
@@ -2077,23 +2108,16 @@ def live_share(force: bool = False, text_and_qr: bool = False) -> bool:
         if stderr:
             print_green(stderr)
         else:
-            print_red("This call should have shown the CURL, but didnt. Stderr: {stderr}, stdout: {stdout}")
+            print_red(f"This call should have shown the CURL, but didnt. Stderr: {stderr}, stdout: {stdout}")
     if stdout:
         print_debug(f"live_share stdout: {stdout}")
 
     return True
 
 def init_live_share() -> bool:
-    with spinner("Initializing live share..."):
-        ret = live_share(True, True)
-
-        return ret
+    ret = live_share(True, True)
 
-async def start_periodic_live_share() -> None:
-    if args.live_share and not os.environ.get("CI"):
-        while True:
-            live_share(force=False)
-            time.sleep(30)
+    return ret
 
 def init_storage(db_url: str) -> None:
     init_engine_and_session_factory(url=db_url, force_init=True)
@@ -2119,7 +2143,11 @@ def try_saving_to_db() -> None:
         else:
             print_red("ax_client was not defined in try_saving_to_db")
             my_exit(101)
-        save_generation_strategy(global_gs)
+
+        if global_gs is not None:
+            save_generation_strategy(global_gs)
+        else:
+            print_red("Not saving generation strategy: global_gs was empty")
     except Exception as e:
         print_debug(f"Failed trying to save sqlite3-DB: {e}")
 
@@ -2150,6 +2178,8 @@ def merge_with_job_infos(df: pd.DataFrame) -> pd.DataFrame:
     return merged
 
 def save_results_csv() -> Optional[str]:
+    log_data()
+
     if args.dryrun:
         return None
 
@@ -2164,6 +2194,9 @@ def save_results_csv() -> Optional[str]:
 
     try:
         df = fetch_and_prepare_trials()
+        if df is None:
+            print_red(f"save_results_csv: fetch_and_prepare_trials returned an empty element: {df}")
+            return None
         write_csv(df, pd_csv)
         write_json_snapshot(pd_json)
         save_experiment_to_file()
@@ -2176,43 +2209,67 @@ def save_results_csv() -> Optional[str]:
     except (SignalUSR, SignalCONT, SignalINT) as e:
         raise type(e)(str(e)) from e
     except Exception as e:
-        print_red(f"While saving all trials as a pandas-dataframe-csv, an error occurred: {e}")
+        print_red(f"\nWhile saving all trials as a pandas-dataframe-csv, an error occurred: {e}")
 
     return pd_csv
 
 def get_results_paths() -> tuple[str, str]:
     return (get_current_run_folder(RESULTS_CSV_FILENAME), get_state_file_name('pd.json'))
 
-def fetch_and_prepare_trials() -> pd.DataFrame:
-    ax_client.experiment.fetch_data()
-    df = ax_client.get_trials_data_frame()
+def ax_client_get_trials_data_frame() -> Optional[pd.DataFrame]:
+    if not ax_client:
+        my_exit(101)
+
+        return None
+
+    return ax_client.get_trials_data_frame()
+
+def fetch_and_prepare_trials() -> Optional[pd.DataFrame]:
+    if not ax_client:
+        return None
 
-    #print("========================")
-    #print("BEFORE merge_with_job_infos:")
-    #print(df["generation_node"])
+    ax_client.experiment.fetch_data()
+    df = ax_client_get_trials_data_frame()
     df = merge_with_job_infos(df)
-    #print("AFTER merge_with_job_infos:")
-    #print(df["generation_node"])
 
     return df
 
-def write_csv(df, path: str) -> None:
+def write_csv(df: pd.DataFrame, path: str) -> None:
     try:
         df = df.sort_values(by=["trial_index"], kind="stable").reset_index(drop=True)
     except KeyError:
         pass
     df.to_csv(path, index=False, float_format="%.30f")
 
-def write_json_snapshot(path: str) -> None:
+def ax_client_to_json_snapshot() -> Optional[dict]:
+    if not ax_client:
+        my_exit(101)
+
+        return None
+
     json_snapshot = ax_client.to_json_snapshot()
-    with open(path, "w", encoding="utf-8") as f:
-        json.dump(json_snapshot, f, indent=4)
+
+    return json_snapshot
+
+def write_json_snapshot(path: str) -> None:
+    if ax_client is not None:
+        json_snapshot = ax_client_to_json_snapshot()
+        if json_snapshot is not None:
+            with open(path, "w", encoding="utf-8") as f:
+                json.dump(json_snapshot, f, indent=4)
+        else:
+            print_debug('json_snapshot from ax_client_to_json_snapshot was None')
+    else:
+        print_red("write_json_snapshot: ax_client was None")
 
 def save_experiment_to_file() -> None:
-    save_experiment(
-        ax_client.experiment,
-        get_state_file_name("ax_client.experiment.json")
-    )
+    if ax_client is not None:
+        save_experiment(
+            ax_client.experiment,
+            get_state_file_name("ax_client.experiment.json")
+        )
+    else:
+        print_red("save_experiment: ax_client is None")
 
 def should_save_to_database() -> bool:
     return args.model not in uncontinuable_models and args.save_to_database
@@ -2401,7 +2458,7 @@ def set_nr_inserted_jobs(new_nr_inserted_jobs: int) -> None:
 
 def write_worker_usage() -> None:
     if len(WORKER_PERCENTAGE_USAGE):
-        csv_filename = get_current_run_folder('worker_usage.csv')
+        csv_filename = get_current_run_folder(worker_usage_file)
 
         csv_columns = ['time', 'num_parallel_jobs', 'nr_current_workers', 'percentage']
 
@@ -2411,35 +2468,39 @@ def write_worker_usage() -> None:
                 csv_writer.writerow(row)
     else:
         if is_slurm_job():
-            print_debug("WORKER_PERCENTAGE_USAGE seems to be empty. Not writing worker_usage.csv")
+            print_debug(f"WORKER_PERCENTAGE_USAGE seems to be empty. Not writing {worker_usage_file}")
 
 def log_system_usage() -> None:
+    global LAST_LOG_TIME
+
+    now = time.time()
+    if now - LAST_LOG_TIME < 30:
+        return
+
+    LAST_LOG_TIME = int(now)
+
     if not get_current_run_folder():
         return
 
     ram_cpu_csv_file_path = os.path.join(get_current_run_folder(), "cpu_ram_usage.csv")
-
     makedirs(os.path.dirname(ram_cpu_csv_file_path))
 
     file_exists = os.path.isfile(ram_cpu_csv_file_path)
 
-    with open(ram_cpu_csv_file_path, mode='a', newline='', encoding="utf-8") as file:
-        writer = csv.writer(file)
-
-        current_time = int(time.time())
-
-        if process is not None:
-            mem_proc = process.memory_info()
-
-            if mem_proc is not None:
-                ram_usage_mb = mem_proc.rss / (1024 * 1024)
-                cpu_usage_percent = psutil.cpu_percent(percpu=False)
+    mem_proc = process.memory_info() if process else None
+    if not mem_proc:
+        return
 
-                if ram_usage_mb > 0 and cpu_usage_percent > 0:
-                    if not file_exists:
-                        writer.writerow(["timestamp", "ram_usage_mb", "cpu_usage_percent"])
+    ram_usage_mb = mem_proc.rss / (1024 * 1024)
+    cpu_usage_percent = psutil.cpu_percent(percpu=False)
+    if ram_usage_mb <= 0 or cpu_usage_percent <= 0:
+        return
 
-                    writer.writerow([current_time, ram_usage_mb, cpu_usage_percent])
+    with open(ram_cpu_csv_file_path, mode='a', newline='', encoding="utf-8") as file:
+        writer = csv.writer(file)
+        if not file_exists:
+            writer.writerow(["timestamp", "ram_usage_mb", "cpu_usage_percent"])
+        writer.writerow([int(now), ram_usage_mb, cpu_usage_percent])
 
 def write_process_info() -> None:
     try:
@@ -2789,10 +2850,20 @@ def print_debug_get_next_trials(got: int, requested: int, _line: int) -> None:
     log_message_to_file(LOGFILE_DEBUG_GET_NEXT_TRIALS, msg, 0, "")
 
 def print_debug_progressbar(msg: str) -> None:
-    time_str = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-    msg = f"{time_str} ({worker_generator_uuid}): {msg}"
+    global last_msg_progressbar, last_msg_raw
+
+    try:
+        with last_lock_print_debug:
+            if msg != last_msg_raw:
+                time_str = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+                full_msg = f"{time_str} ({worker_generator_uuid}): {msg}"
 
-    _debug_progressbar(msg)
+                _debug_progressbar(full_msg)
+
+                last_msg_raw = msg
+                last_msg_progressbar = full_msg
+    except Exception as e:
+        print(f"Error in print_debug_progressbar: {e}", flush=True)
 
 def get_process_info(pid: Any) -> str:
     try:
@@ -3297,135 +3368,457 @@ def parse_experiment_parameters() -> None:
     # Remove duplicates by 'name' key preserving order
     params = list({p['name']: p for p in params}.values())
 
-    experiment_parameters = params
+    experiment_parameters = params # type: ignore[assignment]
 
-def check_factorial_range() -> None:
-    if args.model and args.model == "FACTORIAL":
-        _fatal_error("\n⚠ --model FACTORIAL cannot be used with range parameter", 181)
+def job_calculate_pareto_front(path_to_calculate: str, disable_sixel_and_table: bool = False) -> bool:
+    pf_start_time = time.time()
 
-def check_if_range_types_are_invalid(value_type: str, valid_value_types: list) -> None:
-    if value_type not in valid_value_types:
-        valid_value_types_string = ", ".join(valid_value_types)
-        _fatal_error(f"⚠ {value_type} is not a valid value type. Valid types for range are: {valid_value_types_string}", 181)
+    if not path_to_calculate:
+        return False
 
-def check_range_params_length(this_args: Union[str, list]) -> None:
-    if len(this_args) != 5 and len(this_args) != 4 and len(this_args) != 6:
-        _fatal_error("\n⚠ --parameter for type range must have 4 (or 5, the last one being optional and float by default, or 6, while the last one is true or false) parameters: <NAME> range <START> <END> (<TYPE (int or float)>, <log_scale: bool>)", 181)
+    global CURRENT_RUN_FOLDER
+    global RESULT_CSV_FILE
+    global arg_result_names
 
-def die_if_lower_and_upper_bound_equal_zero(lower_bound: Union[int, float], upper_bound: Union[int, float]) -> None:
-    if upper_bound is None or lower_bound is None:
-        _fatal_error("die_if_lower_and_upper_bound_equal_zero: upper_bound or lower_bound is None. Cannot continue.", 91)
-    if upper_bound == lower_bound:
-        if lower_bound == 0:
-            _fatal_error(f"⚠ Lower bound and upper bound are equal: {lower_bound}, cannot automatically fix this, because they -0 = +0 (usually a quickfix would be to set lower_bound = -upper_bound)", 181)
-        print_red(f"⚠ Lower bound and upper bound are equal: {lower_bound}, setting lower_bound = -upper_bound")
-        if upper_bound is not None:
-            lower_bound = -upper_bound
+    if not path_to_calculate:
+        print_red("Can only calculate pareto front of previous job when --calculate_pareto_front_of_job is set")
+        return False
 
-def format_value(value: Any, float_format: str = '.80f') -> str:
-    try:
-        if isinstance(value, float):
-            s = format(value, float_format)
-            s = s.rstrip('0').rstrip('.') if '.' in s else s
-            return s
-        return str(value)
-    except Exception as e:
-        print_red(f"⚠ Error formatting the number {value}: {e}")
-        return str(value)
+    if not os.path.exists(path_to_calculate):
+        print_red(f"Path '{path_to_calculate}' does not exist")
+        return False
 
-def replace_parameters_in_string(
-    parameters: dict,
-    input_string: str,
-    float_format: str = '.20f',
-    additional_prefixes: list[str] = [],
-    additional_patterns: list[str] = [],
-) -> str:
-    try:
-        prefixes = ['$', '%'] + additional_prefixes
-        patterns = ['{key}', '({key})'] + additional_patterns
+    ax_client_json = f"{path_to_calculate}/state_files/ax_client.experiment.json"
 
-        for key, value in parameters.items():
-            replacement = format_value(value, float_format=float_format)
-            for prefix in prefixes:
-                for pattern in patterns:
-                    token = prefix + pattern.format(key=key)
-                    input_string = input_string.replace(token, replacement)
+    if not os.path.exists(ax_client_json):
+        print_red(f"Path '{ax_client_json}' not found")
+        return False
 
-        input_string = input_string.replace('\r', ' ').replace('\n', ' ')
-        return input_string
+    checkpoint_file: str = f"{path_to_calculate}/state_files/checkpoint.json"
+    if not os.path.exists(checkpoint_file):
+        print_red(f"The checkpoint file '{checkpoint_file}' does not exist")
+        return False
 
-    except Exception as e:
-        print_red(f"\n⚠ Error: {e}")
-        return ""
+    RESULT_CSV_FILE = f"{path_to_calculate}/{RESULTS_CSV_FILENAME}"
+    if not os.path.exists(RESULT_CSV_FILE):
+        print_red(f"{RESULT_CSV_FILE} not found")
+        return False
 
-def get_memory_usage() -> float:
-    user_uid = os.getuid()
+    res_names = []
 
-    memory_usage = float(sum(
-        p.memory_info().rss for p in psutil.process_iter(attrs=['memory_info', 'uids'])
-        if p.info['uids'].real == user_uid
-    ) / (1024 * 1024))
+    res_names_file = f"{path_to_calculate}/result_names.txt"
+    if not os.path.exists(res_names_file):
+        print_red(f"File '{res_names_file}' does not exist")
+        return False
 
-    return memory_usage
+    try:
+        with open(res_names_file, "r", encoding="utf-8") as file:
+            lines = file.readlines()
+    except Exception as e:
+        print_red(f"Error reading file '{res_names_file}': {e}")
+        return False
 
-class MonitorProcess:
-    def __init__(self: Any, pid: int, interval: float = 1.0) -> None:
-        self.pid = pid
-        self.interval = interval
-        self.running = True
-        self.thread = threading.Thread(target=self._monitor)
-        self.thread.daemon = True
+    for line in lines:
+        entry = line.strip()
+        if entry != "":
+            res_names.append(entry)
 
-        fool_linter(f"self.thread.daemon was set to {self.thread.daemon}")
+    if len(res_names) < 2:
+        print_red(f"Error: There are less than 2 result names (is: {len(res_names)}, {', '.join(res_names)}) in {path_to_calculate}. Cannot continue calculating the pareto front.")
+        return False
 
-    def _monitor(self: Any) -> None:
-        try:
-            _internal_process = psutil.Process(self.pid)
-            while self.running and _internal_process.is_running():
-                crf = get_current_run_folder()
+    load_username_to_args(path_to_calculate)
 
-                if crf and crf != "":
-                    log_file_path = os.path.join(crf, "eval_nodes_cpu_ram_logs.txt")
+    CURRENT_RUN_FOLDER = path_to_calculate
 
-                    os.makedirs(os.path.dirname(log_file_path), exist_ok=True)
+    arg_result_names = res_names
 
-                    with open(log_file_path, mode="a", encoding="utf-8") as log_file:
-                        hostname = socket.gethostname()
+    load_experiment_parameters_from_checkpoint_file(checkpoint_file, False)
 
-                        slurm_job_id = os.getenv("SLURM_JOB_ID")
+    if experiment_parameters is None:
+        return False
 
-                        if slurm_job_id:
-                            hostname += f"-SLURM-ID-{slurm_job_id}"
+    show_pareto_or_error_msg(path_to_calculate, res_names, disable_sixel_and_table)
 
-                        total_memory = psutil.virtual_memory().total / (1024 * 1024)
-                        cpu_usage = psutil.cpu_percent(interval=5)
+    pf_end_time = time.time()
 
-                        memory_usage = get_memory_usage()
+    print_debug(f"Calculating the Pareto-front took {pf_end_time - pf_start_time} seconds")
 
-                        unix_timestamp = int(time.time())
+    return True
 
-                        log_file.write(f"\nUnix-Timestamp: {unix_timestamp}, Hostname: {hostname}, CPU: {cpu_usage:.2f}%, RAM: {memory_usage:.2f} MB / {total_memory:.2f} MB\n")
-                time.sleep(self.interval)
-        except psutil.NoSuchProcess:
-            pass
+def show_pareto_or_error_msg(path_to_calculate: str, res_names: list = arg_result_names, disable_sixel_and_table: bool = False) -> None:
+    if args.dryrun:
+        print_debug("Not showing Pareto-frontier data with --dryrun")
+        return None
 
-    def __enter__(self: Any) -> None:
-        self.thread.start()
-        return self
+    if len(res_names) > 1:
+        try:
+            show_pareto_frontier_data(path_to_calculate, res_names, disable_sixel_and_table)
+        except Exception as e:
+            inner_tb = ''.join(traceback.format_exception(type(e), e, e.__traceback__))
+            print_red(f"show_pareto_frontier_data() failed with exception '{e}':\n{inner_tb}")
+    else:
+        print_debug(f"show_pareto_frontier_data will NOT be executed because len(arg_result_names) is {len(arg_result_names)}")
+    return None
 
-    def __exit__(self: Any, exc_type: Any, exc_value: Any, _traceback: Any) -> None:
-        self.running = False
-        self.thread.join()
+def get_pareto_front_data(path_to_calculate: str, res_names: list) -> dict:
+    pareto_front_data: dict = {}
 
-def execute_bash_code_log_time(code: str) -> list:
-    process_item = subprocess.Popen(code, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+    all_combinations = list(combinations(range(len(arg_result_names)), 2))
 
-    with MonitorProcess(process_item.pid):
-        try:
-            stdout, stderr = process_item.communicate()
-            result = subprocess.CompletedProcess(
-                args=code, returncode=process_item.returncode, stdout=stdout, stderr=stderr
-            )
+    skip = False
+
+    for i, j in all_combinations:
+        if not skip:
+            metric_x = arg_result_names[i]
+            metric_y = arg_result_names[j]
+
+            x_minimize = get_result_minimize_flag(path_to_calculate, metric_x)
+            y_minimize = get_result_minimize_flag(path_to_calculate, metric_y)
+
+            try:
+                if metric_x not in pareto_front_data:
+                    pareto_front_data[metric_x] = {}
+
+                pareto_front_data[metric_x][metric_y] = get_calculated_frontier(path_to_calculate, metric_x, metric_y, x_minimize, y_minimize, res_names)
+            except ax.exceptions.core.DataRequiredError as e:
+                print_red(f"Error computing Pareto frontier for {metric_x} and {metric_y}: {e}")
+            except SignalINT:
+                print_red("Calculating Pareto-fronts was cancelled by pressing CTRL-c")
+                skip = True
+
+    return pareto_front_data
+
+def pareto_front_transform_objectives(
+    points: List[Tuple[Any, float, float]],
+    primary_name: str,
+    secondary_name: str
+) -> Tuple[np.ndarray, np.ndarray]:
+    primary_idx = arg_result_names.index(primary_name)
+    secondary_idx = arg_result_names.index(secondary_name)
+
+    x = np.array([p[1] for p in points])
+    y = np.array([p[2] for p in points])
+
+    if arg_result_min_or_max[primary_idx] == "max":
+        x = -x
+    elif arg_result_min_or_max[primary_idx] != "min":
+        raise ValueError(f"Unknown mode for {primary_name}: {arg_result_min_or_max[primary_idx]}")
+
+    if arg_result_min_or_max[secondary_idx] == "max":
+        y = -y
+    elif arg_result_min_or_max[secondary_idx] != "min":
+        raise ValueError(f"Unknown mode for {secondary_name}: {arg_result_min_or_max[secondary_idx]}")
+
+    return x, y
+
+def get_pareto_frontier_points(
+    path_to_calculate: str,
+    primary_objective: str,
+    secondary_objective: str,
+    x_minimize: bool,
+    y_minimize: bool,
+    absolute_metrics: List[str],
+    num_points: int
+) -> Optional[dict]:
+    records = pareto_front_aggregate_data(path_to_calculate)
+
+    if records is None:
+        return None
+
+    points = pareto_front_filter_complete_points(path_to_calculate, records, primary_objective, secondary_objective)
+    x, y = pareto_front_transform_objectives(points, primary_objective, secondary_objective)
+    selected_points = pareto_front_select_pareto_points(x, y, x_minimize, y_minimize, points, num_points)
+    result = pareto_front_build_return_structure(path_to_calculate, selected_points, records, absolute_metrics, primary_objective, secondary_objective)
+
+    return result
+
+def pareto_front_table_read_csv() -> List[Dict[str, str]]:
+    with open(RESULT_CSV_FILE, mode="r", encoding="utf-8", newline="") as f:
+        return list(csv.DictReader(f))
+
+def create_pareto_front_table(idxs: List[int], metric_x: str, metric_y: str) -> Table:
+    table = Table(title=f"Pareto-Front for {metric_y}/{metric_x}:", show_lines=True)
+
+    rows = pareto_front_table_read_csv()
+    if not rows:
+        table.add_column("No data found")
+        return table
+
+    filtered_rows = pareto_front_table_filter_rows(rows, idxs)
+    if not filtered_rows:
+        table.add_column("No matching entries")
+        return table
+
+    param_cols, result_cols = pareto_front_table_get_columns(filtered_rows[0])
+
+    pareto_front_table_add_headers(table, param_cols, result_cols)
+    pareto_front_table_add_rows(table, filtered_rows, param_cols, result_cols)
+
+    return table
+
+def pareto_front_build_return_structure(
+    path_to_calculate: str,
+    selected_points: List[Tuple[Any, float, float]],
+    records: Dict[Tuple[int, str], Dict[str, Dict[str, float]]],
+    absolute_metrics: List[str],
+    primary_name: str,
+    secondary_name: str
+) -> dict:
+    results_csv_file = f"{path_to_calculate}/{RESULTS_CSV_FILENAME}"
+    result_names_file = f"{path_to_calculate}/result_names.txt"
+
+    with open(result_names_file, mode="r", encoding="utf-8") as f:
+        result_names = [line.strip() for line in f if line.strip()]
+
+    csv_rows = {}
+    with open(results_csv_file, mode="r", encoding="utf-8", newline='') as csvfile:
+        reader = csv.DictReader(csvfile)
+        for row in reader:
+            trial_index = int(row['trial_index'])
+            csv_rows[trial_index] = row
+
+    ignored_columns = {'trial_index', 'arm_name', 'trial_status', 'generation_node'}
+    ignored_columns.update(result_names)
+
+    param_dicts = []
+    idxs = []
+    means_dict = defaultdict(list)
+
+    for (trial_index, arm_name), _, _ in selected_points:
+        row = csv_rows.get(trial_index, {})
+        if row == {} or row is None or row['arm_name'] != arm_name:
+            continue
+
+        idxs.append(int(row["trial_index"]))
+
+        param_dict: dict[str, int | float | str] = {}
+        for key, value in row.items():
+            if key not in ignored_columns:
+                try:
+                    param_dict[key] = int(value)
+                except ValueError:
+                    try:
+                        param_dict[key] = float(value)
+                    except ValueError:
+                        param_dict[key] = value
+
+        param_dicts.append(param_dict)
+
+        for metric in absolute_metrics:
+            means_dict[metric].append(records[(trial_index, arm_name)]['means'].get(metric, float("nan")))
+
+    ret = {
+        primary_name: {
+            secondary_name: {
+                "absolute_metrics": absolute_metrics,
+                "param_dicts": param_dicts,
+                "means": dict(means_dict),
+                "idxs": idxs
+            },
+            "absolute_metrics": absolute_metrics
+        }
+    }
+
+    return ret
+
+def pareto_front_aggregate_data(path_to_calculate: str) -> Optional[Dict[Tuple[int, str], Dict[str, Dict[str, float]]]]:
+    results_csv_file = f"{path_to_calculate}/{RESULTS_CSV_FILENAME}"
+    result_names_file = f"{path_to_calculate}/result_names.txt"
+
+    if not os.path.exists(results_csv_file) or not os.path.exists(result_names_file):
+        return None
+
+    with open(result_names_file, mode="r", encoding="utf-8") as f:
+        result_names = [line.strip() for line in f if line.strip()]
+
+    records: dict = defaultdict(lambda: {'means': {}})
+
+    with open(results_csv_file, encoding="utf-8", mode="r", newline='') as csvfile:
+        reader = csv.DictReader(csvfile)
+        for row in reader:
+            trial_index = int(row['trial_index'])
+            arm_name = row['arm_name']
+            key = (trial_index, arm_name)
+
+            for metric in result_names:
+                if metric in row:
+                    try:
+                        records[key]['means'][metric] = float(row[metric])
+                    except ValueError:
+                        continue
+
+    return records
+
+def plot_pareto_frontier_sixel(data: Any, x_metric: str, y_metric: str) -> None:
+    if data is None:
+        print("[italic yellow]The data seems to be empty. Cannot plot pareto frontier.[/]")
+        return
+
+    if not supports_sixel():
+        print(f"[italic yellow]Your console does not support sixel-images. Will not print Pareto-frontier as a matplotlib-sixel-plot for {x_metric}/{y_metric}.[/]")
+        return
+
+    import matplotlib.pyplot as plt
+
+    means = data[x_metric][y_metric]["means"]
+
+    x_values = means[x_metric]
+    y_values = means[y_metric]
+
+    fig, _ax = plt.subplots()
+
+    _ax.scatter(x_values, y_values, s=50, marker='x', c='blue', label='Data Points')
+
+    _ax.set_xlabel(x_metric)
+    _ax.set_ylabel(y_metric)
+
+    _ax.set_title(f'Pareto-Front {x_metric}/{y_metric}')
+
+    _ax.ticklabel_format(style='plain', axis='both', useOffset=False)
+
+    with tempfile.NamedTemporaryFile(suffix=".png", delete=True) as tmp_file:
+        plt.savefig(tmp_file.name, dpi=300)
+
+        print_image_to_cli(tmp_file.name, 1000)
+
+    plt.close(fig)
+
+def pareto_front_table_get_columns(first_row: Dict[str, str]) -> Tuple[List[str], List[str]]:
+    all_columns = list(first_row.keys())
+    ignored_cols = set(special_col_names) - {"trial_index"}
+
+    param_cols = [col for col in all_columns if col not in ignored_cols and col not in arg_result_names and not col.startswith("OO_Info_")]
+    result_cols = [col for col in arg_result_names if col in all_columns]
+    return param_cols, result_cols
+
+def check_factorial_range() -> None:
+    if args.model and args.model == "FACTORIAL":
+        _fatal_error("\n⚠ --model FACTORIAL cannot be used with range parameter", 181)
+
+def check_if_range_types_are_invalid(value_type: str, valid_value_types: list) -> None:
+    if value_type not in valid_value_types:
+        valid_value_types_string = ", ".join(valid_value_types)
+        _fatal_error(f"⚠ {value_type} is not a valid value type. Valid types for range are: {valid_value_types_string}", 181)
+
+def check_range_params_length(this_args: Union[str, list]) -> None:
+    if len(this_args) != 5 and len(this_args) != 4 and len(this_args) != 6:
+        _fatal_error("\n⚠ --parameter for type range must have 4 (or 5, the last one being optional and float by default, or 6, while the last one is true or false) parameters: <NAME> range <START> <END> (<TYPE (int or float)>, <log_scale: bool>)", 181)
+
+def die_if_lower_and_upper_bound_equal_zero(lower_bound: Union[int, float], upper_bound: Union[int, float]) -> None:
+    if upper_bound is None or lower_bound is None:
+        _fatal_error("die_if_lower_and_upper_bound_equal_zero: upper_bound or lower_bound is None. Cannot continue.", 91)
+    if upper_bound == lower_bound:
+        if lower_bound == 0:
+            _fatal_error(f"⚠ Lower bound and upper bound are equal: {lower_bound}, cannot automatically fix this, because they -0 = +0 (usually a quickfix would be to set lower_bound = -upper_bound)", 181)
+        print_red(f"⚠ Lower bound and upper bound are equal: {lower_bound}, setting lower_bound = -upper_bound")
+        if upper_bound is not None:
+            lower_bound = -upper_bound
+
+def format_value(value: Any, float_format: str = '.80f') -> str:
+    try:
+        if isinstance(value, float):
+            s = format(value, float_format)
+            s = s.rstrip('0').rstrip('.') if '.' in s else s
+            return s
+        return str(value)
+    except Exception as e:
+        print_red(f"⚠ Error formatting the number {value}: {e}")
+        return str(value)
+
+def replace_parameters_in_string(
+    parameters: dict,
+    input_string: str,
+    float_format: str = '.20f',
+    additional_prefixes: list[str] = [],
+    additional_patterns: list[str] = [],
+) -> str:
+    try:
+        prefixes = ['$', '%'] + additional_prefixes
+        patterns = ['{' + 'key' + '}', '(' + '{' + 'key' + '}' + ')'] + additional_patterns
+
+        for key, value in parameters.items():
+            replacement = format_value(value, float_format=float_format)
+            for prefix in prefixes:
+                for pattern in patterns:
+                    token = prefix + pattern.format(key=key)
+                    input_string = input_string.replace(token, replacement)
+
+        input_string = input_string.replace('\r', ' ').replace('\n', ' ')
+        return input_string
+
+    except Exception as e:
+        print_red(f"\n⚠ Error: {e}")
+        return ""
+
+def get_memory_usage() -> float:
+    user_uid = os.getuid()
+
+    memory_usage = float(sum(
+        p.memory_info().rss for p in psutil.process_iter(attrs=['memory_info', 'uids'])
+        if p.info['uids'].real == user_uid
+    ) / (1024 * 1024))
+
+    return memory_usage
+
+class MonitorProcess:
+    def __init__(self: Any, pid: int, interval: float = 1.0) -> None:
+        self.pid = pid
+        self.interval = interval
+        self.running = True
+        self.thread = threading.Thread(target=self._monitor)
+        self.thread.daemon = True
+
+        fool_linter(f"self.thread.daemon was set to {self.thread.daemon}")
+
+    def _monitor(self: Any) -> None:
+        try:
+            _internal_process = psutil.Process(self.pid)
+            while self.running and _internal_process.is_running():
+                crf = get_current_run_folder()
+
+                if crf and crf != "":
+                    log_file_path = os.path.join(crf, "eval_nodes_cpu_ram_logs.txt")
+
+                    os.makedirs(os.path.dirname(log_file_path), exist_ok=True)
+
+                    with open(log_file_path, mode="a", encoding="utf-8") as log_file:
+                        hostname = socket.gethostname()
+
+                        slurm_job_id = os.getenv("SLURM_JOB_ID")
+
+                        if slurm_job_id:
+                            hostname += f"-SLURM-ID-{slurm_job_id}"
+
+                        total_memory = psutil.virtual_memory().total / (1024 * 1024)
+                        cpu_usage = psutil.cpu_percent(interval=5)
+
+                        memory_usage = get_memory_usage()
+
+                        unix_timestamp = int(time.time())
+
+                        log_file.write(f"\nUnix-Timestamp: {unix_timestamp}, Hostname: {hostname}, CPU: {cpu_usage:.2f}%, RAM: {memory_usage:.2f} MB / {total_memory:.2f} MB\n")
+                time.sleep(self.interval)
+        except psutil.NoSuchProcess:
+            pass
+
+    def __enter__(self: Any) -> None:
+        self.thread.start()
+        return self
+
+    def __exit__(self: Any, exc_type: Any, exc_value: Any, _traceback: Any) -> None:
+        self.running = False
+        self.thread.join()
+
+def execute_bash_code_log_time(code: str) -> list:
+    process_item = subprocess.Popen(code, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+
+    with MonitorProcess(process_item.pid):
+        try:
+            stdout, stderr = process_item.communicate()
+            result = subprocess.CompletedProcess(
+                args=code, returncode=process_item.returncode, stdout=stdout, stderr=stderr
+            )
             return [result.stdout, result.stderr, result.returncode, None]
         except subprocess.CalledProcessError as e:
             real_exit_code = e.returncode
@@ -3550,7 +3943,7 @@ def _add_to_csv_acquire_lock(lockfile: str, dir_path: str) -> bool:
             time.sleep(wait_time)
             max_wait -= wait_time
         except Exception as e:
-            print("Lock error:", e)
+            print_red(f"Lock error: {e}")
             return False
     return False
 
@@ -3588,11 +3981,11 @@ def _add_to_csv_rewrite_file(file_path: str, rows: List[list], existing_heading:
         writer = csv.writer(tmp_file)
         writer.writerow(all_headings)
         for row in rows[1:]:
-            tmp_file.writerow([
+            tmp_file.writerow([ # type: ignore[attr-defined]
                 row[existing_heading.index(h)] if h in existing_heading else ""
                 for h in all_headings
             ])
-        tmp_file.writerow([
+        tmp_file.writerow([ # type: ignore[attr-defined]
             formatted_data[new_heading.index(h)] if h in new_heading else ""
             for h in all_headings
         ])
@@ -3623,12 +4016,12 @@ def find_file_paths(_text: str) -> List[str]:
 def check_file_info(file_path: str) -> str:
     if not os.path.exists(file_path):
         if not args.tests:
-            print(f"check_file_info: The file {file_path} does not exist.")
+            print_red(f"check_file_info: The file {file_path} does not exist.")
         return ""
 
     if not os.access(file_path, os.R_OK):
         if not args.tests:
-            print(f"check_file_info: The file {file_path} is not readable.")
+            print_red(f"check_file_info: The file {file_path} is not readable.")
         return ""
 
     file_stat = os.stat(file_path)
@@ -3742,7 +4135,7 @@ def count_defective_nodes(file_path: Union[str, None] = None, entry: Any = None)
         return sorted(set(entries))
 
     except Exception as e:
-        print(f"An error has occurred: {e}")
+        print_red(f"An error has occurred: {e}")
         return []
 
 def test_gpu_before_evaluate(return_in_case_of_error: dict) -> Union[None, dict]:
@@ -3753,7 +4146,7 @@ def test_gpu_before_evaluate(return_in_case_of_error: dict) -> Union[None, dict]
 
                 fool_linter(tmp)
         except RuntimeError:
-            print(f"Node {socket.gethostname()} was detected as faulty. It should have had a GPU, but there is an error initializing the CUDA driver. Adding this node to the --exclude list.")
+            print_red(f"Node {socket.gethostname()} was detected as faulty. It should have had a GPU, but there is an error initializing the CUDA driver. Adding this node to the --exclude list.")
             count_defective_nodes(None, socket.gethostname())
             return return_in_case_of_error
         except Exception:
@@ -4224,6 +4617,8 @@ def evaluate(parameters_with_trial_index: dict) -> Optional[Union[int, float, Di
     trial_index = parameters_with_trial_index["trial_idx"]
     submit_time = parameters_with_trial_index["submit_time"]
 
+    print(f'Trial-Index: {trial_index}')
+
     queue_time = abs(int(time.time()) - int(submit_time))
 
     start_nvidia_smi_thread()
@@ -4461,7 +4856,7 @@ def replace_string_with_params(input_string: str, params: list) -> str:
         return replaced_string
     except AssertionError as e:
         error_text = f"Error in replace_string_with_params: {e}"
-        print(error_text)
+        print_red(error_text)
         raise
 
     return ""
@@ -4738,9 +5133,7 @@ def get_sixel_graphics_data(_pd_csv: str, _force: bool = False) -> list:
             _params = [_command, plot, _tmp, plot_type, tmp_file, _width]
             data.append(_params)
         except Exception as e:
-            tb = traceback.format_exc()
-            print_red(f"Error trying to print {plot_type} to CLI: {e}, {tb}")
-            print_debug(f"Error trying to print {plot_type} to CLI: {e}")
+            print_red(f"Error trying to print {plot_type} to CLI: {e}")
 
     return data
 
@@ -4931,15 +5324,17 @@ def abandon_job(job: Job, trial_index: int, reason: str) -> bool:
     if job:
         try:
             if ax_client:
-                _trial = ax_client.get_trial(trial_index)
-                _trial.mark_abandoned(reason=reason)
+                _trial = get_ax_client_trial(trial_index)
+                if _trial is None:
+                    return False
+
+                mark_abandoned(_trial, reason, trial_index)
                 print_debug(f"abandon_job: removing job {job}, trial_index: {trial_index}")
                 global_vars["jobs"].remove((job, trial_index))
             else:
-                _fatal_error("ax_client could not be found", 9)
+                _fatal_error("ax_client could not be found", 101)
         except Exception as e:
-            print(f"ERROR in line {get_line_info()}: {e}")
-            print_debug(f"ERROR in line {get_line_info()}: {e}")
+            print_red(f"ERROR in line {get_line_info()}: {e}")
             return False
         job.cancel()
         return True
@@ -4952,20 +5347,6 @@ def abandon_all_jobs() -> None:
         if not abandoned:
             print_debug(f"Job {job} could not be abandoned.")
 
-def show_pareto_or_error_msg(path_to_calculate: str, res_names: list = arg_result_names, disable_sixel_and_table: bool = False) -> None:
-    if args.dryrun:
-        print_debug("Not showing Pareto-frontier data with --dryrun")
-        return None
-
-    if len(res_names) > 1:
-        try:
-            show_pareto_frontier_data(path_to_calculate, res_names, disable_sixel_and_table)
-        except Exception as e:
-            print_red(f"show_pareto_frontier_data() failed with exception '{e}'")
-    else:
-        print_debug(f"show_pareto_frontier_data will NOT be executed because len(arg_result_names) is {len(arg_result_names)}")
-    return None
-
 def end_program(_force: Optional[bool] = False, exit_code: Optional[int] = None) -> None:
     global END_PROGRAM_RAN
 
@@ -5002,7 +5383,7 @@ def end_program(_force: Optional[bool] = False, exit_code: Optional[int] = None)
             _exit = new_exit
     except (SignalUSR, SignalINT, SignalCONT, KeyboardInterrupt):
         print_red("\n⚠ You pressed CTRL+C or a signal was sent. Program execution halted while ending program.")
-        print("\n⚠ KeyboardInterrupt signal was sent. Ending program will still run.")
+        print_red("\n⚠ KeyboardInterrupt signal was sent. Ending program will still run.")
         new_exit = show_end_table_and_save_end_files()
         if new_exit > 0:
             _exit = new_exit
@@ -5023,21 +5404,31 @@ def end_program(_force: Optional[bool] = False, exit_code: Optional[int] = None)
 
     my_exit(_exit)
 
+def save_ax_client_to_json_file(checkpoint_filepath: str) -> None:
+    if not ax_client:
+        my_exit(101)
+
+        return None
+
+    ax_client.save_to_json_file(checkpoint_filepath)
+
+    return None
+
 def save_checkpoint(trial_nr: int = 0, eee: Union[None, str, Exception] = None) -> None:
     if trial_nr > 3:
         if eee:
-            print(f"Error during saving checkpoint: {eee}")
+            print_red(f"Error during saving checkpoint: {eee}")
         else:
-            print("Error during saving checkpoint")
+            print_red("Error during saving checkpoint")
         return
 
     try:
         checkpoint_filepath = get_state_file_name('checkpoint.json')
 
         if ax_client:
-            ax_client.save_to_json_file(filepath=checkpoint_filepath)
+            save_ax_client_to_json_file(checkpoint_filepath)
         else:
-            _fatal_error("Something went wrong using the ax_client", 9)
+            _fatal_error("Something went wrong using the ax_client", 101)
     except Exception as e:
         save_checkpoint(trial_nr + 1, e)
 
@@ -5198,7 +5589,7 @@ def parse_equation_item(comparer_found: bool, item: str, parsed: list, parsed_or
         })
     elif item in [">=", "<="]:
         if comparer_found:
-            print("There is already one comparison operator! Cannot have more than one in an equation!")
+            print_red("There is already one comparison operator! Cannot have more than one in an equation!")
             return_totally = True
         comparer_found = True
 
@@ -5409,20 +5800,9 @@ def check_equation(variables: list, equation: str) -> Union[str, bool]:
 def set_objectives() -> dict:
     objectives = {}
 
-    for rn in args.result_names:
-        key, value = "", ""
-
-        if "=" in rn:
-            key, value = rn.split('=', 1)
-        else:
-            key = rn
-            value = ""
-
-        if value not in ["min", "max"]:
-            if value:
-                print_yellow(f"Value '{value}' for --result_names {rn} is not a valid value. Must be min or max. Will be set to min.")
-
-            value = "min"
+    k = 0
+    for key in arg_result_names:
+        value = arg_result_min_or_max[k]
 
         _min = True
 
@@ -5430,12 +5810,18 @@ def set_objectives() -> dict:
             _min = False
 
         objectives[key] = ObjectiveProperties(minimize=_min)
+        k = k + 1
 
     return objectives
 
-def set_experiment_constraints(experiment_constraints: Optional[list], experiment_args: dict, _experiment_parameters: Union[dict, list]) -> dict:
-    if experiment_constraints and len(experiment_constraints):
+def set_experiment_constraints(experiment_constraints: Optional[list], experiment_args: dict, _experiment_parameters: Optional[Union[dict, list]]) -> dict:
+    if _experiment_parameters is None:
+        print_red("set_experiment_constraints: _experiment_parameters was None")
+        my_exit(95)
+
+        return {}
 
+    if experiment_constraints and len(experiment_constraints):
         experiment_args["parameter_constraints"] = []
 
         if experiment_constraints:
@@ -5464,11 +5850,15 @@ def set_experiment_constraints(experiment_constraints: Optional[list], experimen
 
     return experiment_args
 
-def replace_parameters_for_continued_jobs(parameter: Optional[list], cli_params_experiment_parameters: Optional[list]) -> None:
+def replace_parameters_for_continued_jobs(parameter: Optional[list], cli_params_experiment_parameters: Optional[dict | list]) -> None:
+    if not experiment_parameters:
+        print_red("replace_parameters_for_continued_jobs: experiment_parameters was False")
+        return None
+
     if args.worker_generator_path:
         return None
 
-    def get_name(obj) -> Optional[str]:
+    def get_name(obj: Any) -> Optional[str]:
         """Extract a parameter name from dict, list, or tuple safely."""
         if isinstance(obj, dict):
             return obj.get("name")
@@ -5550,13 +5940,13 @@ def copy_continue_uuid() -> None:
         print_debug(f"copy_continue_uuid: Source file does not exist: {source_file}")
 
 def load_ax_client_from_experiment_parameters() -> None:
-    #pprint(experiment_parameters)
-    global ax_client
+    if experiment_parameters:
+        global ax_client
 
-    tmp_file_path = get_tmp_file_from_json(experiment_parameters)
-    ax_client = AxClient.load_from_json_file(tmp_file_path)
-    ax_client = cast(AxClient, ax_client)
-    os.unlink(tmp_file_path)
+        tmp_file_path = get_tmp_file_from_json(experiment_parameters)
+        ax_client = AxClient.load_from_json_file(tmp_file_path)
+        ax_client = cast(AxClient, ax_client)
+        os.unlink(tmp_file_path)
 
 def save_checkpoint_for_continued() -> None:
     checkpoint_filepath = get_state_file_name('checkpoint.json')
@@ -5568,12 +5958,15 @@ def save_checkpoint_for_continued() -> None:
         _fatal_error(f"{checkpoint_filepath} not found. Cannot continue_previous_job without.", 47)
 
 def load_original_generation_strategy(original_ax_client_file: str) -> None:
-    with open(original_ax_client_file, encoding="utf-8") as f:
-        loaded_original_ax_client_json = json.load(f)
-        original_generation_strategy = loaded_original_ax_client_json["generation_strategy"]
+    if experiment_parameters:
+        with open(original_ax_client_file, encoding="utf-8") as f:
+            loaded_original_ax_client_json = json.load(f)
+            original_generation_strategy = loaded_original_ax_client_json["generation_strategy"]
 
-        if original_generation_strategy:
-            experiment_parameters["generation_strategy"] = original_generation_strategy
+            if original_generation_strategy:
+                experiment_parameters["generation_strategy"] = original_generation_strategy
+    else:
+        print_red("load_original_generation_strategy: experiment_parameters was empty!")
 
 def wait_for_checkpoint_file(checkpoint_file: str) -> None:
     start_time = time.time()
@@ -5586,10 +5979,6 @@ def wait_for_checkpoint_file(checkpoint_file: str) -> None:
     elapsed = int(time.time() - start_time)
     console.print(f"[green]Checkpoint file found after {elapsed} seconds[/green]   ")
 
-def __get_experiment_parameters__check_ax_client() -> None:
-    if not ax_client:
-        _fatal_error("Something went wrong with the ax_client", 9)
-
 def validate_experiment_parameters() -> None:
     if experiment_parameters is None:
         print_red("Error: experiment_parameters is None.")
@@ -5599,6 +5988,8 @@ def validate_experiment_parameters() -> None:
         print_red(f"Error: experiment_parameters is not a dict: {type(experiment_parameters).__name__}")
         my_exit(95)
 
+        sys.exit(95)
+
     path_checks = [
         ("experiment", experiment_parameters),
         ("search_space", experiment_parameters.get("experiment")),
@@ -5610,7 +6001,12 @@ def validate_experiment_parameters() -> None:
             print_red(f"Error: Missing key '{key}' at level: {current_level}")
             my_exit(95)
 
-def __get_experiment_parameters__load_from_checkpoint(continue_previous_job: str, cli_params_experiment_parameters: Optional[list]) -> Tuple[Any, str, str]:
+def load_from_checkpoint(continue_previous_job: str, cli_params_experiment_parameters: Optional[dict | list]) -> Tuple[Any, str, str]:
+    if not ax_client:
+        print_red("load_from_checkpoint: ax_client was None")
+        my_exit(101)
+        return {}, "", ""
+
     print_debug(f"Load from checkpoint: {continue_previous_job}")
 
     checkpoint_file = f"{continue_previous_job}/state_files/checkpoint.json"
@@ -5636,7 +6032,7 @@ def __get_experiment_parameters__load_from_checkpoint(continue_previous_job: str
 
     replace_parameters_for_continued_jobs(args.parameter, cli_params_experiment_parameters)
 
-    ax_client.save_to_json_file(filepath=original_ax_client_file)
+    save_ax_client_to_json_file(original_ax_client_file)
 
     load_original_generation_strategy(original_ax_client_file)
     load_ax_client_from_experiment_parameters()
@@ -5652,6 +6048,12 @@ def __get_experiment_parameters__load_from_checkpoint(continue_previous_job: str
 
     experiment_constraints = get_constraints()
     if experiment_constraints:
+
+        if not experiment_parameters:
+            print_red("load_from_checkpoint: experiment_parameters was None")
+
+            return {}, "", ""
+
         experiment_args = set_experiment_constraints(
             experiment_constraints,
             experiment_args,
@@ -5660,7 +6062,116 @@ def __get_experiment_parameters__load_from_checkpoint(continue_previous_job: str
 
     return experiment_args, gpu_string, gpu_color
 
-def __get_experiment_parameters__create_new_experiment() -> Tuple[dict, str, str]:
+def get_experiment_args_import_python_script() -> str:
+
+    return """from ax.service.ax_client import AxClient, ObjectiveProperties
+from ax.adapter.registry import Generators
+import random
+
+"""
+
+def get_generate_and_test_random_function_str() -> str:
+    raw_data_entries = ",\n                ".join(
+        f'"{name}": random.uniform(0, 1)' for name in arg_result_names
+    )
+
+    return f"""
+def generate_and_test_random_parameters(n: int) -> None:
+    for _ in range(n):
+        print("======================================")
+        parameters, trial_index = ax_client.get_next_trial()
+        print("Trial Index:", trial_index)
+        print("Suggested parameters:", parameters)
+
+        ax_client.complete_trial(
+            trial_index=trial_index,
+            raw_data={{
+                {raw_data_entries}
+            }}
+        )
+
+generate_and_test_random_parameters({args.num_random_steps + 1})
+"""
+
+def get_global_gs_string() -> str:
+    seed_str = ""
+    if args.seed is not None:
+        seed_str = f"model_kwargs={{'seed': {args.seed}}},"
+
+    return f"""from ax.generation_strategy.generation_strategy import GenerationStep, GenerationStrategy
+
+global_gs = GenerationStrategy(
+    steps=[
+        GenerationStep(
+            generator=Generators.SOBOL,
+            num_trials={args.num_random_steps},
+            max_parallelism=5,
+            {seed_str}
+        ),
+        GenerationStep(
+            generator=Generators.{args.model},
+            num_trials=-1,
+            max_parallelism=5,
+        ),
+    ]
+)
+"""
+
+def get_debug_ax_client_str() -> str:
+    return """
+ax_client = AxClient(
+    verbose_logging=True,
+    enforce_sequential_optimization=False,
+    generation_strategy=global_gs
+)
+"""
+
+def write_ax_debug_python_code(experiment_args: dict) -> None:
+    if args.generation_strategy:
+        print_debug("Cannot write debug code for custom generation_strategy")
+        return None
+
+    if args.model in uncontinuable_models:
+        print_debug(f"Cannot write debug code for uncontinuable mode {args.model}")
+        return None
+
+    python_code = python_code = get_experiment_args_import_python_script() + \
+        get_global_gs_string() + \
+        get_debug_ax_client_str() + \
+        "experiment_args = " + pformat(experiment_args, width=120, compact=False) + \
+        "\nax_client.create_experiment(**experiment_args)\n" + \
+        get_generate_and_test_random_function_str()
+
+    file_path = f"{get_current_run_folder()}/debug.py"
+
+    try:
+        print_debug(python_code)
+        with open(file_path, "w", encoding="utf-8") as f:
+            f.write(python_code)
+    except Exception as e:
+        print_red(f"Error while writing {file_path}: {e}")
+
+    return None
+
+def create_ax_client_experiment(experiment_args: dict) -> None:
+    if not ax_client:
+        my_exit(101)
+
+        return None
+
+    write_ax_debug_python_code(experiment_args)
+
+    ax_client.create_experiment(**experiment_args)
+
+    return None
+
+def create_new_experiment() -> Tuple[dict, str, str]:
+    if ax_client is None:
+        print_red("create_new_experiment: ax_client is None")
+        my_exit(101)
+
+        return {}, "", ""
+
     objectives = set_objectives()
 
     experiment_args = {
@@ -5683,7 +6194,7 @@ def __get_experiment_parameters__create_new_experiment() -> Tuple[dict, str, str
     experiment_args = set_experiment_constraints(get_constraints(), experiment_args, experiment_parameters)
 
     try:
-        ax_client.create_experiment(**experiment_args)
+        create_ax_client_experiment(experiment_args)
         new_metrics = [Metric(k) for k in arg_result_names if k not in ax_client.metric_names]
         ax_client.experiment.add_tracking_metrics(new_metrics)
     except AssertionError as error:
@@ -5697,17 +6208,17 @@ def __get_experiment_parameters__create_new_experiment() -> Tuple[dict, str, str
 
     return experiment_args, gpu_string, gpu_color
 
-def get_experiment_parameters(cli_params_experiment_parameters: Optional[list]) -> Optional[Tuple[AxClient, dict, str, str]]:
+def get_experiment_parameters(cli_params_experiment_parameters: Optional[dict | list]) -> Optional[Tuple[dict, str, str]]:
     continue_previous_job = args.worker_generator_path or args.continue_previous_job
 
-    __get_experiment_parameters__check_ax_client()
+    check_ax_client()
 
     if continue_previous_job:
-        experiment_args, gpu_string, gpu_color = __get_experiment_parameters__load_from_checkpoint(continue_previous_job, cli_params_experiment_parameters)
+        experiment_args, gpu_string, gpu_color = load_from_checkpoint(continue_previous_job, cli_params_experiment_parameters)
     else:
-        experiment_args, gpu_string, gpu_color = __get_experiment_parameters__create_new_experiment()
+        experiment_args, gpu_string, gpu_color = create_new_experiment()
 
-    return ax_client, experiment_args, gpu_string, gpu_color
+    return experiment_args, gpu_string, gpu_color
 
 def get_type_short(typename: str) -> str:
     if typename == "RangeParameter":
@@ -5766,7 +6277,6 @@ def parse_single_experiment_parameter_table(classic_params: Optional[Union[list,
                 _upper = param["bounds"][1]
 
             _possible_int_lower = str(helpers.to_int_when_possible(_lower))
-            #print(f"name: {_name}, _possible_int_lower: {_possible_int_lower}, lower: {_lower}")
             _possible_int_upper = str(helpers.to_int_when_possible(_upper))
 
             rows.append([_name, _short_type, _possible_int_lower, _possible_int_upper, "", value_type, log_scale])
@@ -5843,19 +6353,62 @@ def print_ax_parameter_constraints_table(experiment_args: dict) -> None:
 
     return None
 
+def check_base_for_print_overview() -> Optional[bool]:
+    if args.continue_previous_job is not None and arg_result_names is not None and len(arg_result_names) != 0 and original_result_names is not None and len(original_result_names) != 0:
+        print_yellow("--result_names will be ignored in continued jobs. The result names from the previous job will be used.")
+
+    if ax_client is None:
+        print_red("ax_client was None")
+        return None
+
+    if ax_client.experiment is None:
+        print_red("ax_client.experiment was None")
+        return None
+
+    if ax_client.experiment.optimization_config is None:
+        print_red("ax_client.experiment.optimization_config was None")
+        return None
+
+    return True
+
+def get_config_objectives() -> Any:
+    if not ax_client:
+        print_red("create_new_experiment: ax_client is None")
+        my_exit(101)
+
+        return None
+
+    config_objectives = None
+
+    if ax_client.experiment and ax_client.experiment.optimization_config:
+        opt_config = ax_client.experiment.optimization_config
+        if opt_config.is_moo_problem:
+            objective = getattr(opt_config, "objective", None)
+            if objective and getattr(objective, "objectives", None) is not None:
+                config_objectives = objective.objectives
+            else:
+                print_debug("ax_client.experiment.optimization_config.objective was None")
+        else:
+            config_objectives = [opt_config.objective]
+    else:
+        print_debug("ax_client.experiment or optimization_config was None")
+
+    return config_objectives
+
 def print_result_names_overview_table() -> None:
     if not ax_client:
         _fatal_error("Tried to access ax_client in print_result_names_overview_table, but it failed, because the ax_client was not defined.", 101)
 
         return None
 
-    if args.continue_previous_job is not None and args.result_names is not None and len(args.result_names) != 0 and original_result_names is not None and len(original_result_names) != 0:
-        print_yellow("--result_names will be ignored in continued jobs. The result names from the previous job will be used.")
+    if check_base_for_print_overview() is None:
+        return None
 
-    if ax_client.experiment.optimization_config.is_moo_problem:
-        config_objectives = ax_client.experiment.optimization_config.objective.objectives
-    else:
-        config_objectives = [ax_client.experiment.optimization_config.objective]
+    config_objectives = get_config_objectives()
+
+    if config_objectives is None:
+        print_red("config_objectives not found")
+        return None
 
     res_names = []
     res_min_max = []
@@ -5950,28 +6503,31 @@ def print_overview_tables(classic_params: Optional[Union[list, dict]], experimen
     print_result_names_overview_table()
 
 def update_progress_bar(nr: int) -> None:
-    try:
-        progress_bar.update(nr)
-    except Exception as e:
-        print(f"Error updating progress bar: {e}")
+    log_data()
+
+    if progress_bar is not None:
+        try:
+            progress_bar.update(nr)
+        except Exception as e:
+            print_red(f"Error updating progress bar: {e}")
+    else:
+        print_red("update_progress_bar: progress_bar was None")
 
 def get_current_model_name() -> str:
     if overwritten_to_random:
         return "Random*"
 
+    gs_model = "unknown model"
+
     if ax_client:
         try:
             if args.generation_strategy:
-                idx = getattr(ax_client.generation_strategy, "current_step_index", None)
+                idx = getattr(global_gs, "current_step_index", None)
                 if isinstance(idx, int):
                     if 0 <= idx < len(generation_strategy_names):
                         gs_model = generation_strategy_names[int(idx)]
-                    else:
-                        gs_model = "unknown model"
-                else:
-                    gs_model = "unknown model"
             else:
-                gs_model = getattr(ax_client.generation_strategy, "current_node_name", "unknown model")
+                gs_model = getattr(global_gs, "current_node_name", "unknown model")
 
             if gs_model:
                 return str(gs_model)
@@ -6028,7 +6584,7 @@ def get_workers_string() -> str:
     )
 
 def _get_workers_string_collect_stats() -> dict:
-    stats = {}
+    stats: dict = {}
     for job, _ in global_vars["jobs"][:]:
         state = state_from_job(job)
         stats[state] = stats.get(state, 0) + 1
@@ -6077,8 +6633,8 @@ def submitted_jobs(nr: int = 0) -> int:
 def count_jobs_in_squeue() -> tuple[int, str]:
     global _last_count_time, _last_count_result
 
-    now = time.time()
-    if _last_count_result != (0, "") and now - _last_count_time < 15:
+    now = int(time.time())
+    if _last_count_result != (0, "") and now - _last_count_time < 5:
         return _last_count_result
 
     _len = len(global_vars["jobs"])
@@ -6100,6 +6656,7 @@ def count_jobs_in_squeue() -> tuple[int, str]:
             check=True,
             text=True
         )
+
         if "slurm_load_jobs error" in result.stderr:
             _last_count_result = (_len, "Detected slurm_load_jobs error in stderr.")
             _last_count_time = now
@@ -6140,6 +6697,8 @@ def log_worker_numbers() -> None:
         if len(WORKER_PERCENTAGE_USAGE) == 0 or WORKER_PERCENTAGE_USAGE[len(WORKER_PERCENTAGE_USAGE) - 1] != this_values:
             WORKER_PERCENTAGE_USAGE.append(this_values)
 
+        write_worker_usage()
+
 def get_slurm_in_brackets(in_brackets: list) -> list:
     if is_slurm_job():
         workers_strings = get_workers_string()
@@ -6224,6 +6783,8 @@ def progressbar_description(new_msgs: Union[str, List[str]] = []) -> None:
     global last_progress_bar_desc
     global last_progress_bar_refresh_time
 
+    log_data()
+
     if isinstance(new_msgs, str):
         new_msgs = [new_msgs]
 
@@ -6241,7 +6802,7 @@ def progressbar_description(new_msgs: Union[str, List[str]] = []) -> None:
         print_red("Cannot update progress bar! It is None.")
 
 def clean_completed_jobs() -> None:
-    job_states_to_be_removed = ["early_stopped", "abandoned", "cancelled", "timeout", "interrupted", "failed", "preempted", "node_fail", "boot_fail"]
+    job_states_to_be_removed = ["early_stopped", "abandoned", "cancelled", "timeout", "interrupted", "failed", "preempted", "node_fail", "boot_fail", "finished"]
     job_states_to_be_ignored = ["ready", "completed", "unknown", "pending", "running", "completing", "out_of_memory", "requeued", "resv_del_hold"]
 
     for job, trial_index in global_vars["jobs"][:]:
@@ -6299,7 +6860,7 @@ def load_existing_job_data_into_ax_client() -> None:
     nr_of_imported_jobs = get_nr_of_imported_jobs()
     set_nr_inserted_jobs(NR_INSERTED_JOBS + nr_of_imported_jobs)
 
-def parse_parameter_type_error(_error_message: Union[str, None]) -> Optional[dict]:
+def parse_parameter_type_error(_error_message: Union[Exception, str, None]) -> Optional[dict]:
     if not _error_message:
         return None
 
@@ -6366,7 +6927,7 @@ def get_generation_node_for_index(
     results_list: List[Dict[str, Any]],
     index: int,
     __status: Any,
-    base_str: str
+    base_str: Optional[str]
 ) -> str:
     __status.update(f"{base_str}: Getting generation node")
     try:
@@ -6386,7 +6947,7 @@ def get_generation_node_for_index(
 
         return generation_node
     except Exception as e:
-        print(f"Error while get_generation_node_for_index: {e}")
+        print_red(f"Error while get_generation_node_for_index: {e}")
         return "MANUAL"
 
 def _get_generation_node_for_index_index_valid(
@@ -6491,7 +7052,7 @@ def normalize_path(file_path: str) -> str:
 
 def insert_jobs_from_lists(csv_path: str, arm_params_list: Any, results_list: Any, __status: Any) -> None:
     cnt = 0
-    err_msgs = []
+    err_msgs: list = []
 
     for i, (arm_params, result) in enumerate(zip(arm_params_list, results_list)):
         base_str = f"[bold green]Loading job {i}/{len(results_list)} from {csv_path} into ax_client, result: {result}"
@@ -6525,9 +7086,16 @@ def try_insert_job(csv_path: str, arm_params: Dict, result: Any, i: int, arm_par
             f"This can happen when the csv file has different parameters or results as the main job one's "
             f"or other imported jobs. Error: {e}"
         )
-        if err_msg not in err_msgs:
-            print_red(err_msg)
-            err_msgs.append(err_msg)
+
+        if err_msgs is None:
+            print_red("try_insert_job: err_msgs was None")
+        else:
+            if isinstance(err_msgs, list):
+                if err_msg not in err_msgs:
+                    print_red(err_msg)
+                    err_msgs.append(err_msg)
+            elif isinstance(err_msgs, str):
+                err_msgs += f"\n{err_msg}"
 
     return cnt
 
@@ -6544,33 +7112,52 @@ def update_global_job_counters(cnt: int) -> None:
         set_max_eval(max_eval + cnt)
         set_nr_inserted_jobs(NR_INSERTED_JOBS + cnt)
 
-def __insert_job_into_ax_client__update_status(__status: Optional[Any], base_str: Optional[str], new_text: str) -> None:
+def update_status(__status: Optional[Any], base_str: Optional[str], new_text: str) -> None:
     if __status and base_str:
         __status.update(f"{base_str}: {new_text}")
 
-def __insert_job_into_ax_client__check_ax_client() -> None:
+def check_ax_client() -> None:
     if ax_client is None or not ax_client:
         _fatal_error("insert_job_into_ax_client: ax_client was not defined where it should have been", 101)
 
-def __insert_job_into_ax_client__attach_trial(arm_params: dict) -> Tuple[Any, int]:
+def attach_ax_client_data(arm_params: dict) -> Optional[Tuple[Any, int]]:
+    if not ax_client:
+        my_exit(101)
+
+        return None
+
     new_trial = ax_client.attach_trial(arm_params)
+
+    return new_trial
+
+def attach_trial(arm_params: dict) -> Tuple[Any, int]:
+    if ax_client is None:
+        raise RuntimeError("attach_trial: ax_client was empty")
+
+    new_trial = attach_ax_client_data(arm_params)
     if not isinstance(new_trial, tuple) or len(new_trial) < 2:
         raise RuntimeError("attach_trial didn't return the expected tuple")
     return new_trial
 
-def __insert_job_into_ax_client__get_trial(trial_idx: int) -> Any:
+def get_trial_by_index(trial_idx: int) -> Any:
+    if ax_client is None:
+        raise RuntimeError("get_trial_by_index: ax_client was empty")
+
     trial = ax_client.experiment.trials.get(trial_idx)
     if trial is None:
         raise RuntimeError(f"Trial with index {trial_idx} not found")
     return trial
 
-def __insert_job_into_ax_client__create_generator_run(arm_params: dict, trial_idx: int, new_job_type: str) -> GeneratorRun:
+def create_generator_run(arm_params: dict, trial_idx: int, new_job_type: str) -> GeneratorRun:
     arm = Arm(parameters=arm_params, name=f'{trial_idx}_0')
     return GeneratorRun(arms=[arm], generation_node_name=new_job_type)
 
-def __insert_job_into_ax_client__complete_trial_if_result(trial_idx: int, result: dict, __status: Optional[Any], base_str: Optional[str]) -> None:
+def complete_trial_if_result(trial_idx: int, result: dict, __status: Optional[Any], base_str: Optional[str]) -> None:
+    if ax_client is None:
+        raise RuntimeError("complete_trial_if_result: ax_client was empty")
+
     if f"{result}" != "":
-        __insert_job_into_ax_client__update_status(__status, base_str, "Completing trial")
+        update_status(__status, base_str, "Completing trial")
         is_ok = True
 
         for keyname in result.keys():
@@ -6578,20 +7165,20 @@ def __insert_job_into_ax_client__complete_trial_if_result(trial_idx: int, result
                 is_ok = False
 
         if is_ok:
-            ax_client.complete_trial(trial_index=trial_idx, raw_data=result)
-            __insert_job_into_ax_client__update_status(__status, base_str, "Completed trial")
+            complete_ax_client_trial(trial_idx, result)
+            update_status(__status, base_str, "Completed trial")
         else:
             print_debug("Empty job encountered")
     else:
-        __insert_job_into_ax_client__update_status(__status, base_str, "Found trial without result. Not adding it.")
+        update_status(__status, base_str, "Found trial without result. Not adding it.")
 
-def __insert_job_into_ax_client__save_results_if_needed(__status: Optional[Any], base_str: Optional[str]) -> None:
+def save_results_if_needed(__status: Optional[Any], base_str: Optional[str]) -> None:
     if not args.worker_generator_path:
-        __insert_job_into_ax_client__update_status(__status, base_str, f"Saving {RESULTS_CSV_FILENAME}")
+        update_status(__status, base_str, f"Saving {RESULTS_CSV_FILENAME}")
         save_results_csv()
-        __insert_job_into_ax_client__update_status(__status, base_str, f"Saved {RESULTS_CSV_FILENAME}")
+        update_status(__status, base_str, f"Saved {RESULTS_CSV_FILENAME}")
 
-def __insert_job_into_ax_client__handle_type_error(e: Exception, arm_params: dict) -> bool:
+def handle_insert_job_error(e: Exception, arm_params: dict) -> bool:
     parsed_error = parse_parameter_type_error(e)
     if parsed_error is not None:
         param = parsed_error["parameter_name"]
@@ -6614,37 +7201,37 @@ def insert_job_into_ax_client(
     result: dict,
     new_job_type: str = "MANUAL",
     __status: Optional[Any] = None,
-    base_str: str = None
+    base_str: Optional[str] = None
 ) -> bool:
-    __insert_job_into_ax_client__check_ax_client()
+    check_ax_client()
 
     done_converting = False
     while not done_converting:
         try:
-            __insert_job_into_ax_client__update_status(__status, base_str, "Checking ax client")
+            update_status(__status, base_str, "Checking ax client")
             if ax_client is None:
                 return False
 
-            __insert_job_into_ax_client__update_status(__status, base_str, "Attaching new trial")
-            _, new_trial_idx = __insert_job_into_ax_client__attach_trial(arm_params)
+            update_status(__status, base_str, "Attaching new trial")
+            _, new_trial_idx = attach_trial(arm_params)
 
-            __insert_job_into_ax_client__update_status(__status, base_str, "Getting new trial")
-            trial = __insert_job_into_ax_client__get_trial(new_trial_idx)
-            __insert_job_into_ax_client__update_status(__status, base_str, "Got new trial")
+            update_status(__status, base_str, "Getting new trial")
+            trial = get_trial_by_index(new_trial_idx)
+            update_status(__status, base_str, "Got new trial")
 
-            __insert_job_into_ax_client__update_status(__status, base_str, "Creating new arm")
-            manual_generator_run = __insert_job_into_ax_client__create_generator_run(arm_params, new_trial_idx, new_job_type)
+            update_status(__status, base_str, "Creating new arm")
+            manual_generator_run = create_generator_run(arm_params, new_trial_idx, new_job_type)
             trial._generator_run = manual_generator_run
             fool_linter(trial._generator_run)
 
-            __insert_job_into_ax_client__complete_trial_if_result(new_trial_idx, result, __status, base_str)
+            complete_trial_if_result(new_trial_idx, result, __status, base_str)
             done_converting = True
 
-            __insert_job_into_ax_client__save_results_if_needed(__status, base_str)
+            save_results_if_needed(__status, base_str)
             return True
 
         except ax.exceptions.core.UnsupportedError as e:
-            if not __insert_job_into_ax_client__handle_type_error(e, arm_params):
+            if not handle_insert_job_error(e, arm_params):
                 break
 
     return False
@@ -6975,7 +7562,7 @@ def get_parameters_from_outfile(stdout_path: str) -> Union[None, dict, str]:
         if not args.tests:
             original_print(f"get_parameters_from_outfile: The file '{stdout_path}' was not found.")
     except Exception as e:
-        print(f"get_parameters_from_outfile: There was an error: {e}")
+        print_red(f"get_parameters_from_outfile: There was an error: {e}")
 
     return None
 
@@ -6993,7 +7580,7 @@ def get_hostname_from_outfile(stdout_path: Optional[str]) -> Optional[str]:
         original_print(f"The file '{stdout_path}' was not found.")
         return None
     except Exception as e:
-        print(f"There was an error: {e}")
+        print_red(f"There was an error: {e}")
         return None
 
 def add_to_global_error_list(msg: str) -> None:
@@ -7029,22 +7616,68 @@ def mark_trial_as_failed(trial_index: int, _trial: Any) -> None:
 
             return None
 
-        ax_client.log_trial_failure(trial_index=trial_index)
+        log_ax_client_trial_failure(trial_index)
         _trial.mark_failed(unsafe=True)
     except ValueError as e:
         print_debug(f"mark_trial_as_failed error: {e}")
 
     return None
 
-def _finish_job_core_helper_check_valid_result(result: Union[None, list, int, float, tuple]) -> bool:
+def check_valid_result(result: Union[None, dict]) -> bool:
     possible_val_not_found_values = [
         VAL_IF_NOTHING_FOUND,
         -VAL_IF_NOTHING_FOUND,
         -99999999999999997168788049560464200849936328366177157906432,
         99999999999999997168788049560464200849936328366177157906432
     ]
-    values_to_check = result if isinstance(result, list) else [result]
-    return result is not None and all(r not in possible_val_not_found_values for r in values_to_check)
+
+    def flatten_values(obj: Any) -> Any:
+        values = []
+        try:
+            if isinstance(obj, dict):
+                for v in obj.values():
+                    values.extend(flatten_values(v))
+            elif isinstance(obj, (list, tuple, set)):
+                for v in obj:
+                    values.extend(flatten_values(v))
+            else:
+                values.append(obj)
+        except Exception as e:
+            print_red(f"Error while flattening values: {e}")
+        return values
+
+    if result is None:
+        return False
+
+    try:
+        all_values = flatten_values(result)
+        for val in all_values:
+            if val in possible_val_not_found_values:
+                return False
+        return True
+    except Exception as e:
+        print_red(f"Error while checking result validity: {e}")
+        return False
+
+def update_ax_client_trial(trial_idx: int, result: Union[list, dict]) -> None:
+    if not ax_client:
+        my_exit(101)
+
+        return None
+
+    ax_client.update_trial_data(trial_index=trial_idx, raw_data=result)
+
+    return None
+
+def complete_ax_client_trial(trial_idx: int, result: Union[list, dict]) -> None:
+    if not ax_client:
+        my_exit(101)
+
+        return None
+
+    ax_client.complete_trial(trial_index=trial_idx, raw_data=result)
+
+    return None
 
 def _finish_job_core_helper_complete_trial(trial_index: int, raw_result: dict) -> None:
     if ax_client is None:
@@ -7053,25 +7686,64 @@ def _finish_job_core_helper_complete_trial(trial_index: int, raw_result: dict) -
 
     try:
         print_debug(f"Completing trial: {trial_index} with result: {raw_result}...")
-        ax_client.complete_trial(trial_index=trial_index, raw_data=raw_result)
+        complete_ax_client_trial(trial_index, raw_result)
         print_debug(f"Completing trial: {trial_index} with result: {raw_result}... Done!")
     except ax.exceptions.core.UnsupportedError as e:
         if f"{e}":
             print_debug(f"Completing trial: {trial_index} with result: {raw_result} after failure. Trying to update trial...")
-            ax_client.update_trial_data(trial_index=trial_index, raw_data=raw_result)
+            update_ax_client_trial(trial_index, raw_result)
             print_debug(f"Completing trial: {trial_index} with result: {raw_result} after failure... Done!")
         else:
             _fatal_error(f"Error completing trial: {e}", 234)
 
     return None
 
-def _finish_job_core_helper_mark_success(_trial: ax.core.trial.Trial, result: Union[float, int, tuple]) -> None:
+def format_result_for_display(result: dict) -> str:
+    def safe_float(v: Any) -> str:
+        try:
+            if v is None:
+                return "None"
+            if isinstance(v, (int, float)):
+                if math.isnan(v):
+                    return "NaN"
+                if math.isinf(v):
+                    return "∞" if v > 0 else "-∞"
+                return f"{v:.6f}"
+            return str(v)
+        except Exception as e:
+            return f"<error: {e}>"
+
+    try:
+        if not isinstance(result, dict):
+            return safe_float(result)
+
+        parts = []
+        for key, val in result.items():
+            try:
+                if isinstance(val, (list, tuple)) and len(val) == 2:
+                    main, sem = val
+                    main_str = safe_float(main)
+                    if sem is not None:
+                        sem_str = safe_float(sem)
+                        parts.append(f"{key}: {main_str} (SEM: {sem_str})")
+                    else:
+                        parts.append(f"{key}: {main_str}")
+                else:
+                    parts.append(f"{key}: {safe_float(val)}")
+            except Exception as e:
+                parts.append(f"{key}: <error: {e}>")
+
+        return ", ".join(parts)
+    except Exception as e:
+        return f"<error formatting result: {e}>"
+
+def _finish_job_core_helper_mark_success(_trial: ax.core.trial.Trial, result: dict) -> None:
     print_debug(f"Marking trial {_trial} as completed")
     _trial.mark_completed(unsafe=True)
 
     succeeded_jobs(1)
 
-    progressbar_description(f"new result: {result}")
+    progressbar_description(f"new result: {format_result_for_display(result)}")
     update_progress_bar(1)
 
     save_results_csv()
@@ -7085,8 +7757,8 @@ def _finish_job_core_helper_mark_failure(job: Any, trial_index: int, _trial: Any
     if job:
         try:
             progressbar_description("job_failed")
-            ax_client.log_trial_failure(trial_index=trial_index)
-            _trial.mark_failed(unsafe=True)
+            log_ax_client_trial_failure(trial_index)
+            mark_trial_as_failed(trial_index, _trial)
         except Exception as e:
             print_red(f"\nERROR while trying to mark job as failure: {e}")
         job.cancel()
@@ -7103,16 +7775,16 @@ def finish_job_core(job: Any, trial_index: int, this_jobs_finished: int) -> int:
     result = job.result()
     print_debug(f"finish_job_core: trial-index: {trial_index}, job.result(): {result}, state: {state_from_job(job)}")
 
-    raw_result = result
-    result_keys = list(result.keys())
-    result = result[result_keys[0]]
     this_jobs_finished += 1
 
     if ax_client:
-        _trial = ax_client.get_trial(trial_index)
+        _trial = get_ax_client_trial(trial_index)
+
+        if _trial is None:
+            return 0
 
-        if _finish_job_core_helper_check_valid_result(result):
-            _finish_job_core_helper_complete_trial(trial_index, raw_result)
+        if check_valid_result(result):
+            _finish_job_core_helper_complete_trial(trial_index, result)
 
             try:
                 _finish_job_core_helper_mark_success(_trial, result)
@@ -7120,15 +7792,17 @@ def finish_job_core(job: Any, trial_index: int, this_jobs_finished: int) -> int:
                 if len(arg_result_names) > 1 and count_done_jobs() > 1 and not job_calculate_pareto_front(get_current_run_folder(), True):
                     print_red("job_calculate_pareto_front post job failed")
             except Exception as e:
-                print(f"ERROR in line {get_line_info()}: {e}")
+                print_red(f"ERROR in line {get_line_info()}: {e}")
         else:
             _finish_job_core_helper_mark_failure(job, trial_index, _trial)
     else:
-        _fatal_error("ax_client could not be found or used", 9)
+        _fatal_error("ax_client could not be found or used", 101)
 
     print_debug(f"finish_job_core: removing job {job}, trial_index: {trial_index}")
     global_vars["jobs"].remove((job, trial_index))
 
+    log_data()
+
     force_live_share()
 
     return this_jobs_finished
@@ -7141,11 +7815,14 @@ def _finish_previous_jobs_helper_handle_failed_job(job: Any, trial_index: int) -
     if job:
         try:
             progressbar_description("job_failed")
-            _trial = ax_client.get_trial(trial_index)
-            ax_client.log_trial_failure(trial_index=trial_index)
+            _trial = get_ax_client_trial(trial_index)
+            if _trial is None:
+                return None
+
+            log_ax_client_trial_failure(trial_index)
             mark_trial_as_failed(trial_index, _trial)
         except Exception as e:
-            print(f"ERROR in line {get_line_info()}: {e}")
+            print_debug(f"ERROR in line {get_line_info()}: {e}")
         job.cancel()
         orchestrate_job(job, trial_index)
 
@@ -7160,10 +7837,12 @@ def _finish_previous_jobs_helper_handle_failed_job(job: Any, trial_index: int) -
 
 def _finish_previous_jobs_helper_handle_exception(job: Any, trial_index: int, error: Exception) -> int:
     if "None for metric" in str(error):
-        print_red(
-            f"\n⚠ It seems like the program that was about to be run didn't have 'RESULT: <FLOAT>' in it's output string."
-            f"\nError: {error}\nJob-result: {job.result()}"
-        )
+        err_msg = f"\n⚠ It seems like the program that was about to be run didn't have 'RESULT: <FLOAT>' in it's output string.\nError: {error}\nJob-result: {job.result()}"
+
+        if count_done_jobs() == 0:
+            print_red(err_msg)
+        else:
+            print_debug(err_msg)
     else:
         print_red(f"\n⚠ {error}")
 
@@ -7182,7 +7861,10 @@ def _finish_previous_jobs_helper_process_job(job: Any, trial_index: int, this_jo
         this_jobs_finished += _finish_previous_jobs_helper_handle_exception(job, trial_index, error)
     return this_jobs_finished
 
-def _finish_previous_jobs_helper_check_and_process(job: Any, trial_index: int, this_jobs_finished: int) -> int:
+def _finish_previous_jobs_helper_check_and_process(__args: Tuple[Any, int]) -> int:
+    job, trial_index = __args
+
+    this_jobs_finished = 0
     if job is None:
         print_debug(f"finish_previous_jobs: job {job} is None")
         return this_jobs_finished
@@ -7195,10 +7877,6 @@ def _finish_previous_jobs_helper_check_and_process(job: Any, trial_index: int, t
 
     return this_jobs_finished
 
-def _finish_previous_jobs_helper_wrapper(__args: Tuple[Any, int]) -> int:
-    job, trial_index = __args
-    return _finish_previous_jobs_helper_check_and_process(job, trial_index, 0)
-
 def finish_previous_jobs(new_msgs: List[str] = []) -> None:
     global JOBS_FINISHED
 
@@ -7211,13 +7889,10 @@ def finish_previous_jobs(new_msgs: List[str] = []) -> None:
 
     jobs_copy = global_vars["jobs"][:]
 
-    if len(jobs_copy) > 0:
-        print_debug(f"jobs in finish_previous_jobs: {jobs_copy}")
-
-    finishing_jobs_start_time = time.time()
+    #finishing_jobs_start_time = time.time()
 
     with ThreadPoolExecutor() as finish_job_executor:
-        futures = [finish_job_executor.submit(_finish_previous_jobs_helper_wrapper, (job, trial_index)) for job, trial_index in jobs_copy]
+        futures = [finish_job_executor.submit(_finish_previous_jobs_helper_check_and_process, (job, trial_index)) for job, trial_index in jobs_copy]
 
         for future in as_completed(futures):
             try:
@@ -7225,11 +7900,11 @@ def finish_previous_jobs(new_msgs: List[str] = []) -> None:
             except Exception as e:
                 print_red(f"⚠ Exception in parallel job handling: {e}")
 
-    finishing_jobs_end_time = time.time()
+    #finishing_jobs_end_time = time.time()
 
-    finishing_jobs_runtime = finishing_jobs_end_time - finishing_jobs_start_time
+    #finishing_jobs_runtime = finishing_jobs_end_time - finishing_jobs_start_time
 
-    print_debug(f"Finishing jobs took {finishing_jobs_runtime} second(s)")
+    #print_debug(f"Finishing jobs took {finishing_jobs_runtime} second(s)")
 
     if this_jobs_finished > 0:
         save_results_csv()
@@ -7376,11 +8051,15 @@ def is_already_in_defective_nodes(hostname: str) -> bool:
                     return True
     except Exception as e:
         print_red(f"is_already_in_defective_nodes: Error reading the file {file_path}: {e}")
-        return False
 
     return False
 
 def submit_new_job(parameters: Union[dict, str], trial_index: int) -> Any:
+    if submitit_executor is None:
+        print_red("submit_new_job: submitit_executor was None")
+
+        return None
+
     print_debug(f"Submitting new job for trial_index {trial_index}, parameters {parameters}")
 
     start = time.time()
@@ -7391,25 +8070,49 @@ def submit_new_job(parameters: Union[dict, str], trial_index: int) -> Any:
 
     print_debug(f"Done submitting new job, took {elapsed} seconds")
 
+    log_data()
+
     return new_job
 
+def get_ax_client_trial(trial_index: int) -> Optional[ax.core.trial.Trial]:
+    if not ax_client:
+        my_exit(101)
+
+        return None
+
+    try:
+        log_data()
+
+        return ax_client.get_trial(trial_index)
+    except KeyError:
+        error_without_print(f"get_ax_client_trial: trial_index {trial_index} failed")
+        return None
+
 def orchestrator_start_trial(parameters: Union[dict, str], trial_index: int) -> None:
     if submitit_executor and ax_client:
         new_job = submit_new_job(parameters, trial_index)
-        submitted_jobs(1)
+        if new_job:
+            submitted_jobs(1)
 
-        _trial = ax_client.get_trial(trial_index)
+            _trial = get_ax_client_trial(trial_index)
 
-        try:
-            _trial.mark_staged(unsafe=True)
-        except Exception as e:
-            print_debug(f"orchestrator_start_trial: error {e}")
-        _trial.mark_running(unsafe=True, no_runner_required=True)
+            if _trial is not None:
+                try:
+                    _trial.mark_staged(unsafe=True)
+                except Exception as e:
+                    print_debug(f"orchestrator_start_trial: error {e}")
+                _trial.mark_running(unsafe=True, no_runner_required=True)
 
-        print_debug(f"orchestrator_start_trial: appending job {new_job} to global_vars['jobs'], trial_index: {trial_index}")
-        global_vars["jobs"].append((new_job, trial_index))
+                print_debug(f"orchestrator_start_trial: appending job {new_job} to global_vars['jobs'], trial_index: {trial_index}")
+                global_vars["jobs"].append((new_job, trial_index))
+            else:
+                print_red("Trial was none in orchestrator_start_trial")
+        else:
+            print_red("orchestrator_start_trial: Failed to start new job")
+    elif ax_client:
+        _fatal_error("submitit_executor could not be found properly", 9)
     else:
-        _fatal_error("submitit_executor or ax_client could not be found properly", 9)
+        _fatal_error("ax_client could not be found properly", 101)
 
 def handle_exclude_node(stdout_path: str, hostname_from_out_file: Union[None, str]) -> None:
     stdout_path = check_alternate_path(stdout_path)
@@ -7428,7 +8131,7 @@ def handle_restart(stdout_path: str, trial_index: int) -> None:
     if parameters:
         orchestrator_start_trial(parameters, trial_index)
     else:
-        print(f"Could not determine parameters from outfile {stdout_path} for restarting job")
+        print_red(f"Could not determine parameters from outfile {stdout_path} for restarting job")
 
 def check_alternate_path(path: str) -> str:
     if os.path.exists(path):
@@ -7515,7 +8218,7 @@ def execute_evaluation(_params: list) -> Optional[int]:
     print_debug(f"execute_evaluation({_params})")
     trial_index, parameters, trial_counter, phase = _params
     if not ax_client:
-        _fatal_error("Failed to get ax_client", 9)
+        _fatal_error("Failed to get ax_client", 101)
 
         return None
 
@@ -7524,7 +8227,11 @@ def execute_evaluation(_params: list) -> Optional[int]:
 
         return None
 
-    _trial = ax_client.get_trial(trial_index)
+    _trial = get_ax_client_trial(trial_index)
+
+    if _trial is None:
+        error_without_print(f"execute_evaluation: _trial was not in execute_evaluation for params {_params}")
+        return None
 
     def mark_trial_stage(stage: str, error_msg: str) -> None:
         try:
@@ -7539,15 +8246,18 @@ def execute_evaluation(_params: list) -> Optional[int]:
     try:
         initialize_job_environment()
         new_job = submit_new_job(parameters, trial_index)
-        submitted_jobs(1)
+        if new_job:
+            submitted_jobs(1)
 
-        print_debug(f"execute_evaluation: appending job {new_job} to global_vars['jobs'], trial_index: {trial_index}")
-        global_vars["jobs"].append((new_job, trial_index))
+            print_debug(f"execute_evaluation: appending job {new_job} to global_vars['jobs'], trial_index: {trial_index}")
+            global_vars["jobs"].append((new_job, trial_index))
 
-        mark_trial_stage("mark_running", "Marking the trial as running failed")
-        trial_counter += 1
+            mark_trial_stage("mark_running", "Marking the trial as running failed")
+            trial_counter += 1
 
-        progressbar_description("started new job")
+            progressbar_description("started new job")
+        else:
+            progressbar_description("Failed to start new job")
     except submitit.core.utils.FailedJobError as error:
         handle_failed_job(error, trial_index, new_job)
         trial_counter += 1
@@ -7589,7 +8299,7 @@ def handle_failed_job(error: Union[None, Exception, str], trial_index: int, new_
             my_exit(144)
 
     if new_job is None:
-        print_red("handle_failed_job: job is None")
+        print_debug("handle_failed_job: job is None")
 
         return None
 
@@ -7600,16 +8310,24 @@ def handle_failed_job(error: Union[None, Exception, str], trial_index: int, new_
 
     return None
 
+def log_ax_client_trial_failure(trial_index: int) -> None:
+    if not ax_client:
+        my_exit(101)
+
+        return
+
+    ax_client.log_trial_failure(trial_index=trial_index)
+
 def cancel_failed_job(trial_index: int, new_job: Job) -> None:
     print_debug("Trying to cancel job that failed")
     if new_job:
         try:
             if ax_client:
-                ax_client.log_trial_failure(trial_index=trial_index)
+                log_ax_client_trial_failure(trial_index)
             else:
                 _fatal_error("ax_client not defined", 101)
         except Exception as e:
-            print(f"ERROR in line {get_line_info()}: {e}")
+            print_red(f"ERROR in line {get_line_info()}: {e}")
         new_job.cancel()
 
         print_debug(f"cancel_failed_job: removing job {new_job}, trial_index: {trial_index}")
@@ -7645,10 +8363,12 @@ def show_debug_table_for_break_run_search(_name: str, _max_eval: Optional[int])
         ("failed_jobs()", failed_jobs()),
         ("count_done_jobs()", count_done_jobs()),
         ("_max_eval", _max_eval),
-        ("progress_bar.total", progress_bar.total),
         ("NR_INSERTED_JOBS", NR_INSERTED_JOBS)
     ]
 
+    if progress_bar is not None:
+        rows.append(("progress_bar.total", progress_bar.total))
+
     for row in rows:
         table.add_row(str(row[0]), str(row[1]))
 
@@ -7661,6 +8381,8 @@ def break_run_search(_name: str, _max_eval: Optional[int]) -> bool:
     _submitted_jobs = submitted_jobs()
     _failed_jobs = failed_jobs()
 
+    log_data()
+
     max_failed_jobs = max_eval
 
     if args.max_failed_jobs is not None and args.max_failed_jobs > 0:
@@ -7688,11 +8410,11 @@ def break_run_search(_name: str, _max_eval: Optional[int]) -> bool:
             _ret = True
 
     if args.verbose_break_run_search_table:
-        show_debug_table_for_break_run_search(_name, _max_eval, _ret)
+        show_debug_table_for_break_run_search(_name, _max_eval)
 
     return _ret
 
-def _calculate_nr_of_jobs_to_get(simulated_jobs: int, currently_running_jobs: int) -> int:
+def calculate_nr_of_jobs_to_get(simulated_jobs: int, currently_running_jobs: int) -> int:
     """Calculates the number of jobs to retrieve."""
     return min(
         max_eval + simulated_jobs - count_done_jobs(),
@@ -7705,7 +8427,7 @@ def remove_extra_spaces(text: str) -> str:
         raise ValueError("Input must be a string")
     return re.sub(r'\s+', ' ', text).strip()
 
-def _get_trials_message(nr_of_jobs_to_get: int, full_nr_of_jobs_to_get: int, trial_durations: List[float]) -> str:
+def get_trials_message(nr_of_jobs_to_get: int, full_nr_of_jobs_to_get: int, trial_durations: List[float]) -> str:
     """Generates the appropriate message for the number of trials being retrieved."""
     ret = ""
     if full_nr_of_jobs_to_get > 1:
@@ -7790,51 +8512,51 @@ def get_batched_arms(nr_of_jobs_to_get: int) -> list:
         print_red("get_batched_arms: ax_client was None")
         return []
 
-    # Experiment-Status laden
     load_experiment_state()
 
-    while len(batched_arms) != nr_of_jobs_to_get:
+    while len(batched_arms) < nr_of_jobs_to_get:
         if attempts > args.max_attempts_for_generation:
             print_debug(f"get_batched_arms: Stopped after {attempts} attempts: could not generate enough arms "
                         f"(got {len(batched_arms)} out of {nr_of_jobs_to_get}).")
             break
 
-        remaining = nr_of_jobs_to_get - len(batched_arms)
-        print_debug(f"get_batched_arms: Attempt {attempts + 1}: requesting {remaining} more arm(s).")
-
-        print_debug("get pending observations")
-        t0 = time.time()
-        pending_observations = get_pending_observation_features(experiment=ax_client.experiment)
-        dt = time.time() - t0
-        print_debug(f"got pending observations: {pending_observations} (took {dt:.2f} seconds)")
+        #print_debug(f"get_batched_arms: Attempt {attempts + 1}: requesting 1 more arm")
 
-        print_debug("getting global_gs.gen()")
-        batched_generator_run = global_gs.gen(
+        pending_observations = get_pending_observation_features(
             experiment=ax_client.experiment,
-            n=remaining,
-            pending_observations=pending_observations
+            include_out_of_design_points=True
         )
-        print_debug(f"got global_gs.gen(): {batched_generator_run}")
+
+        try:
+            #print_debug("getting global_gs.gen() with n=1")
+            batched_generator_run: Any = global_gs.gen(
+                experiment=ax_client.experiment,
+                n=1,
+                pending_observations=pending_observations,
+            )
+            print_debug(f"got global_gs.gen(): {batched_generator_run}")
+        except Exception as e:
+            print_debug(f"global_gs.gen failed: {e}")
+            traceback.print_exception(type(e), e, e.__traceback__, file=sys.stderr)
+            break
 
         depth = 0
         path = "batched_generator_run"
-        while isinstance(batched_generator_run, (list, tuple)) and len(batched_generator_run) > 0:
-            print_debug(f"Depth {depth}, path {path}, type {type(batched_generator_run).__name__}, length {len(batched_generator_run)}: {batched_generator_run}")
+        while isinstance(batched_generator_run, (list, tuple)) and len(batched_generator_run) == 1:
+            #print_debug(f"Depth {depth}, path {path}, type {type(batched_generator_run).__name__}, length {len(batched_generator_run)}: {batched_generator_run}")
             batched_generator_run = batched_generator_run[0]
             path += "[0]"
             depth += 1
 
-        print_debug(f"Final flat object at depth {depth}, path {path}: {batched_generator_run} (type {type(batched_generator_run).__name__})")
+        #print_debug(f"Final flat object at depth {depth}, path {path}: {batched_generator_run} (type {type(batched_generator_run).__name__})")
 
-        print_debug("got new arms")
-        new_arms = batched_generator_run.arms
-        print_debug(f"new_arms: {new_arms}")
+        new_arms = getattr(batched_generator_run, "arms", [])
         if not new_arms:
             print_debug("get_batched_arms: No new arms were generated in this attempt.")
         else:
-            print_debug(f"get_batched_arms: Generated {len(new_arms)} new arm(s), wanted {nr_of_jobs_to_get}.")
+            print_debug(f"get_batched_arms: Generated {len(new_arms)} new arm(s), now at {len(batched_arms) + len(new_arms)} of {nr_of_jobs_to_get}.")
+            batched_arms.extend(new_arms)
 
-        batched_arms.extend(new_arms)
         attempts += 1
 
     print_debug(f"get_batched_arms: Finished with {len(batched_arms)} arm(s) after {attempts} attempt(s).")
@@ -7845,7 +8567,7 @@ def fetch_next_trials(nr_of_jobs_to_get: int, recursion: bool = False) -> Tuple[
     die_101_if_no_ax_client_or_experiment_or_gs()
 
     if not ax_client:
-        _fatal_error("ax_client was not defined", 9)
+        _fatal_error("ax_client was not defined", 101)
 
     if global_gs is None:
         _fatal_error("Global generation strategy is not set. This is a bug in OmniOpt2.", 107)
@@ -7873,16 +8595,14 @@ def generate_trials(n: int, recursion: bool) -> Tuple[Dict[int, Any], bool]:
                     retries += 1
                     continue
 
-                print_debug(f"Fetching trial {cnt + 1}/{n}...")
-                progressbar_description(_get_trials_message(cnt + 1, n, trial_durations))
+                progressbar_description(get_trials_message(cnt + 1, n, trial_durations))
 
                 try:
                     result = create_and_handle_trial(arm)
                     if result is not None:
                         trial_index, trial_duration, trial_successful = result
-
                 except TrialRejected as e:
-                    print_debug(f"Trial rejected: {e}")
+                    print_debug(f"generate_trials: Trial rejected, error: {e}")
                     retries += 1
                     continue
 
@@ -7892,14 +8612,23 @@ def generate_trials(n: int, recursion: bool) -> Tuple[Dict[int, Any], bool]:
                     cnt += 1
                     trials_dict[trial_index] = arm.parameters
 
-        return _finalize_generation(trials_dict, cnt, n, start_time)
+        finalized = finalize_generation(trials_dict, cnt, n, start_time)
+
+        return finalized
 
     except Exception as e:
-        return _handle_generation_failure(e, n, recursion)
+        return handle_generation_failure(e, n, recursion)
 
 class TrialRejected(Exception):
     pass
 
+def mark_abandoned(trial: Any, reason: str, trial_index: int) -> None:
+    try:
+        print_debug(f"[INFO] Marking trial {trial.index} ({trial.arm.name}) as abandoned, trial-index: {trial_index}. Reason: {reason}")
+        trial.mark_abandoned(reason)
+    except Exception as e:
+        print_red(f"[ERROR] Could not mark trial as abandoned: {e}")
+
 def create_and_handle_trial(arm: Any) -> Optional[Tuple[int, float, bool]]:
     if ax_client is None:
         print_red("ax_client is None in create_and_handle_trial")
@@ -7925,7 +8654,7 @@ def create_and_handle_trial(arm: Any) -> Optional[Tuple[int, float, bool]]:
     arm = trial.arms[0]
     if deduplicated_arm(arm):
         print_debug(f"Duplicated arm: {arm}")
-        trial.mark_abandoned(reason="Duplication detected")
+        mark_abandoned(trial, "Duplication detected", trial_index)
         raise TrialRejected("Duplicate arm.")
 
     arms_by_name_for_deduplication[arm.name] = arm
@@ -7934,7 +8663,7 @@ def create_and_handle_trial(arm: Any) -> Optional[Tuple[int, float, bool]]:
 
     if not has_no_post_generation_constraints_or_matches_constraints(post_generation_constraints, params):
         print_debug(f"Trial {trial_index} does not meet post-generation constraints. Marking abandoned. Params: {params}, constraints: {post_generation_constraints}")
-        trial.mark_abandoned(reason="Post-Generation-Constraint failed")
+        mark_abandoned(trial, "Post-Generation-Constraint failed", trial_index)
         abandoned_trial_indices.append(trial_index)
         raise TrialRejected("Post-generation constraints not met.")
 
@@ -7942,7 +8671,7 @@ def create_and_handle_trial(arm: Any) -> Optional[Tuple[int, float, bool]]:
     end = time.time()
     return trial_index, float(end - start), True
 
-def _finalize_generation(trials_dict: Dict[int, Any], cnt: int, requested: int, start_time: float) -> Tuple[Dict[int, Any], bool]:
+def finalize_generation(trials_dict: Dict[int, Any], cnt: int, requested: int, start_time: float) -> Tuple[Dict[int, Any], bool]:
     total_time = time.time() - start_time
 
     log_gen_times.append(total_time)
@@ -7953,7 +8682,7 @@ def _finalize_generation(trials_dict: Dict[int, Any], cnt: int, requested: int,
 
     return trials_dict, False
 
-def _handle_generation_failure(
+def handle_generation_failure(
     e: Exception,
     requested: int,
     recursion: bool
@@ -7969,19 +8698,19 @@ def _handle_generation_failure(
     )):
         msg = str(e)
         if msg not in error_8_saved:
-            _print_exhaustion_warning(e, recursion)
+            print_exhaustion_warning(e, recursion)
             error_8_saved.append(msg)
 
         if not recursion and args.revert_to_random_when_seemingly_exhausted:
             print_debug("Switching to random search strategy.")
-            set_global_gs_to_random()
+            set_global_gs_to_sobol()
             return fetch_next_trials(requested, True)
 
-    print_red(f"_handle_generation_failure: General Exception: {e}")
+    print_red(f"handle_generation_failure: General Exception: {e}")
 
     return {}, True
 
-def _print_exhaustion_warning(e: Exception, recursion: bool) -> None:
+def print_exhaustion_warning(e: Exception, recursion: bool) -> None:
     if not recursion and args.revert_to_random_when_seemingly_exhausted:
         print_yellow(f"\n⚠Error 8: {e} From now (done jobs: {count_done_jobs()}) on, random points will be generated.")
     else:
@@ -8026,21 +8755,24 @@ def get_model_gen_kwargs() -> dict:
         "fit_out_of_design": args.fit_out_of_design
     }
 
-def set_global_gs_to_random() -> None:
+def set_global_gs_to_sobol() -> None:
     global global_gs
     global overwritten_to_random
 
+    print("Reverting to SOBOL")
+
     global_gs = GenerationStrategy(
         name="Random*",
         nodes=[
             GenerationNode(
                 node_name="Sobol",
-                generator_specs=[
-                    GeneratorSpec(
-                        Models.SOBOL,
-                        model_gen_kwargs=get_model_gen_kwargs()
-                    )
-                ]
+                should_deduplicate=True,
+                generator_specs=[ # type: ignore[arg-type]
+                    GeneratorSpec( # type: ignore[arg-type]
+                        Models.SOBOL, # type: ignore[arg-type]
+                        model_gen_kwargs=get_model_gen_kwargs() # type: ignore[arg-type]
+                    ) # type: ignore[arg-type]
+                ] # type: ignore[arg-type]
             )
         ]
     )
@@ -8400,16 +9132,21 @@ def get_model_from_name(name: str) -> Any:
             return gen
     raise ValueError(f"Unknown or unsupported model: {name}")
 
-def get_name_from_model(model) -> Optional[str]:
+def get_name_from_model(model: Any) -> str:
     if not isinstance(SUPPORTED_MODELS, (list, set, tuple)):
-        return None
+        raise RuntimeError("get_model_from_name: SUPPORTED_MODELS was not a list, set or tuple. Cannot continue")
 
     model_str = model.value if hasattr(model, "value") else str(model)
 
     model_str_lower = model_str.lower()
     model_map = {m.lower(): m for m in SUPPORTED_MODELS}
 
-    return model_map.get(model_str_lower, None)
+    ret = model_map.get(model_str_lower, None)
+
+    if ret is None:
+        raise RuntimeError("get_name_from_model: failed to get Model")
+
+    return ret
 
 def parse_generation_strategy_string(gen_strat_str: str) -> tuple[list[dict[str, int]], int]:
     gen_strat_list = []
@@ -8420,10 +9157,10 @@ def parse_generation_strategy_string(gen_strat_str: str) -> tuple[list[dict[str,
 
     for s in splitted_by_comma:
         if "=" not in s:
-            print(f"'{s}' does not contain '='")
+            print_red(f"'{s}' does not contain '='")
             my_exit(123)
         if s.count("=") != 1:
-            print(f"There can only be one '=' in the gen_strat_str's element '{s}'")
+            print_red(f"There can only be one '=' in the gen_strat_str's element '{s}'")
             my_exit(123)
 
         model_name, nr_str = s.split("=")
@@ -8433,13 +9170,13 @@ def parse_generation_strategy_string(gen_strat_str: str) -> tuple[list[dict[str,
             _fatal_error(f"Model {matching_model} is not valid for custom generation strategy.", 56)
 
         if not matching_model:
-            print(f"'{model_name}' not found in SUPPORTED_MODELS")
+            print_red(f"'{model_name}' not found in SUPPORTED_MODELS")
             my_exit(123)
 
         try:
             nr = int(nr_str)
         except ValueError:
-            print(f"Invalid number of generations '{nr_str}' for model '{model_name}'")
+            print_red(f"Invalid number of generations '{nr_str}' for model '{model_name}'")
             my_exit(123)
 
         gen_strat_list.append({matching_model: nr})
@@ -8596,11 +9333,12 @@ def create_node(model_name: str, threshold: int, next_model_name: Optional[str])
     if model_name.lower() != "sobol":
         kwargs["model_kwargs"] = get_model_kwargs()
 
-    model_spec = [GeneratorSpec(selected_model, **kwargs)]
+    model_spec = [GeneratorSpec(selected_model, **kwargs)] # type: ignore[arg-type]
 
     res = GenerationNode(
         node_name=model_name,
         generator_specs=model_spec,
+        should_deduplicate=True,
         transition_criteria=trans_crit
     )
 
@@ -8611,7 +9349,7 @@ def get_optimizer_kwargs() -> dict:
         "sequential": False
     }
 
-def create_step(model_name: str, _num_trials: int = -1, index: Optional[int] = None) -> GenerationStep:
+def create_step(model_name: str, _num_trials: int, index: int) -> GenerationStep:
     model_enum = get_model_from_name(model_name)
 
     return GenerationStep(
@@ -8626,17 +9364,16 @@ def create_step(model_name: str, _num_trials: int = -1, index: Optional[int] = N
     )
 
 def set_global_generation_strategy() -> None:
-    with spinner("Setting global generation strategy"):
-        continue_not_supported_on_custom_generation_strategy()
+    continue_not_supported_on_custom_generation_strategy()
 
-        try:
-            if args.generation_strategy is None:
-                setup_default_generation_strategy()
-            else:
-                setup_custom_generation_strategy()
-        except Exception as e:
-            print_red(f"Unexpected error in generation strategy setup: {e}")
-            my_exit(111)
+    try:
+        if args.generation_strategy is None:
+            setup_default_generation_strategy()
+        else:
+            setup_custom_generation_strategy()
+    except Exception as e:
+        print_red(f"Unexpected error in generation strategy setup: {e}")
+        my_exit(111)
 
     if global_gs is None:
         print_red("global_gs is None after setup!")
@@ -8787,10 +9524,14 @@ def execute_trials(
                 result = future.result()
                 print_debug(f"result in execute_trials: {result}")
             except Exception as exc:
-                print_red(f"execute_trials: Error at executing a trial: {exc}")
+                failed_args = future_to_args[future]
+                print_red(f"execute_trials: Error at executing a trial with args {failed_args}: {exc}")
+                traceback.print_exc()
 
     end_time = time.time()
 
+    log_data()
+
     duration = float(end_time - start_time)
     job_submit_durations.append(duration)
     job_submit_nrs.append(cnt)
@@ -8824,20 +9565,20 @@ def create_and_execute_next_runs(next_nr_steps: int, phase: Optional[str], _max_
     done_optimizing: bool = False
 
     try:
-        done_optimizing, trial_index_to_param = _create_and_execute_next_runs_run_loop(_max_eval, phase)
-        _create_and_execute_next_runs_finish(done_optimizing)
+        done_optimizing, trial_index_to_param = create_and_execute_next_runs_run_loop(_max_eval, phase)
+        create_and_execute_next_runs_finish(done_optimizing)
     except Exception as e:
         stacktrace = traceback.format_exc()
         print_debug(f"Warning: create_and_execute_next_runs encountered an exception: {e}\n{stacktrace}")
         return handle_exceptions_create_and_execute_next_runs(e)
 
-    return _create_and_execute_next_runs_return_value(trial_index_to_param)
+    return create_and_execute_next_runs_return_value(trial_index_to_param)
 
-def _create_and_execute_next_runs_run_loop(_max_eval: Optional[int], phase: Optional[str]) -> Tuple[bool, Optional[Dict]]:
+def create_and_execute_next_runs_run_loop(_max_eval: Optional[int], phase: Optional[str]) -> Tuple[bool, Optional[Dict]]:
     done_optimizing = False
     trial_index_to_param: Optional[Dict] = None
 
-    nr_of_jobs_to_get = _calculate_nr_of_jobs_to_get(get_nr_of_imported_jobs(), len(global_vars["jobs"]))
+    nr_of_jobs_to_get = calculate_nr_of_jobs_to_get(get_nr_of_imported_jobs(), len(global_vars["jobs"]))
 
     __max_eval = _max_eval if _max_eval is not None else 0
     new_nr_of_jobs_to_get = min(__max_eval - (submitted_jobs() - failed_jobs()), nr_of_jobs_to_get)
@@ -8851,6 +9592,7 @@ def _create_and_execute_next_runs_run_loop(_max_eval: Optional[int], phase: Opti
 
     for _ in range(range_nr):
         trial_index_to_param, done_optimizing = get_next_trials(get_next_trials_nr)
+        log_data()
         if done_optimizing:
             continue
 
@@ -8875,13 +9617,13 @@ def _create_and_execute_next_runs_run_loop(_max_eval: Optional[int], phase: Opti
 
     return done_optimizing, trial_index_to_param
 
-def _create_and_execute_next_runs_finish(done_optimizing: bool) -> None:
+def create_and_execute_next_runs_finish(done_optimizing: bool) -> None:
     finish_previous_jobs(["finishing jobs"])
 
     if done_optimizing:
         end_program(False, 0)
 
-def _create_and_execute_next_runs_return_value(trial_index_to_param: Optional[Dict]) -> int:
+def create_and_execute_next_runs_return_value(trial_index_to_param: Optional[Dict]) -> int:
     try:
         if trial_index_to_param:
             res = len(trial_index_to_param.keys())
@@ -8992,7 +9734,7 @@ def execute_nvidia_smi() -> None:
                 if not host:
                     print_debug("host not defined")
         except Exception as e:
-            print(f"execute_nvidia_smi: An error occurred: {e}")
+            print_red(f"execute_nvidia_smi: An error occurred: {e}")
         if is_slurm_job() and not args.force_local_execution:
             _sleep(30)
 
@@ -9030,11 +9772,6 @@ def run_search() -> bool:
 
     return False
 
-async def start_logging_daemon() -> None:
-    while True:
-        log_data()
-        time.sleep(30)
-
 def should_break_search() -> bool:
     ret = False
 
@@ -9083,10 +9820,8 @@ def check_search_space_exhaustion(nr_of_items: int) -> bool:
         print_debug(_wrn)
         progressbar_description(_wrn)
 
-        live_share()
         return True
 
-    live_share()
     return False
 
 def finalize_jobs() -> None:
@@ -9100,7 +9835,7 @@ def finalize_jobs() -> None:
         handle_slurm_execution()
 
 def go_through_jobs_that_are_not_completed_yet() -> None:
-    print_debug(f"Waiting for jobs to finish (currently, len(global_vars['jobs']) = {len(global_vars['jobs'])}")
+    #print_debug(f"Waiting for jobs to finish (currently, len(global_vars['jobs']) = {len(global_vars['jobs'])}")
 
     nr_jobs_left = len(global_vars['jobs'])
     if nr_jobs_left == 1:
@@ -9170,7 +9905,7 @@ def parse_orchestrator_file(_f: str, _test: bool = False) -> Union[dict, None]:
 
                 return data
             except Exception as e:
-                print(f"Error while parse_experiment_parameters({_f}): {e}")
+                print_red(f"Error while parse_experiment_parameters({_f}): {e}")
     else:
         print_red(f"{_f} could not be found")
 
@@ -9178,364 +9913,52 @@ def parse_orchestrator_file(_f: str, _test: bool = False) -> Union[dict, None]:
 
 def set_orchestrator() -> None:
     with spinner("Setting orchestrator..."):
-        global orchestrator
-
-        if args.orchestrator_file:
-            if SYSTEM_HAS_SBATCH:
-                orchestrator = parse_orchestrator_file(args.orchestrator_file, False)
-            else:
-                print_yellow("--orchestrator_file will be ignored on non-sbatch-systems.")
-
-def check_if_has_random_steps() -> None:
-    if (not args.continue_previous_job and "--continue" not in sys.argv) and (args.num_random_steps == 0 or not args.num_random_steps) and args.model not in ["EXTERNAL_GENERATOR", "SOBOL", "PSEUDORANDOM"]:
-        _fatal_error("You have no random steps set. This is only allowed in continued jobs. To start, you need either some random steps, or a continued run.", 233)
-
-def add_exclude_to_defective_nodes() -> None:
-    with spinner("Adding excluded nodes..."):
-        if args.exclude:
-            entries = [entry.strip() for entry in args.exclude.split(',')]
-
-            for entry in entries:
-                count_defective_nodes(None, entry)
-
-def check_max_eval(_max_eval: int) -> None:
-    with spinner("Checking max_eval..."):
-        if not _max_eval:
-            _fatal_error("--max_eval needs to be set!", 19)
-
-def parse_parameters() -> Any:
-    cli_params_experiment_parameters = None
-    if args.parameter:
-        parse_experiment_parameters()
-        cli_params_experiment_parameters = experiment_parameters
-
-    return cli_params_experiment_parameters
-
-def create_pareto_front_table(idxs: List[int], metric_x: str, metric_y: str) -> Table:
-    table = Table(title=f"Pareto-Front for {metric_y}/{metric_x}:", show_lines=True)
-
-    rows = _pareto_front_table_read_csv()
-    if not rows:
-        table.add_column("No data found")
-        return table
-
-    filtered_rows = _pareto_front_table_filter_rows(rows, idxs)
-    if not filtered_rows:
-        table.add_column("No matching entries")
-        return table
-
-    param_cols, result_cols = _pareto_front_table_get_columns(filtered_rows[0])
-
-    _pareto_front_table_add_headers(table, param_cols, result_cols)
-    _pareto_front_table_add_rows(table, filtered_rows, param_cols, result_cols)
-
-    return table
-
-def _pareto_front_table_read_csv() -> List[Dict[str, str]]:
-    with open(RESULT_CSV_FILE, mode="r", encoding="utf-8", newline="") as f:
-        return list(csv.DictReader(f))
-
-def _pareto_front_table_filter_rows(rows: List[Dict[str, str]], idxs: List[int]) -> List[Dict[str, str]]:
-    result = []
-    for row in rows:
-        try:
-            trial_index = int(row["trial_index"])
-        except (KeyError, ValueError):
-            continue
-
-        if row.get("trial_status", "").strip().upper() == "COMPLETED" and trial_index in idxs:
-            result.append(row)
-    return result
-
-def _pareto_front_table_get_columns(first_row: Dict[str, str]) -> Tuple[List[str], List[str]]:
-    all_columns = list(first_row.keys())
-    ignored_cols = set(special_col_names) - {"trial_index"}
-
-    param_cols = [col for col in all_columns if col not in ignored_cols and col not in arg_result_names and not col.startswith("OO_Info_")]
-    result_cols = [col for col in arg_result_names if col in all_columns]
-    return param_cols, result_cols
-
-def _pareto_front_table_add_headers(table: Table, param_cols: List[str], result_cols: List[str]) -> None:
-    for col in param_cols:
-        table.add_column(col, justify="center")
-    for col in result_cols:
-        table.add_column(Text(f"{col}", style="cyan"), justify="center")
-
-def _pareto_front_table_add_rows(table: Table, rows: List[Dict[str, str]], param_cols: List[str], result_cols: List[str]) -> None:
-    for row in rows:
-        values = [str(helpers.to_int_when_possible(row[col])) for col in param_cols]
-        result_values = [Text(str(helpers.to_int_when_possible(row[col])), style="cyan") for col in result_cols]
-        table.add_row(*values, *result_values, style="bold green")
-
-def pareto_front_as_rich_table(idxs: list, metric_x: str, metric_y: str) -> Optional[Table]:
-    if not os.path.exists(RESULT_CSV_FILE):
-        print_debug(f"pareto_front_as_rich_table: File '{RESULT_CSV_FILE}' not found")
-        return None
-
-    return create_pareto_front_table(idxs, metric_x, metric_y)
-
-def supports_sixel() -> bool:
-    term = os.environ.get("TERM", "").lower()
-    if "xterm" in term or "mlterm" in term:
-        return True
-
-    try:
-        output = subprocess.run(["tput", "setab", "256"], capture_output=True, text=True, check=True)
-        if output.returncode == 0 and "sixel" in output.stdout.lower():
-            return True
-    except (subprocess.CalledProcessError, FileNotFoundError):
-        pass
-
-    return False
-
-def plot_pareto_frontier_sixel(data: Any, x_metric: str, y_metric: str) -> None:
-    if data is None:
-        print("[italic yellow]The data seems to be empty. Cannot plot pareto frontier.[/]")
-        return
-
-    if not supports_sixel():
-        print(f"[italic yellow]Your console does not support sixel-images. Will not print Pareto-frontier as a matplotlib-sixel-plot for {x_metric}/{y_metric}.[/]")
-        return
-
-    import matplotlib.pyplot as plt
-
-    means = data[x_metric][y_metric]["means"]
-
-    x_values = means[x_metric]
-    y_values = means[y_metric]
-
-    fig, _ax = plt.subplots()
-
-    _ax.scatter(x_values, y_values, s=50, marker='x', c='blue', label='Data Points')
-
-    _ax.set_xlabel(x_metric)
-    _ax.set_ylabel(y_metric)
-
-    _ax.set_title(f'Pareto-Front {x_metric}/{y_metric}')
-
-    _ax.ticklabel_format(style='plain', axis='both', useOffset=False)
-
-    with tempfile.NamedTemporaryFile(suffix=".png", delete=True) as tmp_file:
-        plt.savefig(tmp_file.name, dpi=300)
-
-        print_image_to_cli(tmp_file.name, 1000)
-
-    plt.close(fig)
-
-def _pareto_front_general_validate_shapes(x: np.ndarray, y: np.ndarray) -> None:
-    if x.shape != y.shape:
-        raise ValueError("Input arrays x and y must have the same shape.")
-
-def _pareto_front_general_compare(
-    xi: float, yi: float, xj: float, yj: float,
-    x_minimize: bool, y_minimize: bool
-) -> bool:
-    x_better_eq = xj <= xi if x_minimize else xj >= xi
-    y_better_eq = yj <= yi if y_minimize else yj >= yi
-    x_strictly_better = xj < xi if x_minimize else xj > xi
-    y_strictly_better = yj < yi if y_minimize else yj > yi
-
-    return bool(x_better_eq and y_better_eq and (x_strictly_better or y_strictly_better))
-
-def _pareto_front_general_find_dominated(
-    x: np.ndarray, y: np.ndarray, x_minimize: bool, y_minimize: bool
-) -> np.ndarray:
-    num_points = len(x)
-    is_dominated = np.zeros(num_points, dtype=bool)
-
-    for i in range(num_points):
-        for j in range(num_points):
-            if i == j:
-                continue
-
-            if _pareto_front_general_compare(x[i], y[i], x[j], y[j], x_minimize, y_minimize):
-                is_dominated[i] = True
-                break
-
-    return is_dominated
-
-def pareto_front_general(
-    x: np.ndarray,
-    y: np.ndarray,
-    x_minimize: bool = True,
-    y_minimize: bool = True
-) -> np.ndarray:
-    try:
-        _pareto_front_general_validate_shapes(x, y)
-        is_dominated = _pareto_front_general_find_dominated(x, y, x_minimize, y_minimize)
-        return np.where(~is_dominated)[0]
-    except Exception as e:
-        print("Error in pareto_front_general:", str(e))
-        return np.array([], dtype=int)
-
-def _pareto_front_aggregate_data(path_to_calculate: str) -> Optional[Dict[Tuple[int, str], Dict[str, Dict[str, float]]]]:
-    results_csv_file = f"{path_to_calculate}/{RESULTS_CSV_FILENAME}"
-    result_names_file = f"{path_to_calculate}/result_names.txt"
-
-    if not os.path.exists(results_csv_file) or not os.path.exists(result_names_file):
-        return None
-
-    with open(result_names_file, mode="r", encoding="utf-8") as f:
-        result_names = [line.strip() for line in f if line.strip()]
-
-    records: dict = defaultdict(lambda: {'means': {}})
-
-    with open(results_csv_file, encoding="utf-8", mode="r", newline='') as csvfile:
-        reader = csv.DictReader(csvfile)
-        for row in reader:
-            trial_index = int(row['trial_index'])
-            arm_name = row['arm_name']
-            key = (trial_index, arm_name)
-
-            for metric in result_names:
-                if metric in row:
-                    try:
-                        records[key]['means'][metric] = float(row[metric])
-                    except ValueError:
-                        continue
-
-    return records
-
-def _pareto_front_filter_complete_points(
-    path_to_calculate: str,
-    records: Dict[Tuple[int, str], Dict[str, Dict[str, float]]],
-    primary_name: str,
-    secondary_name: str
-) -> List[Tuple[Tuple[int, str], float, float]]:
-    points = []
-    for key, metrics in records.items():
-        means = metrics['means']
-        if primary_name in means and secondary_name in means:
-            x_val = means[primary_name]
-            y_val = means[secondary_name]
-            points.append((key, x_val, y_val))
-    if len(points) == 0:
-        raise ValueError(f"No full data points with both objectives found in {path_to_calculate}.")
-    return points
-
-def _pareto_front_transform_objectives(
-    points: List[Tuple[Any, float, float]],
-    primary_name: str,
-    secondary_name: str
-) -> Tuple[np.ndarray, np.ndarray]:
-    primary_idx = arg_result_names.index(primary_name)
-    secondary_idx = arg_result_names.index(secondary_name)
-
-    x = np.array([p[1] for p in points])
-    y = np.array([p[2] for p in points])
-
-    if arg_result_min_or_max[primary_idx] == "max":
-        x = -x
-    elif arg_result_min_or_max[primary_idx] != "min":
-        raise ValueError(f"Unknown mode for {primary_name}: {arg_result_min_or_max[primary_idx]}")
-
-    if arg_result_min_or_max[secondary_idx] == "max":
-        y = -y
-    elif arg_result_min_or_max[secondary_idx] != "min":
-        raise ValueError(f"Unknown mode for {secondary_name}: {arg_result_min_or_max[secondary_idx]}")
-
-    return x, y
-
-def _pareto_front_select_pareto_points(
-    x: np.ndarray,
-    y: np.ndarray,
-    x_minimize: bool,
-    y_minimize: bool,
-    points: List[Tuple[Any, float, float]],
-    num_points: int
-) -> List[Tuple[Any, float, float]]:
-    indices = pareto_front_general(x, y, x_minimize, y_minimize)
-    sorted_indices = indices[np.argsort(x[indices])]
-    sorted_indices = sorted_indices[:num_points]
-    selected_points = [points[i] for i in sorted_indices]
-    return selected_points
-
-def _pareto_front_build_return_structure(
-    path_to_calculate: str,
-    selected_points: List[Tuple[Any, float, float]],
-    records: Dict[Tuple[int, str], Dict[str, Dict[str, float]]],
-    absolute_metrics: List[str],
-    primary_name: str,
-    secondary_name: str
-) -> dict:
-    results_csv_file = f"{path_to_calculate}/{RESULTS_CSV_FILENAME}"
-    result_names_file = f"{path_to_calculate}/result_names.txt"
-
-    with open(result_names_file, mode="r", encoding="utf-8") as f:
-        result_names = [line.strip() for line in f if line.strip()]
-
-    csv_rows = {}
-    with open(results_csv_file, mode="r", encoding="utf-8", newline='') as csvfile:
-        reader = csv.DictReader(csvfile)
-        for row in reader:
-            trial_index = int(row['trial_index'])
-            csv_rows[trial_index] = row
-
-    ignored_columns = {'trial_index', 'arm_name', 'trial_status', 'generation_node'}
-    ignored_columns.update(result_names)
-
-    param_dicts = []
-    idxs = []
-    means_dict = defaultdict(list)
-
-    for (trial_index, arm_name), _, _ in selected_points:
-        row = csv_rows.get(trial_index, {})
-        if row == {} or row is None or row['arm_name'] != arm_name:
-            print_debug(f"_pareto_front_build_return_structure: trial_index '{trial_index}' could not be found and row returned as None")
-            continue
+        global orchestrator
 
-        idxs.append(int(row["trial_index"]))
+        if args.orchestrator_file:
+            if SYSTEM_HAS_SBATCH:
+                orchestrator = parse_orchestrator_file(args.orchestrator_file, False)
+            else:
+                print_yellow("--orchestrator_file will be ignored on non-sbatch-systems.")
 
-        param_dict: dict[str, int | float | str] = {}
-        for key, value in row.items():
-            if key not in ignored_columns:
-                try:
-                    param_dict[key] = int(value)
-                except ValueError:
-                    try:
-                        param_dict[key] = float(value)
-                    except ValueError:
-                        param_dict[key] = value
+def check_if_has_random_steps() -> None:
+    if (not args.continue_previous_job and "--continue" not in sys.argv) and (args.num_random_steps == 0 or not args.num_random_steps) and args.model not in ["EXTERNAL_GENERATOR", "SOBOL", "PSEUDORANDOM"]:
+        _fatal_error("You have no random steps set. This is only allowed in continued jobs. To start, you need either some random steps, or a continued run.", 233)
 
-        param_dicts.append(param_dict)
+def add_exclude_to_defective_nodes() -> None:
+    with spinner("Adding excluded nodes..."):
+        if args.exclude:
+            entries = [entry.strip() for entry in args.exclude.split(',')]
 
-        for metric in absolute_metrics:
-            means_dict[metric].append(records[(trial_index, arm_name)]['means'].get(metric, float("nan")))
+            for entry in entries:
+                count_defective_nodes(None, entry)
 
-    ret = {
-        primary_name: {
-            secondary_name: {
-                "absolute_metrics": absolute_metrics,
-                "param_dicts": param_dicts,
-                "means": dict(means_dict),
-                "idxs": idxs
-            },
-            "absolute_metrics": absolute_metrics
-        }
-    }
+def check_max_eval(_max_eval: int) -> None:
+    with spinner("Checking max_eval..."):
+        if not _max_eval:
+            _fatal_error("--max_eval needs to be set!", 19)
 
-    return ret
+def parse_parameters() -> Any:
+    cli_params_experiment_parameters = None
+    if args.parameter:
+        parse_experiment_parameters()
+        cli_params_experiment_parameters = experiment_parameters
 
-def get_pareto_frontier_points(
-    path_to_calculate: str,
-    primary_objective: str,
-    secondary_objective: str,
-    x_minimize: bool,
-    y_minimize: bool,
-    absolute_metrics: List[str],
-    num_points: int
-) -> Optional[dict]:
-    records = _pareto_front_aggregate_data(path_to_calculate)
+    return cli_params_experiment_parameters
 
-    if records is None:
-        return None
+def supports_sixel() -> bool:
+    term = os.environ.get("TERM", "").lower()
+    if "xterm" in term or "mlterm" in term:
+        return True
 
-    points = _pareto_front_filter_complete_points(path_to_calculate, records, primary_objective, secondary_objective)
-    x, y = _pareto_front_transform_objectives(points, primary_objective, secondary_objective)
-    selected_points = _pareto_front_select_pareto_points(x, y, x_minimize, y_minimize, points, num_points)
-    result = _pareto_front_build_return_structure(path_to_calculate, selected_points, records, absolute_metrics, primary_objective, secondary_objective)
+    try:
+        output = subprocess.run(["tput", "setab", "256"], capture_output=True, text=True, check=True)
+        if output.returncode == 0 and "sixel" in output.stdout.lower():
+            return True
+    except (subprocess.CalledProcessError, FileNotFoundError):
+        pass
 
-    return result
+    return False
 
 def save_experiment_state() -> None:
     try:
@@ -9543,14 +9966,14 @@ def save_experiment_state() -> None:
             print_red("save_experiment_state: ax_client or ax_client.experiment is None, cannot save.")
             return
         state_path = get_current_run_folder("experiment_state.json")
-        ax_client.save_to_json_file(state_path)
+        save_ax_client_to_json_file(state_path)
     except Exception as e:
-        print(f"Error saving experiment state: {e}")
+        print_debug(f"Error saving experiment state: {e}")
 
 def wait_for_state_file(state_path: str, min_size: int = 5, max_wait_seconds: int = 60) -> bool:
     try:
         if not os.path.exists(state_path):
-            print(f"[ERROR] File '{state_path}' does not exist.")
+            print_debug(f"[ERROR] File '{state_path}' does not exist.")
             return False
 
         i = 0
@@ -9618,7 +10041,7 @@ def load_experiment_state() -> None:
         return
 
     try:
-        arms_seen = {}
+        arms_seen: dict = {}
         for arm in data.get("arms", []):
             name = arm.get("name")
             sig = arm.get("parameters")
@@ -9769,38 +10192,47 @@ def get_result_minimize_flag(path_to_calculate: str, resname: str) -> bool:
 
     return minmax[index] == "min"
 
-def get_pareto_front_data(path_to_calculate: str, res_names: list) -> dict:
-    pareto_front_data: dict = {}
+def post_job_calculate_pareto_front() -> None:
+    if not args.calculate_pareto_front_of_job:
+        return
 
-    all_combinations = list(combinations(range(len(arg_result_names)), 2))
+    failure = False
 
-    skip = False
+    _paths_to_calculate = []
 
-    for i, j in all_combinations:
-        if not skip:
-            metric_x = arg_result_names[i]
-            metric_y = arg_result_names[j]
+    for _path_to_calculate in list(set(args.calculate_pareto_front_of_job)):
+        try:
+            found_paths = find_results_paths(_path_to_calculate)
 
-            x_minimize = get_result_minimize_flag(path_to_calculate, metric_x)
-            y_minimize = get_result_minimize_flag(path_to_calculate, metric_y)
+            for _fp in found_paths:
+                if _fp not in _paths_to_calculate:
+                    _paths_to_calculate.append(_fp)
+        except (FileNotFoundError, NotADirectoryError) as e:
+            print_red(f"post_job_calculate_pareto_front: find_results_paths('{_path_to_calculate}') failed with {e}")
 
-            try:
-                if metric_x not in pareto_front_data:
-                    pareto_front_data[metric_x] = {}
+            failure = True
 
-                pareto_front_data[metric_x][metric_y] = get_calculated_frontier(path_to_calculate, metric_x, metric_y, x_minimize, y_minimize, res_names)
-            except ax.exceptions.core.DataRequiredError as e:
-                print_red(f"Error computing Pareto frontier for {metric_x} and {metric_y}: {e}")
-            except SignalINT:
-                print_red("Calculating Pareto-fronts was cancelled by pressing CTRL-c")
-                skip = True
+    for _path_to_calculate in _paths_to_calculate:
+        for path_to_calculate in found_paths:
+            if not job_calculate_pareto_front(path_to_calculate):
+                failure = True
 
-    return pareto_front_data
+    if failure:
+        my_exit(24)
+
+    my_exit(0)
+
+def pareto_front_as_rich_table(idxs: list, metric_x: str, metric_y: str) -> Optional[Table]:
+    if not os.path.exists(RESULT_CSV_FILE):
+        print_debug(f"pareto_front_as_rich_table: File '{RESULT_CSV_FILE}' not found")
+        return None
+
+    return create_pareto_front_table(idxs, metric_x, metric_y)
 
 def show_pareto_frontier_data(path_to_calculate: str, res_names: list, disable_sixel_and_table: bool = False) -> None:
     if len(res_names) <= 1:
         print_debug(f"--result_names (has {len(res_names)} entries) must be at least 2.")
-        return
+        return None
 
     pareto_front_data: dict = get_pareto_front_data(path_to_calculate, res_names)
 
@@ -9821,8 +10253,16 @@ def show_pareto_frontier_data(path_to_calculate: str, res_names: list, disable_s
                 else:
                     print(f"Not showing Pareto-front-sixel for {path_to_calculate}")
 
-            if len(calculated_frontier[metric_x][metric_y]["idxs"]):
-                pareto_points[metric_x][metric_y] = sorted(calculated_frontier[metric_x][metric_y]["idxs"])
+            if calculated_frontier is None:
+                print_debug("ERROR: calculated_frontier is None")
+                return None
+
+            try:
+                if len(calculated_frontier[metric_x][metric_y]["idxs"]):
+                    pareto_points[metric_x][metric_y] = sorted(calculated_frontier[metric_x][metric_y]["idxs"])
+            except AttributeError:
+                print_debug(f"ERROR: calculated_frontier structure invalid for ({metric_x}, {metric_y})")
+                return None
 
             rich_table = pareto_front_as_rich_table(
                 calculated_frontier[metric_x][metric_y]["idxs"],
@@ -9847,6 +10287,8 @@ def show_pareto_frontier_data(path_to_calculate: str, res_names: list, disable_s
 
     live_share_after_pareto()
 
+    return None
+
 def show_available_hardware_and_generation_strategy_string(gpu_string: str, gpu_color: str) -> None:
     cpu_count = os.cpu_count()
 
@@ -9858,9 +10300,11 @@ def show_available_hardware_and_generation_strategy_string(gpu_string: str, gpu_
         pass
 
     if gpu_string:
-        console.print(f"[green]You have {cpu_count} CPUs available for the main process.[/green] [{gpu_color}]{gpu_string}[/{gpu_color}] [green]{gs_string}[/green]")
+        console.print(f"[green]You have {cpu_count} CPUs available for the main process.[/green] [{gpu_color}]{gpu_string}[/{gpu_color}]")
     else:
-        print_green(f"You have {cpu_count} CPUs available for the main process. {gs_string}")
+        print_green(f"You have {cpu_count} CPUs available for the main process.")
+
+    print_green(gs_string)
 
 def write_args_overview_table() -> None:
     table = Table(title="Arguments Overview")
@@ -10127,112 +10571,6 @@ def find_results_paths(base_path: str) -> list:
 
     return list(set(found_paths))
 
-def post_job_calculate_pareto_front() -> None:
-    if not args.calculate_pareto_front_of_job:
-        return
-
-    failure = False
-
-    _paths_to_calculate = []
-
-    for _path_to_calculate in list(set(args.calculate_pareto_front_of_job)):
-        try:
-            found_paths = find_results_paths(_path_to_calculate)
-
-            for _fp in found_paths:
-                if _fp not in _paths_to_calculate:
-                    _paths_to_calculate.append(_fp)
-        except (FileNotFoundError, NotADirectoryError) as e:
-            print_red(f"post_job_calculate_pareto_front: find_results_paths('{_path_to_calculate}') failed with {e}")
-
-            failure = True
-
-    for _path_to_calculate in _paths_to_calculate:
-        for path_to_calculate in found_paths:
-            if not job_calculate_pareto_front(path_to_calculate):
-                failure = True
-
-    if failure:
-        my_exit(24)
-
-    my_exit(0)
-
-def job_calculate_pareto_front(path_to_calculate: str, disable_sixel_and_table: bool = False) -> bool:
-    pf_start_time = time.time()
-
-    if not path_to_calculate:
-        return False
-
-    global CURRENT_RUN_FOLDER
-    global RESULT_CSV_FILE
-    global arg_result_names
-
-    if not path_to_calculate:
-        print_red("Can only calculate pareto front of previous job when --calculate_pareto_front_of_job is set")
-        return False
-
-    if not os.path.exists(path_to_calculate):
-        print_red(f"Path '{path_to_calculate}' does not exist")
-        return False
-
-    ax_client_json = f"{path_to_calculate}/state_files/ax_client.experiment.json"
-
-    if not os.path.exists(ax_client_json):
-        print_red(f"Path '{ax_client_json}' not found")
-        return False
-
-    checkpoint_file: str = f"{path_to_calculate}/state_files/checkpoint.json"
-    if not os.path.exists(checkpoint_file):
-        print_red(f"The checkpoint file '{checkpoint_file}' does not exist")
-        return False
-
-    RESULT_CSV_FILE = f"{path_to_calculate}/{RESULTS_CSV_FILENAME}"
-    if not os.path.exists(RESULT_CSV_FILE):
-        print_red(f"{RESULT_CSV_FILE} not found")
-        return False
-
-    res_names = []
-
-    res_names_file = f"{path_to_calculate}/result_names.txt"
-    if not os.path.exists(res_names_file):
-        print_red(f"File '{res_names_file}' does not exist")
-        return False
-
-    try:
-        with open(res_names_file, "r", encoding="utf-8") as file:
-            lines = file.readlines()
-    except Exception as e:
-        print_red(f"Error reading file '{res_names_file}': {e}")
-        return False
-
-    for line in lines:
-        entry = line.strip()
-        if entry != "":
-            res_names.append(entry)
-
-    if len(res_names) < 2:
-        print_red(f"Error: There are less than 2 result names (is: {len(res_names)}, {', '.join(res_names)}) in {path_to_calculate}. Cannot continue calculating the pareto front.")
-        return False
-
-    load_username_to_args(path_to_calculate)
-
-    CURRENT_RUN_FOLDER = path_to_calculate
-
-    arg_result_names = res_names
-
-    load_experiment_parameters_from_checkpoint_file(checkpoint_file, False)
-
-    if experiment_parameters is None:
-        return False
-
-    show_pareto_or_error_msg(path_to_calculate, res_names, disable_sixel_and_table)
-
-    pf_end_time = time.time()
-
-    print_debug(f"Calculating the Pareto-front took {pf_end_time - pf_start_time} seconds")
-
-    return True
-
 def set_arg_states_from_continue() -> None:
     if args.continue_previous_job and not args.num_random_steps:
         num_random_steps_file = f"{args.continue_previous_job}/state_files/num_random_steps"
@@ -10266,7 +10604,7 @@ def write_result_names_file() -> None:
         except Exception as e:
             print_red(f"Error trying to open file '{fn}': {e}")
 
-def run_program_once(params=None) -> None:
+def run_program_once(params: Optional[dict] = None) -> None:
     if not args.run_program_once:
         print_debug("[yellow]No setup script specified (run_program_once). Skipping setup.[/yellow]")
         return
@@ -10275,27 +10613,27 @@ def run_program_once(params=None) -> None:
         params = {}
 
     if isinstance(args.run_program_once, str):
-        command_str = args.run_program_once
+        command_str = decode_if_base64(args.run_program_once)
         for k, v in params.items():
             placeholder = f"%({k})"
             command_str = command_str.replace(placeholder, str(v))
 
-        with spinner(f"Executing command: [cyan]{command_str}[/cyan]"):
-            result = subprocess.run(command_str, shell=True, check=True)
-            if result.returncode == 0:
-                console.log("[bold green]Setup script completed successfully ✅[/bold green]")
-            else:
-                console.log(f"[bold red]Setup script failed with exit code {result.returncode} ❌[/bold red]")
+        print(f"Executing command: [cyan]{command_str}[/cyan]")
+        result = subprocess.run(command_str, shell=True, check=True)
+        if result.returncode == 0:
+            print("[bold green]Setup script completed successfully ✅[/bold green]")
+        else:
+            print(f"[bold red]Setup script failed with exit code {result.returncode} ❌[/bold red]")
 
-                my_exit(57)
+            my_exit(57)
 
     elif isinstance(args.run_program_once, (list, tuple)):
         with spinner("run_program_once: Executing command list: [cyan]{args.run_program_once}[/cyan]"):
             result = subprocess.run(args.run_program_once, check=True)
             if result.returncode == 0:
-                console.log("[bold green]Setup script completed successfully ✅[/bold green]")
+                print("[bold green]Setup script completed successfully ✅[/bold green]")
             else:
-                console.log(f"[bold red]Setup script failed with exit code {result.returncode} ❌[/bold red]")
+                print(f"[bold red]Setup script failed with exit code {result.returncode} ❌[/bold red]")
 
                 my_exit(57)
 
@@ -10305,7 +10643,7 @@ def run_program_once(params=None) -> None:
         my_exit(57)
 
 def show_omniopt_call() -> None:
-    def remove_ui_url(arg_str) -> str:
+    def remove_ui_url(arg_str: str) -> str:
         return re.sub(r'(?:--ui_url(?:=\S+)?(?:\s+\S+)?)', '', arg_str).strip()
 
     original_argv = " ".join(sys.argv[1:])
@@ -10313,8 +10651,14 @@ def show_omniopt_call() -> None:
 
     original_print(oo_call + " " + cleaned)
 
+    if args.dependency is not None and args.dependency != "":
+        print(f"Dependency: {args.dependency}")
+
+    if args.ui_url is not None and args.ui_url != "":
+        print_yellow("--ui_url is deprecated. Do not use it anymore. It will be ignored and one day be removed.")
+
 def main() -> None:
-    global RESULT_CSV_FILE, ax_client, LOGFILE_DEBUG_GET_NEXT_TRIALS
+    global RESULT_CSV_FILE, LOGFILE_DEBUG_GET_NEXT_TRIALS
 
     check_if_has_random_steps()
 
@@ -10396,15 +10740,13 @@ def main() -> None:
     exp_params = get_experiment_parameters(cli_params_experiment_parameters)
 
     if exp_params is not None:
-        ax_client, experiment_args, gpu_string, gpu_color = exp_params
+        experiment_args, gpu_string, gpu_color = exp_params
         print_debug(f"experiment_parameters: {experiment_parameters}")
 
         set_orchestrator()
 
         init_live_share()
 
-        start_periodic_live_share()
-
         show_available_hardware_and_generation_strategy_string(gpu_string, gpu_color)
 
         original_print(f"Run-Program: {global_vars['joined_run_program']}")
@@ -10419,6 +10761,8 @@ def main() -> None:
 
         write_files_and_show_overviews()
 
+        live_share()
+
         #if args.continue_previous_job:
         #    insert_jobs_from_csv(f"{args.continue_previous_job}/{RESULTS_CSV_FILENAME}")
 
@@ -10496,21 +10840,37 @@ def initialize_nvidia_logs() -> None:
 def build_gui_url(config: argparse.Namespace) -> str:
     base_url = get_base_url()
     params = collect_params(config)
-    return f"{base_url}?{urlencode(params, doseq=True)}"
+    ret = f"{base_url}?{urlencode(params, doseq=True)}"
+
+    return ret
+
+def get_result_names_for_url(value: List) -> str:
+    d = dict(v.split("=", 1) if "=" in v else (v, "min") for v in value)
+    s = " ".join(f"{k}={v}" for k, v in d.items())
+
+    return s
 
 def collect_params(config: argparse.Namespace) -> dict:
     params = {}
+    user_home = os.path.expanduser("~")
+
     for attr, value in vars(config).items():
         if attr == "run_program":
             params[attr] = global_vars["joined_run_program"]
+        elif attr == "result_names" and value:
+            params[attr] = get_result_names_for_url(value)
         elif attr == "parameter" and value is not None:
             params.update(process_parameters(config.parameter))
+        elif attr == "root_venv_dir":
+            if value is not None and os.path.abspath(value) != os.path.abspath(user_home):
+                params[attr] = value
         elif isinstance(value, bool):
             params[attr] = int(value)
         elif isinstance(value, list):
             params[attr] = value
         elif value is not None:
             params[attr] = value
+
     return params
 
 def process_parameters(parameters: list) -> dict:
@@ -10563,7 +10923,7 @@ def get_base_url() -> str:
 def write_ui_url() -> None:
     url = build_gui_url(args)
     with open(get_current_run_folder("ui_url.txt"), mode="a", encoding="utf-8") as myfile:
-        myfile.write(decode_if_base64(url))
+        myfile.write(url)
 
 def handle_random_steps() -> None:
     if args.parameter and args.continue_previous_job and random_steps <= 0:
@@ -10622,8 +10982,6 @@ def run_search_with_progress_bar() -> None:
     wait_for_jobs_to_complete()
 
 def complex_tests(_program_name: str, wanted_stderr: str, wanted_exit_code: int, wanted_signal: Union[int, None], res_is_none: bool = False) -> int:
-    #print_yellow(f"Test suite: {_program_name}")
-
     nr_errors: int = 0
 
     program_path: str = f"./.tests/test_wronggoing_stuff.bin/bin/{_program_name}"
@@ -10697,7 +11055,7 @@ def test_find_paths(program_code: str) -> int:
     for i in files:
         if i not in string:
             if os.path.exists(i):
-                print("Missing {i} in find_file_paths string!")
+                print(f"Missing {i} in find_file_paths string!")
                 nr_errors += 1
 
     return nr_errors
@@ -11078,17 +11436,16 @@ Exit-Code: 159
 
     my_exit(nr_errors)
 
-def main_outside() -> None:
+def main_wrapper() -> None:
     print(f"Run-UUID: {run_uuid}")
 
     auto_wrap_namespace(globals())
 
     print_logo()
 
-    start_logging_daemon()
-
     fool_linter(args.num_cpus_main_job)
     fool_linter(args.flame_graph)
+    fool_linter(args.memray)
 
     with warnings.catch_warnings():
         warnings.simplefilter("ignore")
@@ -11123,7 +11480,7 @@ def main_outside() -> None:
 def stack_trace_wrapper(func: Any, regex: Any = None) -> Any:
     pattern = re.compile(regex) if regex else None
 
-    def wrapped(*args, **kwargs):
+    def wrapped(*args: Any, **kwargs: Any) -> None:
         if pattern and not pattern.search(func.__name__):
             return func(*args, **kwargs)
 
@@ -11145,6 +11502,9 @@ def stack_trace_wrapper(func: Any, regex: Any = None) -> Any:
 def auto_wrap_namespace(namespace: Any) -> Any:
     enable_beartype = any(os.getenv(v) for v in ("ENABLE_BEARTYPE", "CI"))
 
+    if args.beartype:
+        enable_beartype = True
+
     excluded_functions = {
         "log_time_and_memory_wrapper",
         "collect_runtime_stats",
@@ -11153,8 +11513,6 @@ def auto_wrap_namespace(namespace: Any) -> Any:
         "_record_stats",
         "_open",
         "_check_memory_leak",
-        "start_periodic_live_share",
-        "start_logging_daemon",
         "get_current_run_folder",
         "show_func_name_wrapper"
     }
@@ -11180,7 +11538,7 @@ def auto_wrap_namespace(namespace: Any) -> Any:
 
 if __name__ == "__main__":
     try:
-        main_outside()
+        main_wrapper()
     except (SignalUSR, SignalINT, SignalCONT) as e:
-        print_red(f"main_outside failed with exception {e}")
+        print_red(f"main_wrapper failed with exception {e}")
         end_program(True)