omniopt2 8178__py3-none-any.whl → 9171__py3-none-any.whl

.omniopt.py

@@ -2,7 +2,7 @@
 
 #from mayhemmonkey import MayhemMonkey
 #mayhemmonkey = MayhemMonkey()
-#mayhemmonkey.set_function_fail_after_count("open", 201)
+#mayhemmonkey.set_function_fail_after_count("open", 10)
 #mayhemmonkey.set_function_error_rate("open", 0.1)
 #mayhemmonkey.set_function_group_error_rate(["io", "math"], 0.8)
 #mayhemmonkey.install_faulty()
@@ -26,10 +26,16 @@ import traceback
 import inspect
 import tracemalloc
 import resource
+from urllib.parse import urlencode
 import psutil
 
 FORCE_EXIT: bool = False
 
+LAST_LOG_TIME: int = 0
+last_msg_progressbar = ""
+last_msg_raw = None
+last_lock_print_debug = threading.Lock()
+
 def force_exit(signal_number: Any, frame: Any) -> Any:
     global FORCE_EXIT
 
@@ -59,10 +65,10 @@ _last_count_time = 0
 _last_count_result: tuple[int, str] = (0, "")
 
 _total_time = 0.0
-_func_times = defaultdict(float)
-_func_mem = defaultdict(float)
-_func_call_paths = defaultdict(Counter)
-_last_mem = defaultdict(float)
+_func_times: defaultdict = defaultdict(float)
+_func_mem: defaultdict = defaultdict(float)
+_func_call_paths: defaultdict = defaultdict(Counter)
+_last_mem: defaultdict = defaultdict(float)
 _leak_threshold_mb = 10.0
 generation_strategy_names: list = []
 default_max_range_difference: int = 1000000
@@ -82,6 +88,7 @@ log_nr_gen_jobs: list[int] = []
 generation_strategy_human_readable: str = ""
 oo_call: str = "./omniopt"
 progress_bar_length: int = 0
+worker_usage_file = 'worker_usage.csv'
 
 if os.environ.get("CUSTOM_VIRTUAL_ENV") == "1":
     oo_call = "omniopt"
@@ -97,7 +104,7 @@ joined_valid_occ_types: str = ", ".join(valid_occ_types)
 SUPPORTED_MODELS: list = ["SOBOL", "FACTORIAL", "SAASBO", "BOTORCH_MODULAR", "UNIFORM", "BO_MIXED", "RANDOMFOREST", "EXTERNAL_GENERATOR", "PSEUDORANDOM", "TPE"]
 joined_supported_models: str = ", ".join(SUPPORTED_MODELS)
 
-special_col_names: list = ["arm_name", "generation_method", "trial_index", "trial_status", "generation_node", "idxs", "start_time", "end_time", "run_time", "exit_code", "program_string", "signal", "hostname", "submit_time", "queue_time", "metric_name", "mean", "sem", "worker_generator_uuid"]
+special_col_names: list = ["arm_name", "generation_method", "trial_index", "trial_status", "generation_node", "idxs", "start_time", "end_time", "run_time", "exit_code", "program_string", "signal", "hostname", "submit_time", "queue_time", "metric_name", "mean", "sem", "worker_generator_uuid", "runtime", "status"]
 
 IGNORABLE_COLUMNS: list = ["start_time", "end_time", "hostname", "signal", "exit_code", "run_time", "program_string"] + special_col_names
 
@@ -151,12 +158,6 @@ try:
             message="Ax currently requires a sqlalchemy version below 2.0.*",
         )
 
-        warnings.filterwarnings(
-            "ignore",
-            category=RuntimeWarning,
-            message="coroutine 'start_logging_daemon' was never awaited"
-        )
-
     with spinner("Importing argparse..."):
         import argparse
 
@@ -202,6 +203,9 @@ try:
     with spinner("Importing rich.pretty..."):
         from rich.pretty import pprint
 
+    with spinner("Importing pformat..."):
+        from pprint import pformat
+
     with spinner("Importing rich.prompt..."):
         from rich.prompt import Prompt, FloatPrompt, IntPrompt
 
@@ -235,9 +239,6 @@ try:
     with spinner("Importing uuid..."):
         import uuid
 
-    #with spinner("Importing qrcode..."):
-    #    import qrcode
-
     with spinner("Importing cowsay..."):
         import cowsay
 
@@ -268,6 +269,9 @@ try:
     with spinner("Importing beartype..."):
         from beartype import beartype
 
+    with spinner("Importing rendering stuff..."):
+        from ax.plot.base import AxPlotConfig
+
     with spinner("Importing statistics..."):
         import statistics
 
@@ -331,7 +335,7 @@ def show_func_name_wrapper(func: F) -> F:
 
         return result
 
-    return wrapper  # type: ignore
+    return wrapper # type: ignore
 
 def log_time_and_memory_wrapper(func: F) -> F:
     @functools.wraps(func)
@@ -358,7 +362,7 @@ def log_time_and_memory_wrapper(func: F) -> F:
 
         return result
 
-    return wrapper  # type: ignore
+    return wrapper # type: ignore
 
 def _record_stats(func_name: str, elapsed: float, mem_diff: float, mem_after: float, mem_peak: float) -> None:
     global _total_time
@@ -379,16 +383,9 @@ def _record_stats(func_name: str, elapsed: float, mem_diff: float, mem_after: fl
         call_path_str = " -> ".join(short_stack)
         _func_call_paths[func_name][call_path_str] += 1
 
-        print(
-            f"Function '{func_name}' took {elapsed:.4f}s "
-            f"(total {percent_if_added:.1f}% of tracked time)"
-        )
-        print(
-            f"Memory before: {mem_after - mem_diff:.2f} MB, after: {mem_after:.2f} MB, "
-            f"diff: {mem_diff:+.2f} MB, peak during call: {mem_peak:.2f} MB"
-        )
+        print(f"Function '{func_name}' took {elapsed:.4f}s (total {percent_if_added:.1f}% of tracked time)")
+        print(f"Memory before: {mem_after - mem_diff:.2f} MB, after: {mem_after:.2f} MB, diff: {mem_diff:+.2f} MB, peak during call: {mem_peak:.2f} MB")
 
-        # NEU: Runtime Stats
         runtime_stats = collect_runtime_stats()
         print("=== Runtime Stats ===")
         print(f"RSS: {runtime_stats['rss_MB']:.2f} MB, VMS: {runtime_stats['vms_MB']:.2f} MB")
@@ -447,7 +444,7 @@ RESET: str = "\033[0m"
 
 uuid_regex: Pattern = re.compile(r"^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-4[a-fA-F0-9]{3}-[89aAbB][a-fA-F0-9]{3}-[a-fA-F0-9]{12}$")
 
-worker_generator_uuid: str = uuid.uuid4()
+worker_generator_uuid: str = str(uuid.uuid4())
 
 new_uuid: str = str(uuid.uuid4())
 run_uuid: str = os.getenv("RUN_UUID", new_uuid)
@@ -476,7 +473,7 @@ def get_current_run_folder(name: Optional[str] = None) -> str:
 
     return CURRENT_RUN_FOLDER
 
-def get_state_file_name(name) -> str:
+def get_state_file_name(name: str) -> str:
     state_files_folder = f"{get_current_run_folder()}/state_files/"
     makedirs(state_files_folder)
 
@@ -500,6 +497,24 @@ try:
         dier: FunctionType = helpers.dier
         is_equal: FunctionType = helpers.is_equal
         is_not_equal: FunctionType = helpers.is_not_equal
+    with spinner("Importing pareto..."):
+        pareto_file: str = f"{script_dir}/.pareto.py"
+        spec = importlib.util.spec_from_file_location(
+            name="pareto",
+            location=pareto_file,
+        )
+        if spec is not None and spec.loader is not None:
+            pareto = importlib.util.module_from_spec(spec)
+            spec.loader.exec_module(pareto)
+        else:
+            raise ImportError(f"Could not load module from {pareto_file}")
+
+        pareto_front_table_filter_rows: FunctionType = pareto.pareto_front_table_filter_rows
+        pareto_front_table_add_headers: FunctionType = pareto.pareto_front_table_add_headers
+        pareto_front_table_add_rows: FunctionType = pareto.pareto_front_table_add_rows
+        pareto_front_filter_complete_points: FunctionType = pareto.pareto_front_filter_complete_points
+        pareto_front_select_pareto_points: FunctionType = pareto.pareto_front_select_pareto_points
+
 except KeyboardInterrupt:
     print("You pressed CTRL-c while importing the helpers file")
     sys.exit(0)
@@ -523,15 +538,13 @@ def is_slurm_job() -> bool:
         return True
     return False
 
-def _sleep(t: int) -> int:
+def _sleep(t: Union[float, int]) -> None:
     if args is not None and not args.no_sleep:
         try:
             time.sleep(t)
         except KeyboardInterrupt:
             pass
 
-    return t
-
 LOG_DIR: str = "logs"
 makedirs(LOG_DIR)
 
@@ -544,6 +557,17 @@ logfile_worker_creation_logs: str = f'{log_uuid_dir}_worker_creation_logs'
 logfile_trial_index_to_param_logs: str = f'{log_uuid_dir}_trial_index_to_param_logs'
 LOGFILE_DEBUG_GET_NEXT_TRIALS: Union[str, None] = None
 
+def error_without_print(text: str) -> None:
+    print_debug(text)
+
+    if get_current_run_folder():
+        try:
+            with open(get_current_run_folder("oo_errors.txt"), mode="a", encoding="utf-8") as myfile:
+                myfile.write(text + "\n\n")
+        except (OSError, FileNotFoundError) as e:
+            helpers.print_color("red", f"Error: {e}. This may mean that the {get_current_run_folder()} was deleted during the run. Could not write '{text} to {get_current_run_folder()}/oo_errors.txt'")
+            sys.exit(99)
+
 def print_red(text: str) -> None:
     helpers.print_color("red", text)
 
@@ -577,20 +601,22 @@ def _debug(msg: str, _lvl: int = 0, eee: Union[None, str, Exception] = None) ->
 def _get_debug_json(time_str: str, msg: str) -> str:
     function_stack = []
     try:
-        frame = inspect.currentframe().f_back  # skip _get_debug_json
-        while frame:
-            func_name = _function_name_cache.get(frame.f_code)
-            if func_name is None:
-                func_name = frame.f_code.co_name
-                _function_name_cache[frame.f_code] = func_name
-
-            if func_name not in ("<module>", "print_debug", "wrapper"):
-                function_stack.append({
-                    "function": func_name,
-                    "line_number": frame.f_lineno
-                })
-
-            frame = frame.f_back
+        cf = inspect.currentframe()
+        if cf:
+            frame = cf.f_back  # skip _get_debug_json
+            while frame:
+                func_name = _function_name_cache.get(frame.f_code)
+                if func_name is None:
+                    func_name = frame.f_code.co_name
+                    _function_name_cache[frame.f_code] = func_name
+
+                if func_name not in ("<module>", "print_debug", "wrapper"):
+                    function_stack.append({
+                        "function": func_name,
+                        "line_number": frame.f_lineno
+                    })
+
+                frame = frame.f_back
     except (SignalUSR, SignalINT, SignalCONT):
         print_red("\n⚠ You pressed CTRL-C. This is ignored in _get_debug_json.")
 
@@ -599,20 +625,46 @@ def _get_debug_json(time_str: str, msg: str) -> str:
         separators=(",", ":")  # no pretty indent → smaller, faster
     ).replace('\r', '').replace('\n', '')
 
-def print_debug(msg: str) -> None:
-    original_msg = msg
+def print_stack_paths() -> None:
+    stack = inspect.stack()[1:]  # skip current frame
+    stack.reverse()
 
-    time_str: str = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+    last_filename = None
+    for depth, frame_info in enumerate(stack):
+        filename = frame_info.filename
+        lineno = frame_info.lineno
+        func_name = frame_info.function
 
-    stack_trace_element = _get_debug_json(time_str, msg)
+        if func_name in ["<module>", "print_debug"]:
+            continue
 
-    msg = f"{stack_trace_element}"
+        if filename != last_filename:
+            print(filename)
+            last_filename = filename
+            indent = ""
+        else:
+            indent = " " * 4 * depth
+
+        print(f"{indent}↳ {func_name}:{lineno}")
+
+def print_debug(msg: str) -> None:
+    time_str = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+
+    stack = traceback.extract_stack()[:-1]
+    stack_funcs = [frame.name for frame in stack]
 
-    _debug(msg)
+    if "args" in globals() and args and hasattr(args, "debug_stack_regex") and args.debug_stack_regex:
+        matched = any(any(re.match(regex, func) for regex in args.debug_stack_regex) for func in stack_funcs)
+        if matched:
+            print(f"DEBUG: {msg}")
+            print_stack_paths()
+
+    stack_trace_element = _get_debug_json(time_str, msg)
+    _debug(stack_trace_element)
 
     try:
         with open(logfile_bare, mode='a', encoding="utf-8") as f:
-            original_print(original_msg, file=f)
+            original_print(msg, file=f)
     except FileNotFoundError:
         print_red("It seems like the run's folder was deleted during the run. Cannot continue.")
         sys.exit(99)
@@ -666,11 +718,14 @@ def my_exit(_code: int = 0) -> None:
     if is_skip_search() and os.getenv("SKIP_SEARCH_EXIT_CODE"):
         skip_search_exit_code = os.getenv("SKIP_SEARCH_EXIT_CODE")
 
+        skip_search_exit_code_found = None
+
         try:
-            sys.exit(int(skip_search_exit_code))
+            if skip_search_exit_code_found is not None:
+                skip_search_exit_code_found = int(skip_search_exit_code)
+                sys.exit(skip_search_exit_code_found)
         except ValueError:
-            print(f"Trying to look for SKIP_SEARCH_EXIT_CODE failed. Exiting with original exit code {_code}")
-            sys.exit(_code)
+            print_debug(f"Trying to look for SKIP_SEARCH_EXIT_CODE failed. Exiting with original exit code {_code}")
 
     sys.exit(_code)
 
@@ -730,6 +785,7 @@ _DEFAULT_SPECIALS: Dict[str, Any] = {
 class ConfigLoader:
     runtime_debug: bool
     show_func_name: bool
+    debug_stack_regex: str
     number_of_generators: int
     disable_previous_job_constraint: bool
     save_to_database: bool
@@ -770,11 +826,13 @@ class ConfigLoader:
     parameter: Optional[List[str]]
     experiment_constraints: Optional[List[str]]
     main_process_gb: int
+    beartype: bool
     worker_timeout: int
     slurm_signal_delay_s: int
     gridsearch: bool
     auto_exclude_defective_hosts: bool
     debug: bool
+    debug_stack_trace_regex: Optional[str]
     num_restarts: int
     raw_samples: int
     show_generate_time_table: bool
@@ -789,6 +847,7 @@ class ConfigLoader:
     run_program_once: str
     mem_gb: int
     flame_graph: bool
+    memray: bool
     continue_previous_job: Optional[str]
     calculate_pareto_front_of_job: Optional[List[str]]
     revert_to_random_when_seemingly_exhausted: bool
@@ -950,6 +1009,7 @@ class ConfigLoader:
         debug.add_argument('--verbose_break_run_search_table', help='Verbose logging for break_run_search', action='store_true', default=False)
         debug.add_argument('--debug', help='Enable debugging', action='store_true', default=False)
         debug.add_argument('--flame_graph', help='Enable flame-graphing. Makes everything slower, but creates a flame graph', action='store_true', default=False)
+        debug.add_argument('--memray', help='Use memray to show memory usage', action='store_true', default=False)
         debug.add_argument('--no_sleep', help='Disables sleeping for fast job generation (not to be used on HPC)', action='store_true', default=False)
         debug.add_argument('--tests', help='Run simple internal tests', action='store_true', default=False)
         debug.add_argument('--show_worker_percentage_table_at_end', help='Show a table of percentage of usage of max worker over time', action='store_true', default=False)
@@ -961,7 +1021,10 @@ class ConfigLoader:
         debug.add_argument('--just_return_defaults', help='Just return defaults in dryrun', action='store_true', default=False)
         debug.add_argument('--prettyprint', help='Shows stdout and stderr in a pretty printed format', action='store_true', default=False)
         debug.add_argument('--runtime_debug', help='Logs which functions use most of the time', action='store_true', default=False)
+        debug.add_argument('--debug_stack_regex', help='Only print debug messages if call stack matches any regex', type=str, default='')
+        debug.add_argument('--debug_stack_trace_regex', help='Show compact call stack with arrows if any function in stack matches regex', type=str, default=None)
         debug.add_argument('--show_func_name', help='Show func name before each execution and when it is done', action='store_true', default=False)
+        debug.add_argument('--beartype', help='Use beartype', action='store_true', default=False)
 
     def load_config(self: Any, config_path: str, file_format: str) -> dict:
         if not os.path.isfile(config_path):
@@ -1199,11 +1262,15 @@ for _rn in args.result_names:
         _key = _rn
         _min_or_max = __default_min_max
 
+    _min_or_max = re.sub(r"'", "", _min_or_max)
+
     if _min_or_max not in ["min", "max"]:
         if _min_or_max:
             print_yellow(f"Value for determining whether to minimize or maximize was neither 'min' nor 'max' for key '{_key}', but '{_min_or_max}'. It will be set to the default, which is '{__default_min_max}' instead.")
         _min_or_max = __default_min_max
 
+    _key = re.sub(r"'", "", _key)
+
     if _key in arg_result_names:
         console.print(f"[red]The --result_names option '{_key}' was specified multiple times![/]")
         sys.exit(50)
@@ -1304,27 +1371,14 @@ try:
     with spinner("Importing ExternalGenerationNode..."):
         from ax.generation_strategy.external_generation_node import ExternalGenerationNode
 
-    with spinner("Importing MaxTrials..."):
-        from ax.generation_strategy.transition_criterion import MaxTrials
+    with spinner("Importing MinTrials..."):
+        from ax.generation_strategy.transition_criterion import MinTrials
 
     with spinner("Importing GeneratorSpec..."):
         from ax.generation_strategy.generator_spec import GeneratorSpec
 
-    #except Exception:
-    #    with spinner("Fallback: Importing ax.generation_strategy.generation_node..."):
-    #        import ax.generation_strategy.generation_node
-
-    #    with spinner("Fallback: Importing GenerationStep, GenerationStrategy from ax.generation_strategy..."):
-    #        from ax.generation_strategy.generation_node import GenerationNode, GenerationStep
-
-    #    with spinner("Fallback: Importing ExternalGenerationNode..."):
-    #        from ax.generation_strategy.external_generation_node import ExternalGenerationNode
-
-    #    with spinner("Fallback: Importing MaxTrials..."):
-    #        from ax.generation_strategy.transition_criterion import MaxTrials
-
-    with spinner("Importing Models from ax.generation_strategy.registry..."):
-        from ax.adapter.registry import Models
+    with spinner("Importing Generators from ax.generation_strategy.registry..."):
+        from ax.adapter.registry import Generators
 
     with spinner("Importing get_pending_observation_features..."):
         from ax.core.utils import get_pending_observation_features
@@ -1410,7 +1464,7 @@ class RandomForestGenerationNode(ExternalGenerationNode):
     def __init__(self: Any, regressor_options: Dict[str, Any] = {}, seed: Optional[int] = None, num_samples: int = 1) -> None:
         print_debug("Initializing RandomForestGenerationNode...")
         t_init_start = time.monotonic()
-        super().__init__(node_name="RANDOMFOREST")
+        super().__init__(name="RANDOMFOREST")
         self.num_samples: int = num_samples
         self.seed: int = seed
 
@@ -1430,6 +1484,9 @@ class RandomForestGenerationNode(ExternalGenerationNode):
     def update_generator_state(self: Any, experiment: Experiment, data: Data) -> None:
         search_space = experiment.search_space
         parameter_names = list(search_space.parameters.keys())
+        if experiment.optimization_config is None:
+            print_red("Error: update_generator_state is None")
+            return
         metric_names = list(experiment.optimization_config.metrics.keys())
 
         completed_trials = [
@@ -1441,7 +1498,7 @@ class RandomForestGenerationNode(ExternalGenerationNode):
         y = np.zeros([num_completed_trials, 1])
 
         for t_idx, trial in enumerate(completed_trials):
-            trial_parameters = trial.arm.parameters
+            trial_parameters = trial.arms[t_idx].parameters
             x[t_idx, :] = np.array([trial_parameters[p] for p in parameter_names])
             trial_df = data.df[data.df["trial_index"] == trial.index]
             y[t_idx, 0] = trial_df[trial_df["metric_name"] == metric_names[0]]["mean"].item()
@@ -1581,10 +1638,18 @@ class RandomForestGenerationNode(ExternalGenerationNode):
     def _format_best_sample(self: Any, best_sample: TParameterization, reverse_choice_map: dict) -> None:
         for name in best_sample.keys():
             param = self.parameters.get(name)
+            best_sample_by_name = best_sample[name]
+
             if isinstance(param, RangeParameter) and param.parameter_type == ParameterType.INT:
-                best_sample[name] = int(round(best_sample[name]))
+                if best_sample_by_name is not None:
+                    best_sample[name] = int(round(float(best_sample_by_name)))
+                else:
+                    print_debug("best_sample_by_name was empty")
             elif isinstance(param, ChoiceParameter):
-                best_sample[name] = str(reverse_choice_map.get(int(best_sample[name])))
+                if best_sample_by_name is not None:
+                    best_sample[name] = str(reverse_choice_map.get(int(best_sample_by_name)))
+                else:
+                    print_debug("best_sample_by_name was empty")
 
 decoder_registry["RandomForestGenerationNode"] = RandomForestGenerationNode
 
@@ -1632,10 +1697,10 @@ class InteractiveCLIGenerationNode(ExternalGenerationNode):
 
     def __init__(
         self: Any,
-        node_name: str = "INTERACTIVE_GENERATOR",
+        name: str = "INTERACTIVE_GENERATOR",
     ) -> None:
         t0 = time.monotonic()
-        super().__init__(node_name=node_name)
+        super().__init__(name=name)
         self.parameters = None
         self.minimize = None
         self.data = None
@@ -1791,10 +1856,10 @@ class InteractiveCLIGenerationNode(ExternalGenerationNode):
 
 @dataclass(init=False)
 class ExternalProgramGenerationNode(ExternalGenerationNode):
-    def __init__(self: Any, external_generator: str = args.external_generator, node_name: str = "EXTERNAL_GENERATOR") -> None:
+    def __init__(self: Any, external_generator: str = args.external_generator, name: str = "EXTERNAL_GENERATOR") -> None:
         print_debug("Initializing ExternalProgramGenerationNode...")
         t_init_start = time.monotonic()
-        super().__init__(node_name=node_name)
+        super().__init__(name=name)
         self.seed: int = args.seed
         self.external_generator: str = decode_if_base64(external_generator)
         self.constraints = None
@@ -2015,24 +2080,15 @@ def run_live_share_command(force: bool = False) -> Tuple[str, str]:
             return str(result.stdout), str(result.stderr)
         except subprocess.CalledProcessError as e:
             if e.stderr:
-                original_print(f"run_live_share_command: command failed with error: {e}, stderr: {e.stderr}")
+                print_debug(f"run_live_share_command: command failed with error: {e}, stderr: {e.stderr}")
             else:
-                original_print(f"run_live_share_command: command failed with error: {e}")
+                print_debug(f"run_live_share_command: command failed with error: {e}")
             return "", str(e.stderr)
         except Exception as e:
             print(f"run_live_share_command: An error occurred: {e}")
 
     return "", ""
 
-#def extract_and_print_qr(text: str) -> None:
-#    match = re.search(r"(https?://\S+|\b[\w.-]+@[\w.-]+\.\w+\b|\b\d{10,}\b)", text)
-#    if match:
-#        data = match.group(0)
-#        qr = qrcode.QRCode(box_size=1, error_correction=qrcode.constants.ERROR_CORRECT_L, border=0)
-#        qr.add_data(data)
-#        qr.make()
-#        qr.print_ascii(out=sys.stdout)
-
 def force_live_share() -> bool:
     if args.live_share:
         return live_share(True)
@@ -2040,6 +2096,8 @@ def force_live_share() -> bool:
     return False
 
 def live_share(force: bool = False, text_and_qr: bool = False) -> bool:
+    log_data()
+
     if not get_current_run_folder():
         print(f"live_share: get_current_run_folder was empty or false: {get_current_run_folder()}")
         return False
@@ -2052,25 +2110,24 @@ def live_share(force: bool = False, text_and_qr: bool = False) -> bool:
     if text_and_qr:
         if stderr:
             print_green(stderr)
-            #extract_and_print_qr(stderr)
         else:
-            print_red("This call should have shown the CURL, but didnt. Stderr: {stderr}, stdout: {stdout}")
+            if stderr and stdout:
+                print_red(f"This call should have shown the CURL, but didnt. Stderr: {stderr}, stdout: {stdout}")
+            elif stderr:
+                print_red(f"This call should have shown the CURL, but didnt. Stderr: {stderr}")
+            elif stdout:
+                print_red(f"This call should have shown the CURL, but didnt. Stdout: {stdout}")
+            else:
+                print_red("This call should have shown the CURL, but didnt.")
     if stdout:
         print_debug(f"live_share stdout: {stdout}")
 
     return True
 
 def init_live_share() -> bool:
-    with spinner("Initializing live share..."):
-        ret = live_share(True, True)
-
-        return ret
+    ret = live_share(True, True)
 
-async def start_periodic_live_share() -> None:
-    if args.live_share and not os.environ.get("CI"):
-        while True:
-            live_share(force=False)
-            time.sleep(30)
+    return ret
 
 def init_storage(db_url: str) -> None:
     init_engine_and_session_factory(url=db_url, force_init=True)
@@ -2096,29 +2153,33 @@ def try_saving_to_db() -> None:
         else:
             print_red("ax_client was not defined in try_saving_to_db")
             my_exit(101)
-        save_generation_strategy(global_gs)
+
+        if global_gs is not None:
+            save_generation_strategy(global_gs)
+        else:
+            print_red("Not saving generation strategy: global_gs was empty")
     except Exception as e:
         print_debug(f"Failed trying to save sqlite3-DB: {e}")
 
-def merge_with_job_infos(pd_frame: pd.DataFrame) -> pd.DataFrame:
+def merge_with_job_infos(df: pd.DataFrame) -> pd.DataFrame:
     job_infos_path = os.path.join(get_current_run_folder(), "job_infos.csv")
     if not os.path.exists(job_infos_path):
-        return pd_frame
+        return df
 
     job_df = pd.read_csv(job_infos_path)
 
-    if 'trial_index' not in pd_frame.columns or 'trial_index' not in job_df.columns:
+    if 'trial_index' not in df.columns or 'trial_index' not in job_df.columns:
         raise ValueError("Both DataFrames must contain a 'trial_index' column.")
 
-    job_df_filtered = job_df[job_df['trial_index'].isin(pd_frame['trial_index'])]
+    job_df_filtered = job_df[job_df['trial_index'].isin(df['trial_index'])]
 
-    new_cols = [col for col in job_df_filtered.columns if col != 'trial_index' and col not in pd_frame.columns]
+    new_cols = [col for col in job_df_filtered.columns if col != 'trial_index' and col not in df.columns]
 
     job_df_reduced = job_df_filtered[['trial_index'] + new_cols]
 
-    merged = pd.merge(pd_frame, job_df_reduced, on='trial_index', how='left')
+    merged = pd.merge(df, job_df_reduced, on='trial_index', how='left')
 
-    old_cols = [col for col in pd_frame.columns if col != 'trial_index']
+    old_cols = [col for col in df.columns if col != 'trial_index']
 
     new_order = ['trial_index'] + new_cols + old_cols
 
@@ -2126,48 +2187,9 @@ def merge_with_job_infos(pd_frame: pd.DataFrame) -> pd.DataFrame:
 
     return merged
 
-def reindex_trials(df: pd.DataFrame) -> pd.DataFrame:
-    """
-    Ensure trial_index is sequential and arm_name unique.
-    Keep arm_name unless all parameters except 'order', 'hostname', 'queue_time' match.
-    """
-    if "trial_index" not in df.columns or "arm_name" not in df.columns:
-        return df
-
-    # Sort by something stable (queue_time if available)
-    sort_cols = ["queue_time"] if "queue_time" in df.columns else df.columns.tolist()
-    df = df.sort_values(by=sort_cols, ignore_index=True)
-
-    # Mapping from "parameter signature" to assigned arm_name
-    seen_signatures = {}
-    new_arm_names = []
-
-    for new_idx, row in df.iterrows():
-        # Create signature without 'order', 'hostname', 'queue_time', 'trial_index', 'arm_name'
-        ignore_cols = {"order", "hostname", "queue_time", "trial_index", "arm_name"}
-        signature = tuple((col, row[col]) for col in df.columns if col not in ignore_cols)
-
-        if signature in seen_signatures:
-            # Collision → make a unique name
-            base_name = seen_signatures[signature]
-            suffix = 1
-            new_name = f"{base_name}_{suffix}"
-            while new_name in new_arm_names:
-                suffix += 1
-                new_name = f"{base_name}_{suffix}"
-            new_arm_names.append(new_name)
-        else:
-            # First occurrence → use new_idx as trial index in name
-            new_name = f"{new_idx}_0"
-            seen_signatures[signature] = f"{new_idx}_0"
-            new_arm_names.append(new_name)
-
-        df.at[new_idx, "trial_index"] = new_idx
-        df.at[new_idx, "arm_name"] = new_name
-
-    return df
-
 def save_results_csv() -> Optional[str]:
+    log_data()
+
     if args.dryrun:
         return None
 
@@ -2181,8 +2203,11 @@ def save_results_csv() -> Optional[str]:
     save_checkpoint()
 
     try:
-        pd_frame = fetch_and_prepare_trials()
-        write_csv(pd_frame, pd_csv)
+        df = fetch_and_prepare_trials()
+        if df is None:
+            print_red(f"save_results_csv: fetch_and_prepare_trials returned an empty element: {df}")
+            return None
+        write_csv(df, pd_csv)
         write_json_snapshot(pd_json)
         save_experiment_to_file()
 
@@ -2194,32 +2219,67 @@ def save_results_csv() -> Optional[str]:
     except (SignalUSR, SignalCONT, SignalINT) as e:
         raise type(e)(str(e)) from e
     except Exception as e:
-        print_red(f"While saving all trials as a pandas-dataframe-csv, an error occurred: {e}")
+        print_red(f"\nWhile saving all trials as a pandas-dataframe-csv, an error occurred: {e}")
 
     return pd_csv
 
 def get_results_paths() -> tuple[str, str]:
     return (get_current_run_folder(RESULTS_CSV_FILENAME), get_state_file_name('pd.json'))
 
-def fetch_and_prepare_trials() -> pd.DataFrame:
+def ax_client_get_trials_data_frame() -> Optional[pd.DataFrame]:
+    if not ax_client:
+        my_exit(101)
+
+        return None
+
+    return ax_client.get_trials_data_frame()
+
+def fetch_and_prepare_trials() -> Optional[pd.DataFrame]:
+    if not ax_client:
+        return None
+
     ax_client.experiment.fetch_data()
-    df = ax_client.get_trials_data_frame()
+    df = ax_client_get_trials_data_frame()
     df = merge_with_job_infos(df)
-    return reindex_trials(df)
 
-def write_csv(df, path: str) -> None:
+    return df
+
+def write_csv(df: pd.DataFrame, path: str) -> None:
+    try:
+        df = df.sort_values(by=["trial_index"], kind="stable").reset_index(drop=True)
+    except KeyError:
+        pass
     df.to_csv(path, index=False, float_format="%.30f")
 
-def write_json_snapshot(path: str) -> None:
+def ax_client_to_json_snapshot() -> Optional[dict]:
+    if not ax_client:
+        my_exit(101)
+
+        return None
+
     json_snapshot = ax_client.to_json_snapshot()
-    with open(path, "w", encoding="utf-8") as f:
-        json.dump(json_snapshot, f, indent=4)
+
+    return json_snapshot
+
+def write_json_snapshot(path: str) -> None:
+    if ax_client is not None:
+        json_snapshot = ax_client_to_json_snapshot()
+        if json_snapshot is not None:
+            with open(path, "w", encoding="utf-8") as f:
+                json.dump(json_snapshot, f, indent=4)
+        else:
+            print_debug('json_snapshot from ax_client_to_json_snapshot was None')
+    else:
+        print_red("write_json_snapshot: ax_client was None")
 
 def save_experiment_to_file() -> None:
-    save_experiment(
-        ax_client.experiment,
-        get_state_file_name("ax_client.experiment.json")
-    )
+    if ax_client is not None:
+        save_experiment(
+            ax_client.experiment,
+            get_state_file_name("ax_client.experiment.json")
+        )
+    else:
+        print_red("save_experiment: ax_client is None")
 
 def should_save_to_database() -> bool:
     return args.model not in uncontinuable_models and args.save_to_database
@@ -2408,7 +2468,7 @@ def set_nr_inserted_jobs(new_nr_inserted_jobs: int) -> None:
 
 def write_worker_usage() -> None:
     if len(WORKER_PERCENTAGE_USAGE):
-        csv_filename = get_current_run_folder('worker_usage.csv')
+        csv_filename = get_current_run_folder(worker_usage_file)
 
         csv_columns = ['time', 'num_parallel_jobs', 'nr_current_workers', 'percentage']
 
@@ -2418,35 +2478,39 @@ def write_worker_usage() -> None:
                 csv_writer.writerow(row)
     else:
         if is_slurm_job():
-            print_debug("WORKER_PERCENTAGE_USAGE seems to be empty. Not writing worker_usage.csv")
+            print_debug(f"WORKER_PERCENTAGE_USAGE seems to be empty. Not writing {worker_usage_file}")
 
 def log_system_usage() -> None:
+    global LAST_LOG_TIME
+
+    now = time.time()
+    if now - LAST_LOG_TIME < 30:
+        return
+
+    LAST_LOG_TIME = int(now)
+
     if not get_current_run_folder():
         return
 
     ram_cpu_csv_file_path = os.path.join(get_current_run_folder(), "cpu_ram_usage.csv")
-
     makedirs(os.path.dirname(ram_cpu_csv_file_path))
 
     file_exists = os.path.isfile(ram_cpu_csv_file_path)
 
-    with open(ram_cpu_csv_file_path, mode='a', newline='', encoding="utf-8") as file:
-        writer = csv.writer(file)
-
-        current_time = int(time.time())
-
-        if process is not None:
-            mem_proc = process.memory_info()
-
-            if mem_proc is not None:
-                ram_usage_mb = mem_proc.rss / (1024 * 1024)
-                cpu_usage_percent = psutil.cpu_percent(percpu=False)
+    mem_proc = process.memory_info() if process else None
+    if not mem_proc:
+        return
 
-                if ram_usage_mb > 0 and cpu_usage_percent > 0:
-                    if not file_exists:
-                        writer.writerow(["timestamp", "ram_usage_mb", "cpu_usage_percent"])
+    ram_usage_mb = mem_proc.rss / (1024 * 1024)
+    cpu_usage_percent = psutil.cpu_percent(percpu=False)
+    if ram_usage_mb <= 0 or cpu_usage_percent <= 0:
+        return
 
-                    writer.writerow([current_time, ram_usage_mb, cpu_usage_percent])
+    with open(ram_cpu_csv_file_path, mode='a', newline='', encoding="utf-8") as file:
+        writer = csv.writer(file)
+        if not file_exists:
+            writer.writerow(["timestamp", "ram_usage_mb", "cpu_usage_percent"])
+        writer.writerow([int(now), ram_usage_mb, cpu_usage_percent])
 
 def write_process_info() -> None:
     try:
@@ -2598,9 +2662,6 @@ def _debug_worker_creation(msg: str, _lvl: int = 0, eee: Union[None, str, Except
 def append_to_nvidia_smi_logs(_file: str, _host: str, result: str, _lvl: int = 0, eee: Union[None, str, Exception] = None) -> None:
     log_message_to_file(_file, result, _lvl, str(eee))
 
-def _debug_get_next_trials(msg: str, _lvl: int = 0, eee: Union[None, str, Exception] = None) -> None:
-    log_message_to_file(LOGFILE_DEBUG_GET_NEXT_TRIALS, msg, _lvl, str(eee))
-
 def _debug_progressbar(msg: str, _lvl: int = 0, eee: Union[None, str, Exception] = None) -> None:
     log_message_to_file(logfile_progressbar, msg, _lvl, str(eee))
 
@@ -2796,13 +2857,23 @@ def print_debug_get_next_trials(got: int, requested: int, _line: int) -> None:
     time_str: str = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
     msg: str = f"{time_str}, {got}, {requested}"
 
-    _debug_get_next_trials(msg)
+    log_message_to_file(LOGFILE_DEBUG_GET_NEXT_TRIALS, msg, 0, "")
 
 def print_debug_progressbar(msg: str) -> None:
-    time_str = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-    msg = f"{time_str} ({worker_generator_uuid}): {msg}"
+    global last_msg_progressbar, last_msg_raw
+
+    try:
+        with last_lock_print_debug:
+            if msg != last_msg_raw:
+                time_str = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+                full_msg = f"{time_str} ({worker_generator_uuid}): {msg}"
+
+                _debug_progressbar(full_msg)
 
-    _debug_progressbar(msg)
+                last_msg_raw = msg
+                last_msg_progressbar = full_msg
+    except Exception as e:
+        print(f"Error in print_debug_progressbar: {e}", flush=True)
 
 def get_process_info(pid: Any) -> str:
     try:
@@ -3307,116 +3378,438 @@ def parse_experiment_parameters() -> None:
     # Remove duplicates by 'name' key preserving order
     params = list({p['name']: p for p in params}.values())
 
-    experiment_parameters = params
+    experiment_parameters = params # type: ignore[assignment]
 
-def check_factorial_range() -> None:
-    if args.model and args.model == "FACTORIAL":
-        _fatal_error("\n⚠ --model FACTORIAL cannot be used with range parameter", 181)
+def job_calculate_pareto_front(path_to_calculate: str, disable_sixel_and_table: bool = False) -> bool:
+    pf_start_time = time.time()
 
-def check_if_range_types_are_invalid(value_type: str, valid_value_types: list) -> None:
-    if value_type not in valid_value_types:
-        valid_value_types_string = ", ".join(valid_value_types)
-        _fatal_error(f"⚠ {value_type} is not a valid value type. Valid types for range are: {valid_value_types_string}", 181)
+    if not path_to_calculate:
+        return False
 
-def check_range_params_length(this_args: Union[str, list]) -> None:
-    if len(this_args) != 5 and len(this_args) != 4 and len(this_args) != 6:
-        _fatal_error("\n⚠ --parameter for type range must have 4 (or 5, the last one being optional and float by default, or 6, while the last one is true or false) parameters: <NAME> range <START> <END> (<TYPE (int or float)>, <log_scale: bool>)", 181)
+    global CURRENT_RUN_FOLDER
+    global RESULT_CSV_FILE
+    global arg_result_names
 
-def die_if_lower_and_upper_bound_equal_zero(lower_bound: Union[int, float], upper_bound: Union[int, float]) -> None:
-    if upper_bound is None or lower_bound is None:
-        _fatal_error("die_if_lower_and_upper_bound_equal_zero: upper_bound or lower_bound is None. Cannot continue.", 91)
-    if upper_bound == lower_bound:
-        if lower_bound == 0:
-            _fatal_error(f"⚠ Lower bound and upper bound are equal: {lower_bound}, cannot automatically fix this, because they -0 = +0 (usually a quickfix would be to set lower_bound = -upper_bound)", 181)
-        print_red(f"⚠ Lower bound and upper bound are equal: {lower_bound}, setting lower_bound = -upper_bound")
-        if upper_bound is not None:
-            lower_bound = -upper_bound
+    if not path_to_calculate:
+        print_red("Can only calculate pareto front of previous job when --calculate_pareto_front_of_job is set")
+        return False
 
-def format_value(value: Any, float_format: str = '.80f') -> str:
-    try:
-        if isinstance(value, float):
-            s = format(value, float_format)
-            s = s.rstrip('0').rstrip('.') if '.' in s else s
-            return s
-        return str(value)
-    except Exception as e:
-        print_red(f"⚠ Error formatting the number {value}: {e}")
-        return str(value)
+    if not os.path.exists(path_to_calculate):
+        print_red(f"Path '{path_to_calculate}' does not exist")
+        return False
 
-def replace_parameters_in_string(
-    parameters: dict,
-    input_string: str,
-    float_format: str = '.20f',
-    additional_prefixes: list[str] = [],
-    additional_patterns: list[str] = [],
-) -> str:
-    try:
-        prefixes = ['$', '%'] + additional_prefixes
-        patterns = ['{key}', '({key})'] + additional_patterns
+    ax_client_json = f"{path_to_calculate}/state_files/ax_client.experiment.json"
 
-        for key, value in parameters.items():
-            replacement = format_value(value, float_format=float_format)
-            for prefix in prefixes:
-                for pattern in patterns:
-                    token = prefix + pattern.format(key=key)
-                    input_string = input_string.replace(token, replacement)
+    if not os.path.exists(ax_client_json):
+        print_red(f"Path '{ax_client_json}' not found")
+        return False
 
-        input_string = input_string.replace('\r', ' ').replace('\n', ' ')
-        return input_string
+    checkpoint_file: str = f"{path_to_calculate}/state_files/checkpoint.json"
+    if not os.path.exists(checkpoint_file):
+        print_red(f"The checkpoint file '{checkpoint_file}' does not exist")
+        return False
 
-    except Exception as e:
-        print_red(f"\n⚠ Error: {e}")
-        return ""
+    RESULT_CSV_FILE = f"{path_to_calculate}/{RESULTS_CSV_FILENAME}"
+    if not os.path.exists(RESULT_CSV_FILE):
+        print_red(f"{RESULT_CSV_FILE} not found")
+        return False
 
-def get_memory_usage() -> float:
-    user_uid = os.getuid()
+    res_names = []
 
-    memory_usage = float(sum(
-        p.memory_info().rss for p in psutil.process_iter(attrs=['memory_info', 'uids'])
-        if p.info['uids'].real == user_uid
-    ) / (1024 * 1024))
+    res_names_file = f"{path_to_calculate}/result_names.txt"
+    if not os.path.exists(res_names_file):
+        print_red(f"File '{res_names_file}' does not exist")
+        return False
 
-    return memory_usage
+    try:
+        with open(res_names_file, "r", encoding="utf-8") as file:
+            lines = file.readlines()
+    except Exception as e:
+        print_red(f"Error reading file '{res_names_file}': {e}")
+        return False
 
-class MonitorProcess:
-    def __init__(self: Any, pid: int, interval: float = 1.0) -> None:
-        self.pid = pid
-        self.interval = interval
-        self.running = True
-        self.thread = threading.Thread(target=self._monitor)
-        self.thread.daemon = True
+    for line in lines:
+        entry = line.strip()
+        if entry != "":
+            res_names.append(entry)
 
-        fool_linter(f"self.thread.daemon was set to {self.thread.daemon}")
+    if len(res_names) < 2:
+        print_red(f"Error: There are less than 2 result names (is: {len(res_names)}, {', '.join(res_names)}) in {path_to_calculate}. Cannot continue calculating the pareto front.")
+        return False
 
-    def _monitor(self: Any) -> None:
-        try:
-            _internal_process = psutil.Process(self.pid)
-            while self.running and _internal_process.is_running():
-                crf = get_current_run_folder()
+    load_username_to_args(path_to_calculate)
 
-                if crf and crf != "":
-                    log_file_path = os.path.join(crf, "eval_nodes_cpu_ram_logs.txt")
+    CURRENT_RUN_FOLDER = path_to_calculate
 
-                    os.makedirs(os.path.dirname(log_file_path), exist_ok=True)
+    arg_result_names = res_names
 
-                    with open(log_file_path, mode="a", encoding="utf-8") as log_file:
-                        hostname = socket.gethostname()
+    load_experiment_parameters_from_checkpoint_file(checkpoint_file, False)
 
-                        slurm_job_id = os.getenv("SLURM_JOB_ID")
+    if experiment_parameters is None:
+        return False
 
-                        if slurm_job_id:
-                            hostname += f"-SLURM-ID-{slurm_job_id}"
+    show_pareto_or_error_msg(path_to_calculate, res_names, disable_sixel_and_table)
 
-                        total_memory = psutil.virtual_memory().total / (1024 * 1024)
-                        cpu_usage = psutil.cpu_percent(interval=5)
+    pf_end_time = time.time()
 
-                        memory_usage = get_memory_usage()
+    print_debug(f"Calculating the Pareto-front took {pf_end_time - pf_start_time} seconds")
 
-                        unix_timestamp = int(time.time())
+    return True
 
-                        log_file.write(f"\nUnix-Timestamp: {unix_timestamp}, Hostname: {hostname}, CPU: {cpu_usage:.2f}%, RAM: {memory_usage:.2f} MB / {total_memory:.2f} MB\n")
-                time.sleep(self.interval)
-        except psutil.NoSuchProcess:
+def show_pareto_or_error_msg(path_to_calculate: str, res_names: list = arg_result_names, disable_sixel_and_table: bool = False) -> None:
+    if args.dryrun:
+        print_debug("Not showing Pareto-frontier data with --dryrun")
+        return None
+
+    if len(res_names) > 1:
+        try:
+            show_pareto_frontier_data(path_to_calculate, res_names, disable_sixel_and_table)
+        except Exception as e:
+            inner_tb = ''.join(traceback.format_exception(type(e), e, e.__traceback__))
+            print_red(f"show_pareto_frontier_data() failed with exception '{e}':\n{inner_tb}")
+    else:
+        print_debug(f"show_pareto_frontier_data will NOT be executed because len(arg_result_names) is {len(arg_result_names)}")
+    return None
+
+def get_pareto_front_data(path_to_calculate: str, res_names: list) -> dict:
+    pareto_front_data: dict = {}
+
+    all_combinations = list(combinations(range(len(arg_result_names)), 2))
+
+    skip = False
+
+    for i, j in all_combinations:
+        if not skip:
+            metric_x = arg_result_names[i]
+            metric_y = arg_result_names[j]
+
+            x_minimize = get_result_minimize_flag(path_to_calculate, metric_x)
+            y_minimize = get_result_minimize_flag(path_to_calculate, metric_y)
+
+            try:
+                if metric_x not in pareto_front_data:
+                    pareto_front_data[metric_x] = {}
+
+                pareto_front_data[metric_x][metric_y] = get_calculated_frontier(path_to_calculate, metric_x, metric_y, x_minimize, y_minimize, res_names)
+            except ax.exceptions.core.DataRequiredError as e:
+                print_red(f"Error computing Pareto frontier for {metric_x} and {metric_y}: {e}")
+            except SignalINT:
+                print_red("Calculating Pareto-fronts was cancelled by pressing CTRL-c")
+                skip = True
+
+    return pareto_front_data
+
+def pareto_front_transform_objectives(
+    points: List[Tuple[Any, float, float]],
+    primary_name: str,
+    secondary_name: str
+) -> Tuple[np.ndarray, np.ndarray]:
+    primary_idx = arg_result_names.index(primary_name)
+    secondary_idx = arg_result_names.index(secondary_name)
+
+    x = np.array([p[1] for p in points])
+    y = np.array([p[2] for p in points])
+
+    if arg_result_min_or_max[primary_idx] == "max":
+        x = -x
+    elif arg_result_min_or_max[primary_idx] != "min":
+        raise ValueError(f"Unknown mode for {primary_name}: {arg_result_min_or_max[primary_idx]}")
+
+    if arg_result_min_or_max[secondary_idx] == "max":
+        y = -y
+    elif arg_result_min_or_max[secondary_idx] != "min":
+        raise ValueError(f"Unknown mode for {secondary_name}: {arg_result_min_or_max[secondary_idx]}")
+
+    return x, y
+
+def get_pareto_frontier_points(
+    path_to_calculate: str,
+    primary_objective: str,
+    secondary_objective: str,
+    x_minimize: bool,
+    y_minimize: bool,
+    absolute_metrics: List[str],
+    num_points: int
+) -> Optional[dict]:
+    records = pareto_front_aggregate_data(path_to_calculate)
+
+    if records is None:
+        return None
+
+    points = pareto_front_filter_complete_points(path_to_calculate, records, primary_objective, secondary_objective)
+    x, y = pareto_front_transform_objectives(points, primary_objective, secondary_objective)
+    selected_points = pareto_front_select_pareto_points(x, y, x_minimize, y_minimize, points, num_points)
+    result = pareto_front_build_return_structure(path_to_calculate, selected_points, records, absolute_metrics, primary_objective, secondary_objective)
+
+    return result
+
+def pareto_front_table_read_csv() -> List[Dict[str, str]]:
+    with open(RESULT_CSV_FILE, mode="r", encoding="utf-8", newline="") as f:
+        return list(csv.DictReader(f))
+
+def create_pareto_front_table(idxs: List[int], metric_x: str, metric_y: str) -> Table:
+    table = Table(title=f"Pareto-Front for {metric_y}/{metric_x}:", show_lines=True)
+
+    rows = pareto_front_table_read_csv()
+    if not rows:
+        table.add_column("No data found")
+        return table
+
+    filtered_rows = pareto_front_table_filter_rows(rows, idxs)
+    if not filtered_rows:
+        table.add_column("No matching entries")
+        return table
+
+    param_cols, result_cols = pareto_front_table_get_columns(filtered_rows[0])
+
+    pareto_front_table_add_headers(table, param_cols, result_cols)
+    pareto_front_table_add_rows(table, filtered_rows, param_cols, result_cols)
+
+    return table
+
+def pareto_front_build_return_structure(
+    path_to_calculate: str,
+    selected_points: List[Tuple[Any, float, float]],
+    records: Dict[Tuple[int, str], Dict[str, Dict[str, float]]],
+    absolute_metrics: List[str],
+    primary_name: str,
+    secondary_name: str
+) -> dict:
+    results_csv_file = f"{path_to_calculate}/{RESULTS_CSV_FILENAME}"
+    result_names_file = f"{path_to_calculate}/result_names.txt"
+
+    with open(result_names_file, mode="r", encoding="utf-8") as f:
+        result_names = [line.strip() for line in f if line.strip()]
+
+    csv_rows = {}
+    with open(results_csv_file, mode="r", encoding="utf-8", newline='') as csvfile:
+        reader = csv.DictReader(csvfile)
+        for row in reader:
+            trial_index = int(row['trial_index'])
+            csv_rows[trial_index] = row
+
+    ignored_columns = {'trial_index', 'arm_name', 'trial_status', 'generation_node'}
+    ignored_columns.update(result_names)
+
+    param_dicts = []
+    idxs = []
+    means_dict = defaultdict(list)
+
+    for (trial_index, arm_name), _, _ in selected_points:
+        row = csv_rows.get(trial_index, {})
+        if row == {} or row is None or row['arm_name'] != arm_name:
+            continue
+
+        idxs.append(int(row["trial_index"]))
+
+        param_dict: dict[str, int | float | str] = {}
+        for key, value in row.items():
+            if key not in ignored_columns:
+                try:
+                    param_dict[key] = int(value)
+                except ValueError:
+                    try:
+                        param_dict[key] = float(value)
+                    except ValueError:
+                        param_dict[key] = value
+
+        param_dicts.append(param_dict)
+
+        for metric in absolute_metrics:
+            means_dict[metric].append(records[(trial_index, arm_name)]['means'].get(metric, float("nan")))
+
+    ret = {
+        primary_name: {
+            secondary_name: {
+                "absolute_metrics": absolute_metrics,
+                "param_dicts": param_dicts,
+                "means": dict(means_dict),
+                "idxs": idxs
+            },
+            "absolute_metrics": absolute_metrics
+        }
+    }
+
+    return ret
+
+def pareto_front_aggregate_data(path_to_calculate: str) -> Optional[Dict[Tuple[int, str], Dict[str, Dict[str, float]]]]:
+    results_csv_file = f"{path_to_calculate}/{RESULTS_CSV_FILENAME}"
+    result_names_file = f"{path_to_calculate}/result_names.txt"
+
+    if not os.path.exists(results_csv_file) or not os.path.exists(result_names_file):
+        return None
+
+    with open(result_names_file, mode="r", encoding="utf-8") as f:
+        result_names = [line.strip() for line in f if line.strip()]
+
+    records: dict = defaultdict(lambda: {'means': {}})
+
+    with open(results_csv_file, encoding="utf-8", mode="r", newline='') as csvfile:
+        reader = csv.DictReader(csvfile)
+        for row in reader:
+            trial_index = int(row['trial_index'])
+            arm_name = row['arm_name']
+            key = (trial_index, arm_name)
+
+            for metric in result_names:
+                if metric in row:
+                    try:
+                        records[key]['means'][metric] = float(row[metric])
+                    except ValueError:
+                        continue
+
+    return records
+
+def plot_pareto_frontier_sixel(data: Any, x_metric: str, y_metric: str) -> None:
+    if data is None:
+        print("[italic yellow]The data seems to be empty. Cannot plot pareto frontier.[/]")
+        return
+
+    if not supports_sixel():
+        print(f"[italic yellow]Your console does not support sixel-images. Will not print Pareto-frontier as a matplotlib-sixel-plot for {x_metric}/{y_metric}.[/]")
+        return
+
+    import matplotlib.pyplot as plt
+
+    means = data[x_metric][y_metric]["means"]
+
+    x_values = means[x_metric]
+    y_values = means[y_metric]
+
+    fig, _ax = plt.subplots()
+
+    _ax.scatter(x_values, y_values, s=50, marker='x', c='blue', label='Data Points')
+
+    _ax.set_xlabel(x_metric)
+    _ax.set_ylabel(y_metric)
+
+    _ax.set_title(f'Pareto-Front {x_metric}/{y_metric}')
+
+    _ax.ticklabel_format(style='plain', axis='both', useOffset=False)
+
+    with tempfile.NamedTemporaryFile(suffix=".png", delete=True) as tmp_file:
+        plt.savefig(tmp_file.name, dpi=300)
+
+        print_image_to_cli(tmp_file.name, 1000)
+
+    plt.close(fig)
+
+def pareto_front_table_get_columns(first_row: Dict[str, str]) -> Tuple[List[str], List[str]]:
+    all_columns = list(first_row.keys())
+    ignored_cols = set(special_col_names) - {"trial_index"}
+
+    param_cols = [col for col in all_columns if col not in ignored_cols and col not in arg_result_names and not col.startswith("OO_Info_")]
+    result_cols = [col for col in arg_result_names if col in all_columns]
+    return param_cols, result_cols
+
+def check_factorial_range() -> None:
+    if args.model and args.model == "FACTORIAL":
+        _fatal_error("\n⚠ --model FACTORIAL cannot be used with range parameter", 181)
+
+def check_if_range_types_are_invalid(value_type: str, valid_value_types: list) -> None:
+    if value_type not in valid_value_types:
+        valid_value_types_string = ", ".join(valid_value_types)
+        _fatal_error(f"⚠ {value_type} is not a valid value type. Valid types for range are: {valid_value_types_string}", 181)
+
+def check_range_params_length(this_args: Union[str, list]) -> None:
+    if len(this_args) != 5 and len(this_args) != 4 and len(this_args) != 6:
+        _fatal_error("\n⚠ --parameter for type range must have 4 (or 5, the last one being optional and float by default, or 6, while the last one is true or false) parameters: <NAME> range <START> <END> (<TYPE (int or float)>, <log_scale: bool>)", 181)
+
+def die_if_lower_and_upper_bound_equal_zero(lower_bound: Union[int, float], upper_bound: Union[int, float]) -> None:
+    if upper_bound is None or lower_bound is None:
+        _fatal_error("die_if_lower_and_upper_bound_equal_zero: upper_bound or lower_bound is None. Cannot continue.", 91)
+    if upper_bound == lower_bound:
+        if lower_bound == 0:
+            _fatal_error(f"⚠ Lower bound and upper bound are equal: {lower_bound}, cannot automatically fix this, because they -0 = +0 (usually a quickfix would be to set lower_bound = -upper_bound)", 181)
+        print_red(f"⚠ Lower bound and upper bound are equal: {lower_bound}, setting lower_bound = -upper_bound")
+        if upper_bound is not None:
+            lower_bound = -upper_bound
+
+def format_value(value: Any, float_format: str = '.80f') -> str:
+    try:
+        if isinstance(value, float):
+            s = format(value, float_format)
+            s = s.rstrip('0').rstrip('.') if '.' in s else s
+            return s
+        return str(value)
+    except Exception as e:
+        print_red(f"⚠ Error formatting the number {value}: {e}")
+        return str(value)
+
+def replace_parameters_in_string(
+    parameters: dict,
+    input_string: str,
+    float_format: str = '.20f',
+    additional_prefixes: list[str] = [],
+    additional_patterns: list[str] = [],
+) -> str:
+    try:
+        prefixes = ['$', '%'] + additional_prefixes
+        patterns = ['{' + 'key' + '}', '(' + '{' + 'key' + '}' + ')'] + additional_patterns
+
+        for key, value in parameters.items():
+            replacement = format_value(value, float_format=float_format)
+            for prefix in prefixes:
+                for pattern in patterns:
+                    token = prefix + pattern.format(key=key)
+                    input_string = input_string.replace(token, replacement)
+
+        input_string = input_string.replace('\r', ' ').replace('\n', ' ')
+        return input_string
+
+    except Exception as e:
+        print_red(f"\n⚠ Error: {e}")
+        return ""
+
+def get_memory_usage() -> float:
+    user_uid = os.getuid()
+
+    memory_usage = float(sum(
+        p.memory_info().rss for p in psutil.process_iter(attrs=['memory_info', 'uids'])
+        if p.info['uids'].real == user_uid
+    ) / (1024 * 1024))
+
+    return memory_usage
+
+class MonitorProcess:
+    def __init__(self: Any, pid: int, interval: float = 1.0) -> None:
+        self.pid = pid
+        self.interval = interval
+        self.running = True
+        self.thread = threading.Thread(target=self._monitor)
+        self.thread.daemon = True
+
+        fool_linter(f"self.thread.daemon was set to {self.thread.daemon}")
+
+    def _monitor(self: Any) -> None:
+        try:
+            _internal_process = psutil.Process(self.pid)
+            while self.running and _internal_process.is_running():
+                crf = get_current_run_folder()
+
+                if crf and crf != "":
+                    log_file_path = os.path.join(crf, "eval_nodes_cpu_ram_logs.txt")
+
+                    os.makedirs(os.path.dirname(log_file_path), exist_ok=True)
+
+                    with open(log_file_path, mode="a", encoding="utf-8") as log_file:
+                        hostname = socket.gethostname()
+
+                        slurm_job_id = os.getenv("SLURM_JOB_ID")
+
+                        if slurm_job_id:
+                            hostname += f"-SLURM-ID-{slurm_job_id}"
+
+                        total_memory = psutil.virtual_memory().total / (1024 * 1024)
+                        cpu_usage = psutil.cpu_percent(interval=5)
+
+                        memory_usage = get_memory_usage()
+
+                        unix_timestamp = int(time.time())
+
+                        log_file.write(f"\nUnix-Timestamp: {unix_timestamp}, Hostname: {hostname}, CPU: {cpu_usage:.2f}%, RAM: {memory_usage:.2f} MB / {total_memory:.2f} MB\n")
+                time.sleep(self.interval)
+        except psutil.NoSuchProcess:
             pass
 
     def __enter__(self: Any) -> None:
@@ -3560,7 +3953,7 @@ def _add_to_csv_acquire_lock(lockfile: str, dir_path: str) -> bool:
             time.sleep(wait_time)
             max_wait -= wait_time
         except Exception as e:
-            print("Lock error:", e)
+            print_red(f"Lock error: {e}")
             return False
     return False
 
@@ -3598,11 +3991,11 @@ def _add_to_csv_rewrite_file(file_path: str, rows: List[list], existing_heading:
         writer = csv.writer(tmp_file)
         writer.writerow(all_headings)
         for row in rows[1:]:
-            tmp_file.writerow([
+            tmp_file.writerow([ # type: ignore[attr-defined]
                 row[existing_heading.index(h)] if h in existing_heading else ""
                 for h in all_headings
             ])
-        tmp_file.writerow([
+        tmp_file.writerow([ # type: ignore[attr-defined]
             formatted_data[new_heading.index(h)] if h in new_heading else ""
             for h in all_headings
         ])
@@ -3633,12 +4026,12 @@ def find_file_paths(_text: str) -> List[str]:
 def check_file_info(file_path: str) -> str:
     if not os.path.exists(file_path):
         if not args.tests:
-            print(f"check_file_info: The file {file_path} does not exist.")
+            print_red(f"check_file_info: The file {file_path} does not exist.")
         return ""
 
     if not os.access(file_path, os.R_OK):
         if not args.tests:
-            print(f"check_file_info: The file {file_path} is not readable.")
+            print_red(f"check_file_info: The file {file_path} is not readable.")
         return ""
 
     file_stat = os.stat(file_path)
@@ -3698,7 +4091,7 @@ def write_failed_logs(data_dict: Optional[dict], error_description: str = "") ->
         data = [list(data_dict.values())]
     else:
         print_debug("No data_dict provided, writing only error description.")
-        data = [[]]  # leeres Datenfeld, nur error_description kommt dazu
+        data = [[]]
 
     if error_description:
         headers.append('error_description')
@@ -3752,7 +4145,7 @@ def count_defective_nodes(file_path: Union[str, None] = None, entry: Any = None)
         return sorted(set(entries))
 
     except Exception as e:
-        print(f"An error has occurred: {e}")
+        print_red(f"An error has occurred: {e}")
         return []
 
 def test_gpu_before_evaluate(return_in_case_of_error: dict) -> Union[None, dict]:
@@ -3763,7 +4156,7 @@ def test_gpu_before_evaluate(return_in_case_of_error: dict) -> Union[None, dict]
 
                 fool_linter(tmp)
         except RuntimeError:
-            print(f"Node {socket.gethostname()} was detected as faulty. It should have had a GPU, but there is an error initializing the CUDA driver. Adding this node to the --exclude list.")
+            print_red(f"Node {socket.gethostname()} was detected as faulty. It should have had a GPU, but there is an error initializing the CUDA driver. Adding this node to the --exclude list.")
             count_defective_nodes(None, socket.gethostname())
             return return_in_case_of_error
         except Exception:
@@ -3937,7 +4330,7 @@ def _write_job_infos_csv_build_headline(parameters_keys: List[str], extra_vars_n
         "run_time",
         "program_string",
         *parameters_keys,
-        *arg_result_names,  # arg_result_names muss global definiert sein
+        *arg_result_names,
         "exit_code",
         "signal",
         "hostname",
@@ -4234,6 +4627,8 @@ def evaluate(parameters_with_trial_index: dict) -> Optional[Union[int, float, Di
     trial_index = parameters_with_trial_index["trial_idx"]
     submit_time = parameters_with_trial_index["submit_time"]
 
+    print(f'Trial-Index: {trial_index}')
+
     queue_time = abs(int(time.time()) - int(submit_time))
 
     start_nvidia_smi_thread()
@@ -4471,7 +4866,7 @@ def replace_string_with_params(input_string: str, params: list) -> str:
         return replaced_string
     except AssertionError as e:
         error_text = f"Error in replace_string_with_params: {e}"
-        print(error_text)
+        print_red(error_text)
         raise
 
     return ""
@@ -4748,9 +5143,7 @@ def get_sixel_graphics_data(_pd_csv: str, _force: bool = False) -> list:
             _params = [_command, plot, _tmp, plot_type, tmp_file, _width]
             data.append(_params)
         except Exception as e:
-            tb = traceback.format_exc()
-            print_red(f"Error trying to print {plot_type} to CLI: {e}, {tb}")
-            print_debug(f"Error trying to print {plot_type} to CLI: {e}")
+            print_red(f"Error trying to print {plot_type} to CLI: {e}")
 
     return data
 
@@ -4858,7 +5251,7 @@ def process_best_result(res_name: str, print_to_file: bool) -> int:
 
     if str(best_result) in [NO_RESULT, None, "None"]:
         print_red(f"Best {res_name} could not be determined")
-        return 87
+        return 87 # exit-code: 87
 
     total_str = f"total: {_count_done_jobs(RESULT_CSV_FILE) - NR_INSERTED_JOBS}"
     if NR_INSERTED_JOBS:
@@ -4941,15 +5334,17 @@ def abandon_job(job: Job, trial_index: int, reason: str) -> bool:
     if job:
         try:
             if ax_client:
-                _trial = ax_client.get_trial(trial_index)
-                _trial.mark_abandoned(reason=reason)
+                _trial = get_ax_client_trial(trial_index)
+                if _trial is None:
+                    return False
+
+                mark_abandoned(_trial, reason, trial_index)
                 print_debug(f"abandon_job: removing job {job}, trial_index: {trial_index}")
                 global_vars["jobs"].remove((job, trial_index))
             else:
-                _fatal_error("ax_client could not be found", 9)
+                _fatal_error("ax_client could not be found", 101)
         except Exception as e:
-            print(f"ERROR in line {get_line_info()}: {e}")
-            print_debug(f"ERROR in line {get_line_info()}: {e}")
+            print_red(f"ERROR in line {get_line_info()}: {e}")
             return False
         job.cancel()
         return True
@@ -4962,23 +5357,120 @@ def abandon_all_jobs() -> None:
         if not abandoned:
             print_debug(f"Job {job} could not be abandoned.")
 
-def show_pareto_or_error_msg(path_to_calculate: str, res_names: list = arg_result_names, disable_sixel_and_table: bool = False) -> None:
-    if args.dryrun:
-        print_debug("Not showing Pareto-frontier data with --dryrun")
+def write_result_to_trace_file(res: str) -> bool:
+    if res is None:
+        sys.stderr.write("Provided result is None, nothing to write\n")
+        return False
+
+    target_folder = get_current_run_folder()
+    target_file = os.path.join(target_folder, "optimization_trace.html")
+
+    try:
+        file_handle = open(target_file, "w", encoding="utf-8")
+    except OSError as error:
+        sys.stderr.write("Unable to open target file for writing\n")
+        sys.stderr.write(str(error) + "\n")
+        return False
+
+    try:
+        written = file_handle.write(str(res))
+        file_handle.flush()
+
+        if written == 0:
+            sys.stderr.write("No data was written to the file\n")
+            file_handle.close()
+            return False
+    except Exception as error:
+        sys.stderr.write("Error occurred while writing to file\n")
+        sys.stderr.write(str(error) + "\n")
+        file_handle.close()
+        return False
+
+    try:
+        file_handle.close()
+    except Exception as error:
+        sys.stderr.write("Failed to properly close file\n")
+        sys.stderr.write(str(error) + "\n")
+        return False
+
+    return True
+
+def render(plot_config: AxPlotConfig) -> None:
+    if plot_config is None or "data" not in plot_config:
         return None
 
-    if len(res_names) > 1:
-        try:
-            show_pareto_frontier_data(path_to_calculate, res_names, disable_sixel_and_table)
-        except Exception as e:
-            print_red(f"show_pareto_frontier_data() failed with exception '{e}'")
-    else:
-        print_debug(f"show_pareto_frontier_data will NOT be executed because len(arg_result_names) is {len(arg_result_names)}")
+    res: str = plot_config.data # type: ignore
+
+    repair_funcs = """
+function decodeBData(obj) {
+        if (!obj || typeof obj !== "object") {
+            return obj;
+        }
+
+        if (obj.bdata && obj.dtype) {
+            var binary_string = atob(obj.bdata);
+            var len = binary_string.length;
+            var bytes = new Uint8Array(len);
+
+            for (var i = 0; i < len; i++) {
+                bytes[i] = binary_string.charCodeAt(i);
+            }
+
+            switch (obj.dtype) {
+                case "i1": return Array.from(new Int8Array(bytes.buffer));
+                case "i2": return Array.from(new Int16Array(bytes.buffer));
+                case "i4": return Array.from(new Int32Array(bytes.buffer));
+                case "f4": return Array.from(new Float32Array(bytes.buffer));
+                case "f8": return Array.from(new Float64Array(bytes.buffer));
+                default:
+                    console.error("Unknown dtype:", obj.dtype);
+                    return [];
+            }
+        }
+
+        return obj;
+}
+
+function repairTraces(traces) {
+        var fixed = [];
+
+        for (var i = 0; i < traces.length; i++) {
+            var t = traces[i];
+
+            if (t.x) {
+                t.x = decodeBData(t.x);
+            }
+
+            if (t.y) {
+                t.y = decodeBData(t.y);
+            }
+
+            fixed.push(t);
+        }
+
+        return fixed;
+}
+    """
+
+    res = str(res)
+
+    res = f"<div id='plot' style='width:100%;height:600px;'></div>\n<script type='text/javascript' src='https://cdn.plot.ly/plotly-latest.min.js'></script><script>{repair_funcs}\nconst True = true;\nconst False = false;\nconst data = {res};\ndata.data = repairTraces(data.data);\nPlotly.newPlot(document.getElementById('plot'), data.data, data.layout);</script>"
+
+    write_result_to_trace_file(res)
+
     return None
 
 def end_program(_force: Optional[bool] = False, exit_code: Optional[int] = None) -> None:
     global END_PROGRAM_RAN
 
+    #dier(global_gs.current_node.generator_specs[0]._fitted_adapter.generator._surrogate.training_data[0].X)
+    #dier(global_gs.current_node.generator_specs[0]._fitted_adapter.generator._surrogate.training_data[0].Y)
+    #dier(global_gs.current_node.generator_specs[0]._fitted_adapter.generator._surrogate.outcomes)
+
+    if ax_client is not None:
+        if len(arg_result_names) == 1:
+            render(ax_client.get_optimization_trace())
+
     wait_for_jobs_to_complete()
 
     show_pareto_or_error_msg(get_current_run_folder(), arg_result_names)
@@ -5012,7 +5504,7 @@ def end_program(_force: Optional[bool] = False, exit_code: Optional[int] = None)
             _exit = new_exit
     except (SignalUSR, SignalINT, SignalCONT, KeyboardInterrupt):
         print_red("\n⚠ You pressed CTRL+C or a signal was sent. Program execution halted while ending program.")
-        print("\n⚠ KeyboardInterrupt signal was sent. Ending program will still run.")
+        print_red("\n⚠ KeyboardInterrupt signal was sent. Ending program will still run.")
         new_exit = show_end_table_and_save_end_files()
         if new_exit > 0:
             _exit = new_exit
@@ -5021,8 +5513,6 @@ def end_program(_force: Optional[bool] = False, exit_code: Optional[int] = None)
 
     abandon_all_jobs()
 
-    save_results_csv()
-
     if exit_code:
         _exit = exit_code
 
@@ -5035,21 +5525,31 @@ def end_program(_force: Optional[bool] = False, exit_code: Optional[int] = None)
 
     my_exit(_exit)
 
+def save_ax_client_to_json_file(checkpoint_filepath: str) -> None:
+    if not ax_client:
+        my_exit(101)
+
+        return None
+
+    ax_client.save_to_json_file(checkpoint_filepath)
+
+    return None
+
 def save_checkpoint(trial_nr: int = 0, eee: Union[None, str, Exception] = None) -> None:
     if trial_nr > 3:
         if eee:
-            print(f"Error during saving checkpoint: {eee}")
+            print_red(f"Error during saving checkpoint: {eee}")
         else:
-            print("Error during saving checkpoint")
+            print_red("Error during saving checkpoint")
         return
 
     try:
         checkpoint_filepath = get_state_file_name('checkpoint.json')
 
         if ax_client:
-            ax_client.save_to_json_file(filepath=checkpoint_filepath)
+            save_ax_client_to_json_file(checkpoint_filepath)
         else:
-            _fatal_error("Something went wrong using the ax_client", 9)
+            _fatal_error("Something went wrong using the ax_client", 101)
     except Exception as e:
         save_checkpoint(trial_nr + 1, e)
 
@@ -5210,7 +5710,7 @@ def parse_equation_item(comparer_found: bool, item: str, parsed: list, parsed_or
         })
     elif item in [">=", "<="]:
         if comparer_found:
-            print("There is already one comparison operator! Cannot have more than one in an equation!")
+            print_red("There is already one comparison operator! Cannot have more than one in an equation!")
             return_totally = True
         comparer_found = True
 
@@ -5421,20 +5921,9 @@ def check_equation(variables: list, equation: str) -> Union[str, bool]:
 def set_objectives() -> dict:
     objectives = {}
 
-    for rn in args.result_names:
-        key, value = "", ""
-
-        if "=" in rn:
-            key, value = rn.split('=', 1)
-        else:
-            key = rn
-            value = ""
-
-        if value not in ["min", "max"]:
-            if value:
-                print_yellow(f"Value '{value}' for --result_names {rn} is not a valid value. Must be min or max. Will be set to min.")
-
-            value = "min"
+    k = 0
+    for key in arg_result_names:
+        value = arg_result_min_or_max[k]
 
         _min = True
 
@@ -5442,12 +5931,18 @@ def set_objectives() -> dict:
             _min = False
 
         objectives[key] = ObjectiveProperties(minimize=_min)
+        k = k + 1
 
     return objectives
 
-def set_experiment_constraints(experiment_constraints: Optional[list], experiment_args: dict, _experiment_parameters: Union[dict, list]) -> dict:
-    if experiment_constraints and len(experiment_constraints):
+def set_experiment_constraints(experiment_constraints: Optional[list], experiment_args: dict, _experiment_parameters: Optional[Union[dict, list]]) -> dict:
+    if _experiment_parameters is None:
+        print_red("set_experiment_constraints: _experiment_parameters was None")
+        my_exit(95)
+
+        return {}
 
+    if experiment_constraints and len(experiment_constraints):
         experiment_args["parameter_constraints"] = []
 
         if experiment_constraints:
@@ -5476,11 +5971,15 @@ def set_experiment_constraints(experiment_constraints: Optional[list], experimen
 
     return experiment_args
 
-def replace_parameters_for_continued_jobs(parameter: Optional[list], cli_params_experiment_parameters: Optional[list]) -> None:
+def replace_parameters_for_continued_jobs(parameter: Optional[list], cli_params_experiment_parameters: Optional[dict | list]) -> None:
+    if not experiment_parameters:
+        print_red("replace_parameters_for_continued_jobs: experiment_parameters was False")
+        return None
+
     if args.worker_generator_path:
         return None
 
-    def get_name(obj) -> Optional[str]:
+    def get_name(obj: Any) -> Optional[str]:
         """Extract a parameter name from dict, list, or tuple safely."""
         if isinstance(obj, dict):
             return obj.get("name")
@@ -5562,13 +6061,13 @@ def copy_continue_uuid() -> None:
         print_debug(f"copy_continue_uuid: Source file does not exist: {source_file}")
 
 def load_ax_client_from_experiment_parameters() -> None:
-    #pprint(experiment_parameters)
-    global ax_client
+    if experiment_parameters:
+        global ax_client
 
-    tmp_file_path = get_tmp_file_from_json(experiment_parameters)
-    ax_client = AxClient.load_from_json_file(tmp_file_path)
-    ax_client = cast(AxClient, ax_client)
-    os.unlink(tmp_file_path)
+        tmp_file_path = get_tmp_file_from_json(experiment_parameters)
+        ax_client = AxClient.load_from_json_file(tmp_file_path)
+        ax_client = cast(AxClient, ax_client)
+        os.unlink(tmp_file_path)
 
 def save_checkpoint_for_continued() -> None:
     checkpoint_filepath = get_state_file_name('checkpoint.json')
@@ -5580,12 +6079,15 @@ def save_checkpoint_for_continued() -> None:
         _fatal_error(f"{checkpoint_filepath} not found. Cannot continue_previous_job without.", 47)
 
 def load_original_generation_strategy(original_ax_client_file: str) -> None:
-    with open(original_ax_client_file, encoding="utf-8") as f:
-        loaded_original_ax_client_json = json.load(f)
-        original_generation_strategy = loaded_original_ax_client_json["generation_strategy"]
+    if experiment_parameters:
+        with open(original_ax_client_file, encoding="utf-8") as f:
+            loaded_original_ax_client_json = json.load(f)
+            original_generation_strategy = loaded_original_ax_client_json["generation_strategy"]
 
-        if original_generation_strategy:
-            experiment_parameters["generation_strategy"] = original_generation_strategy
+            if original_generation_strategy:
+                experiment_parameters["generation_strategy"] = original_generation_strategy
+    else:
+        print_red("load_original_generation_strategy: experiment_parameters was empty!")
 
 def wait_for_checkpoint_file(checkpoint_file: str) -> None:
     start_time = time.time()
@@ -5598,10 +6100,6 @@ def wait_for_checkpoint_file(checkpoint_file: str) -> None:
     elapsed = int(time.time() - start_time)
     console.print(f"[green]Checkpoint file found after {elapsed} seconds[/green]   ")
 
-def __get_experiment_parameters__check_ax_client() -> None:
-    if not ax_client:
-        _fatal_error("Something went wrong with the ax_client", 9)
-
 def validate_experiment_parameters() -> None:
     if experiment_parameters is None:
         print_red("Error: experiment_parameters is None.")
@@ -5611,6 +6109,8 @@ def validate_experiment_parameters() -> None:
         print_red(f"Error: experiment_parameters is not a dict: {type(experiment_parameters).__name__}")
         my_exit(95)
 
+        sys.exit(95)
+
     path_checks = [
         ("experiment", experiment_parameters),
         ("search_space", experiment_parameters.get("experiment")),
@@ -5622,7 +6122,12 @@ def validate_experiment_parameters() -> None:
             print_red(f"Error: Missing key '{key}' at level: {current_level}")
             my_exit(95)
 
-def __get_experiment_parameters__load_from_checkpoint(continue_previous_job: str, cli_params_experiment_parameters: Optional[list]) -> Tuple[Any, str, str]:
+def load_from_checkpoint(continue_previous_job: str, cli_params_experiment_parameters: Optional[dict | list]) -> Tuple[Any, str, str]:
+    if not ax_client:
+        print_red("load_from_checkpoint: ax_client was None")
+        my_exit(101)
+        return {}, "", ""
+
     print_debug(f"Load from checkpoint: {continue_previous_job}")
 
     checkpoint_file = f"{continue_previous_job}/state_files/checkpoint.json"
@@ -5648,7 +6153,7 @@ def __get_experiment_parameters__load_from_checkpoint(continue_previous_job: str
 
     replace_parameters_for_continued_jobs(args.parameter, cli_params_experiment_parameters)
 
-    ax_client.save_to_json_file(filepath=original_ax_client_file)
+    save_ax_client_to_json_file(original_ax_client_file)
 
     load_original_generation_strategy(original_ax_client_file)
     load_ax_client_from_experiment_parameters()
@@ -5664,6 +6169,12 @@ def __get_experiment_parameters__load_from_checkpoint(continue_previous_job: str
 
     experiment_constraints = get_constraints()
     if experiment_constraints:
+
+        if not experiment_parameters:
+            print_red("load_from_checkpoint: experiment_parameters was None")
+
+            return {}, "", ""
+
         experiment_args = set_experiment_constraints(
             experiment_constraints,
             experiment_args,
@@ -5672,7 +6183,116 @@ def __get_experiment_parameters__load_from_checkpoint(continue_previous_job: str
 
     return experiment_args, gpu_string, gpu_color
 
-def __get_experiment_parameters__create_new_experiment() -> Tuple[dict, str, str]:
+def get_experiment_args_import_python_script() -> str:
+
+    return """from ax.service.ax_client import AxClient, ObjectiveProperties
+from ax.adapter.registry import Generators
+import random
+
+"""
+
+def get_generate_and_test_random_function_str() -> str:
+    raw_data_entries = ",\n                ".join(
+        f'"{name}": random.uniform(0, 1)' for name in arg_result_names
+    )
+
+    return f"""
+def generate_and_test_random_parameters(n: int) -> None:
+    for _ in range(n):
+        print("======================================")
+        parameters, trial_index = ax_client.get_next_trial()
+        print("Trial Index:", trial_index)
+        print("Suggested parameters:", parameters)
+
+        ax_client.complete_trial(
+            trial_index=trial_index,
+            raw_data={{
+                {raw_data_entries}
+            }}
+        )
+
+generate_and_test_random_parameters({args.num_random_steps + 1})
+"""
+
+def get_global_gs_string() -> str:
+    seed_str = ""
+    if args.seed is not None:
+        seed_str = f"model_kwargs={{'seed': {args.seed}}},"
+
+    return f"""from ax.generation_strategy.generation_strategy import GenerationStep, GenerationStrategy
+
+global_gs = GenerationStrategy(
+    steps=[
+        GenerationStep(
+            generator=Generators.SOBOL,
+            num_trials={args.num_random_steps},
+            max_parallelism=5,
+            {seed_str}
+        ),
+        GenerationStep(
+            generator=Generators.{args.model},
+            num_trials=-1,
+            max_parallelism=5,
+        ),
+    ]
+)
+"""
+
+def get_debug_ax_client_str() -> str:
+    return """
+ax_client = AxClient(
+    verbose_logging=True,
+    enforce_sequential_optimization=False,
+    generation_strategy=global_gs
+)
+"""
+
+def write_ax_debug_python_code(experiment_args: dict) -> None:
+    if args.generation_strategy:
+        print_debug("Cannot write debug code for custom generation_strategy")
+        return None
+
+    if args.model in uncontinuable_models:
+        print_debug(f"Cannot write debug code for uncontinuable mode {args.model}")
+        return None
+
+    python_code = python_code = get_experiment_args_import_python_script() + \
+        get_global_gs_string() + \
+        get_debug_ax_client_str() + \
+        "experiment_args = " + pformat(experiment_args, width=120, compact=False) + \
+        "\nax_client.create_experiment(**experiment_args)\n" + \
+        get_generate_and_test_random_function_str()
+
+    file_path = f"{get_current_run_folder()}/debug.py"
+
+    try:
+        print_debug(python_code)
+        with open(file_path, "w", encoding="utf-8") as f:
+            f.write(python_code)
+    except Exception as e:
+        print_red(f"Error while writing {file_path}: {e}")
+
+    return None
+
+def create_ax_client_experiment(experiment_args: dict) -> None:
+    if not ax_client:
+        my_exit(101)
+
+        return None
+
+    write_ax_debug_python_code(experiment_args)
+
+    ax_client.create_experiment(**experiment_args)
+
+    return None
+
+def create_new_experiment() -> Tuple[dict, str, str]:
+    if ax_client is None:
+        print_red("create_new_experiment: ax_client is None")
+        my_exit(101)
+
+        return {}, "", ""
+
     objectives = set_objectives()
 
     experiment_args = {
@@ -5695,7 +6315,7 @@ def __get_experiment_parameters__create_new_experiment() -> Tuple[dict, str, str
     experiment_args = set_experiment_constraints(get_constraints(), experiment_args, experiment_parameters)
 
     try:
-        ax_client.create_experiment(**experiment_args)
+        create_ax_client_experiment(experiment_args)
         new_metrics = [Metric(k) for k in arg_result_names if k not in ax_client.metric_names]
         ax_client.experiment.add_tracking_metrics(new_metrics)
     except AssertionError as error:
@@ -5709,17 +6329,17 @@ def __get_experiment_parameters__create_new_experiment() -> Tuple[dict, str, str
 
     return experiment_args, gpu_string, gpu_color
 
-def get_experiment_parameters(cli_params_experiment_parameters: Optional[list]) -> Optional[Tuple[AxClient, dict, str, str]]:
+def get_experiment_parameters(cli_params_experiment_parameters: Optional[dict | list]) -> Optional[Tuple[dict, str, str]]:
     continue_previous_job = args.worker_generator_path or args.continue_previous_job
 
-    __get_experiment_parameters__check_ax_client()
+    check_ax_client()
 
     if continue_previous_job:
-        experiment_args, gpu_string, gpu_color = __get_experiment_parameters__load_from_checkpoint(continue_previous_job, cli_params_experiment_parameters)
+        experiment_args, gpu_string, gpu_color = load_from_checkpoint(continue_previous_job, cli_params_experiment_parameters)
     else:
-        experiment_args, gpu_string, gpu_color = __get_experiment_parameters__create_new_experiment()
+        experiment_args, gpu_string, gpu_color = create_new_experiment()
 
-    return ax_client, experiment_args, gpu_string, gpu_color
+    return experiment_args, gpu_string, gpu_color
 
 def get_type_short(typename: str) -> str:
     if typename == "RangeParameter":
@@ -5778,7 +6398,6 @@ def parse_single_experiment_parameter_table(classic_params: Optional[Union[list,
                 _upper = param["bounds"][1]
 
             _possible_int_lower = str(helpers.to_int_when_possible(_lower))
-            #print(f"name: {_name}, _possible_int_lower: {_possible_int_lower}, lower: {_lower}")
             _possible_int_upper = str(helpers.to_int_when_possible(_upper))
 
             rows.append([_name, _short_type, _possible_int_lower, _possible_int_upper, "", value_type, log_scale])
@@ -5855,19 +6474,62 @@ def print_ax_parameter_constraints_table(experiment_args: dict) -> None:
 
     return None
 
+def check_base_for_print_overview() -> Optional[bool]:
+    if args.continue_previous_job is not None and arg_result_names is not None and len(arg_result_names) != 0 and original_result_names is not None and len(original_result_names) != 0:
+        print_yellow("--result_names will be ignored in continued jobs. The result names from the previous job will be used.")
+
+    if ax_client is None:
+        print_red("ax_client was None")
+        return None
+
+    if ax_client.experiment is None:
+        print_red("ax_client.experiment was None")
+        return None
+
+    if ax_client.experiment.optimization_config is None:
+        print_red("ax_client.experiment.optimization_config was None")
+        return None
+
+    return True
+
+def get_config_objectives() -> Any:
+    if not ax_client:
+        print_red("create_new_experiment: ax_client is None")
+        my_exit(101)
+
+        return None
+
+    config_objectives = None
+
+    if ax_client.experiment and ax_client.experiment.optimization_config:
+        opt_config = ax_client.experiment.optimization_config
+        if opt_config.is_moo_problem:
+            objective = getattr(opt_config, "objective", None)
+            if objective and getattr(objective, "objectives", None) is not None:
+                config_objectives = objective.objectives
+            else:
+                print_debug("ax_client.experiment.optimization_config.objective was None")
+        else:
+            config_objectives = [opt_config.objective]
+    else:
+        print_debug("ax_client.experiment or optimization_config was None")
+
+    return config_objectives
+
 def print_result_names_overview_table() -> None:
     if not ax_client:
         _fatal_error("Tried to access ax_client in print_result_names_overview_table, but it failed, because the ax_client was not defined.", 101)
 
         return None
 
-    if args.continue_previous_job is not None and args.result_names is not None and len(args.result_names) != 0 and original_result_names is not None and len(original_result_names) != 0:
-        print_yellow("--result_names will be ignored in continued jobs. The result names from the previous job will be used.")
+    if check_base_for_print_overview() is None:
+        return None
 
-    if ax_client.experiment.optimization_config.is_moo_problem:
-        config_objectives = ax_client.experiment.optimization_config.objective.objectives
-    else:
-        config_objectives = [ax_client.experiment.optimization_config.objective]
+    config_objectives = get_config_objectives()
+
+    if config_objectives is None:
+        print_red("config_objectives not found")
+        return None
 
     res_names = []
     res_min_max = []
@@ -5962,28 +6624,31 @@ def print_overview_tables(classic_params: Optional[Union[list, dict]], experimen
     print_result_names_overview_table()
 
 def update_progress_bar(nr: int) -> None:
-    try:
-        progress_bar.update(nr)
-    except Exception as e:
-        print(f"Error updating progress bar: {e}")
+    log_data()
+
+    if progress_bar is not None:
+        try:
+            progress_bar.update(nr)
+        except Exception as e:
+            print_red(f"Error updating progress bar: {e}")
+    else:
+        print_red("update_progress_bar: progress_bar was None")
 
 def get_current_model_name() -> str:
     if overwritten_to_random:
         return "Random*"
 
+    gs_model = "unknown model"
+
     if ax_client:
         try:
             if args.generation_strategy:
-                idx = getattr(ax_client.generation_strategy, "current_step_index", None)
+                idx = getattr(global_gs, "current_step_index", None)
                 if isinstance(idx, int):
                     if 0 <= idx < len(generation_strategy_names):
                         gs_model = generation_strategy_names[int(idx)]
-                    else:
-                        gs_model = "unknown model"
-                else:
-                    gs_model = "unknown model"
             else:
-                gs_model = getattr(ax_client.generation_strategy, "current_node_name", "unknown model")
+                gs_model = getattr(global_gs, "current_node_name", "unknown model")
 
             if gs_model:
                 return str(gs_model)
@@ -6040,7 +6705,7 @@ def get_workers_string() -> str:
     )
 
 def _get_workers_string_collect_stats() -> dict:
-    stats = {}
+    stats: dict = {}
     for job, _ in global_vars["jobs"][:]:
         state = state_from_job(job)
         stats[state] = stats.get(state, 0) + 1
@@ -6089,8 +6754,8 @@ def submitted_jobs(nr: int = 0) -> int:
 def count_jobs_in_squeue() -> tuple[int, str]:
     global _last_count_time, _last_count_result
 
-    now = time.time()
-    if _last_count_result != (0, "") and now - _last_count_time < 15:
+    now = int(time.time())
+    if _last_count_result != (0, "") and now - _last_count_time < 5:
         return _last_count_result
 
     _len = len(global_vars["jobs"])
@@ -6112,6 +6777,7 @@ def count_jobs_in_squeue() -> tuple[int, str]:
             check=True,
             text=True
         )
+
         if "slurm_load_jobs error" in result.stderr:
             _last_count_result = (_len, "Detected slurm_load_jobs error in stderr.")
             _last_count_time = now
@@ -6152,6 +6818,8 @@ def log_worker_numbers() -> None:
         if len(WORKER_PERCENTAGE_USAGE) == 0 or WORKER_PERCENTAGE_USAGE[len(WORKER_PERCENTAGE_USAGE) - 1] != this_values:
             WORKER_PERCENTAGE_USAGE.append(this_values)
 
+        write_worker_usage()
+
 def get_slurm_in_brackets(in_brackets: list) -> list:
     if is_slurm_job():
         workers_strings = get_workers_string()
@@ -6236,6 +6904,8 @@ def progressbar_description(new_msgs: Union[str, List[str]] = []) -> None:
     global last_progress_bar_desc
     global last_progress_bar_refresh_time
 
+    log_data()
+
     if isinstance(new_msgs, str):
         new_msgs = [new_msgs]
 
@@ -6253,7 +6923,7 @@ def progressbar_description(new_msgs: Union[str, List[str]] = []) -> None:
         print_red("Cannot update progress bar! It is None.")
 
 def clean_completed_jobs() -> None:
-    job_states_to_be_removed = ["early_stopped", "abandoned", "cancelled", "timeout", "interrupted", "failed", "preempted", "node_fail", "boot_fail"]
+    job_states_to_be_removed = ["early_stopped", "abandoned", "cancelled", "timeout", "interrupted", "failed", "preempted", "node_fail", "boot_fail", "finished"]
     job_states_to_be_ignored = ["ready", "completed", "unknown", "pending", "running", "completing", "out_of_memory", "requeued", "resv_del_hold"]
 
     for job, trial_index in global_vars["jobs"][:]:
@@ -6311,7 +6981,7 @@ def load_existing_job_data_into_ax_client() -> None:
     nr_of_imported_jobs = get_nr_of_imported_jobs()
     set_nr_inserted_jobs(NR_INSERTED_JOBS + nr_of_imported_jobs)
 
-def parse_parameter_type_error(_error_message: Union[str, None]) -> Optional[dict]:
+def parse_parameter_type_error(_error_message: Union[Exception, str, None]) -> Optional[dict]:
     if not _error_message:
         return None
 
@@ -6378,7 +7048,7 @@ def get_generation_node_for_index(
     results_list: List[Dict[str, Any]],
     index: int,
     __status: Any,
-    base_str: str
+    base_str: Optional[str]
 ) -> str:
     __status.update(f"{base_str}: Getting generation node")
     try:
@@ -6398,7 +7068,7 @@ def get_generation_node_for_index(
 
         return generation_node
     except Exception as e:
-        print(f"Error while get_generation_node_for_index: {e}")
+        print_red(f"Error while get_generation_node_for_index: {e}")
         return "MANUAL"
 
 def _get_generation_node_for_index_index_valid(
@@ -6461,111 +7131,154 @@ def _get_generation_node_for_index_floats_match(
         return False
     return abs(row_val_num - val) <= tolerance
 
+def validate_and_convert_params_for_jobs_from_csv(arm_params: Dict) -> Dict:
+    corrected_params: Dict[Any, Any] = {}
+
+    if experiment_parameters is not None:
+        for param in experiment_parameters:
+            name = param["name"]
+            expected_type = param.get("value_type", "str")
+
+            if name not in arm_params:
+                continue
+
+            value = arm_params[name]
+
+            try:
+                if param["type"] == "range":
+                    if expected_type == "int":
+                        corrected_params[name] = int(value)
+                    elif expected_type == "float":
+                        corrected_params[name] = float(value)
+                elif param["type"] == "choice":
+                    corrected_params[name] = str(value)
+            except (ValueError, TypeError):
+                corrected_params[name] = None
+
+    return corrected_params
+
 def insert_jobs_from_csv(this_csv_file_path: str) -> None:
     with spinner(f"Inserting job into CSV from {this_csv_file_path}") as __status:
-        this_csv_file_path = this_csv_file_path.replace("//", "/")
+        this_csv_file_path = normalize_path(this_csv_file_path)
 
-        if not os.path.exists(this_csv_file_path):
+        if not helpers.file_exists(this_csv_file_path):
             print_red(f"--load_data_from_existing_jobs: Cannot find {this_csv_file_path}")
-
             return
 
-        def validate_and_convert_params(arm_params: Dict) -> Dict:
-            corrected_params: Dict[Any, Any] = {}
+        arm_params_list, results_list = parse_csv(this_csv_file_path)
+        insert_jobs_from_lists(this_csv_file_path, arm_params_list, results_list, __status)
 
-            if experiment_parameters is not None:
-                for param in experiment_parameters:
-                    name = param["name"]
-                    expected_type = param.get("value_type", "str")
+def normalize_path(file_path: str) -> str:
+    return file_path.replace("//", "/")
 
-                    if name not in arm_params:
-                        continue
+def insert_jobs_from_lists(csv_path: str, arm_params_list: Any, results_list: Any, __status: Any) -> None:
+    cnt = 0
+    err_msgs: list = []
 
-                    value = arm_params[name]
+    for i, (arm_params, result) in enumerate(zip(arm_params_list, results_list)):
+        base_str = f"[bold green]Loading job {i}/{len(results_list)} from {csv_path} into ax_client, result: {result}"
+        __status.update(base_str)
 
-                    try:
-                        if param["type"] == "range":
-                            if expected_type == "int":
-                                corrected_params[name] = int(value)
-                            elif expected_type == "float":
-                                corrected_params[name] = float(value)
-                        elif param["type"] == "choice":
-                            corrected_params[name] = str(value)
-                    except (ValueError, TypeError):
-                        corrected_params[name] = None
-
-            return corrected_params
+        if not args.worker_generator_path:
+            arm_params = validate_and_convert_params_for_jobs_from_csv(arm_params)
 
-        arm_params_list, results_list = parse_csv(this_csv_file_path)
+        cnt = try_insert_job(csv_path, arm_params, result, i, arm_params_list, results_list, __status, base_str, cnt, err_msgs)
 
-        cnt = 0
+    summarize_insertions(csv_path, cnt)
+    update_global_job_counters(cnt)
 
-        err_msgs = []
+def try_insert_job(csv_path: str, arm_params: Dict, result: Any, i: int, arm_params_list: Any, results_list: Any, __status: Any, base_str: Optional[str], cnt: int, err_msgs: Optional[Union[str, list[str]]]) -> int:
+    try:
+        gen_node_name = get_generation_node_for_index(csv_path, arm_params_list, results_list, i, __status, base_str)
 
-        i = 0
-        for arm_params, result in zip(arm_params_list, results_list):
-            base_str = f"[bold green]Loading job {i}/{len(results_list)} from {this_csv_file_path} into ax_client, result: {result}"
-            __status.update(base_str)
-            if not args.worker_generator_path:
-                arm_params = validate_and_convert_params(arm_params)
+        if not result:
+            print_yellow("Encountered job without a result")
+            return cnt
 
-            try:
-                gen_node_name = get_generation_node_for_index(this_csv_file_path, arm_params_list, results_list, i, __status, base_str)
+        if insert_job_into_ax_client(arm_params, result, gen_node_name, __status, base_str):
+            cnt += 1
+            print_debug(f"Inserted one job from {csv_path}, arm_params: {arm_params}, results: {result}")
+        else:
+            print_red(f"Failed to insert one job from {csv_path}, arm_params: {arm_params}, results: {result}")
 
-                if len(result):
-                    if insert_job_into_ax_client(arm_params, result, gen_node_name, __status, base_str):
-                        cnt += 1
+    except ValueError as e:
+        err_msg = (
+            f"Failed to insert job(s) from {csv_path} into ax_client. "
+            f"This can happen when the csv file has different parameters or results as the main job one's "
+            f"or other imported jobs. Error: {e}"
+        )
 
-                        print_debug(f"Inserted one job from {this_csv_file_path}, arm_params: {arm_params}, results: {result}")
-                    else:
-                        print_red(f"Failed to insert one job from {this_csv_file_path}, arm_params: {arm_params}, results: {result}")
-                else:
-                    print_yellow("Encountered job without a result")
-            except ValueError as e:
-                err_msg = f"Failed to insert job(s) from {this_csv_file_path} into ax_client. This can happen when the csv file has different parameters or results as the main job one's or other imported jobs. Error: {e}"
+        if err_msgs is None:
+            print_red("try_insert_job: err_msgs was None")
+        else:
+            if isinstance(err_msgs, list):
                 if err_msg not in err_msgs:
                     print_red(err_msg)
                     err_msgs.append(err_msg)
+            elif isinstance(err_msgs, str):
+                err_msgs += f"\n{err_msg}"
 
-            i = i + 1
+    return cnt
 
-        if cnt:
-            if cnt == 1:
-                print_yellow(f"Inserted one job from {this_csv_file_path}")
-            else:
-                print_yellow(f"Inserted {cnt} jobs from {this_csv_file_path}")
+def summarize_insertions(csv_path: str, cnt: int) -> None:
+    if cnt == 0:
+        return
+    if cnt == 1:
+        print_yellow(f"Inserted one job from {csv_path}")
+    else:
+        print_yellow(f"Inserted {cnt} jobs from {csv_path}")
 
-        if not args.worker_generator_path:
-            set_max_eval(max_eval + cnt)
-            set_nr_inserted_jobs(NR_INSERTED_JOBS + cnt)
+def update_global_job_counters(cnt: int) -> None:
+    if not args.worker_generator_path:
+        set_max_eval(max_eval + cnt)
+        set_nr_inserted_jobs(NR_INSERTED_JOBS + cnt)
 
-def __insert_job_into_ax_client__update_status(__status: Optional[Any], base_str: Optional[str], new_text: str) -> None:
+def update_status(__status: Optional[Any], base_str: Optional[str], new_text: str) -> None:
     if __status and base_str:
         __status.update(f"{base_str}: {new_text}")
 
-def __insert_job_into_ax_client__check_ax_client() -> None:
+def check_ax_client() -> None:
     if ax_client is None or not ax_client:
         _fatal_error("insert_job_into_ax_client: ax_client was not defined where it should have been", 101)
 
-def __insert_job_into_ax_client__attach_trial(arm_params: dict) -> Tuple[Any, int]:
+def attach_ax_client_data(arm_params: dict) -> Optional[Tuple[Any, int]]:
+    if not ax_client:
+        my_exit(101)
+
+        return None
+
     new_trial = ax_client.attach_trial(arm_params)
+
+    return new_trial
+
+def attach_trial(arm_params: dict) -> Tuple[Any, int]:
+    if ax_client is None:
+        raise RuntimeError("attach_trial: ax_client was empty")
+
+    new_trial = attach_ax_client_data(arm_params)
     if not isinstance(new_trial, tuple) or len(new_trial) < 2:
         raise RuntimeError("attach_trial didn't return the expected tuple")
     return new_trial
 
-def __insert_job_into_ax_client__get_trial(trial_idx: int) -> Any:
+def get_trial_by_index(trial_idx: int) -> Any:
+    if ax_client is None:
+        raise RuntimeError("get_trial_by_index: ax_client was empty")
+
     trial = ax_client.experiment.trials.get(trial_idx)
     if trial is None:
         raise RuntimeError(f"Trial with index {trial_idx} not found")
     return trial
 
-def __insert_job_into_ax_client__create_generator_run(arm_params: dict, trial_idx: int, new_job_type: str) -> GeneratorRun:
+def create_generator_run(arm_params: dict, trial_idx: int, new_job_type: str) -> GeneratorRun:
     arm = Arm(parameters=arm_params, name=f'{trial_idx}_0')
     return GeneratorRun(arms=[arm], generation_node_name=new_job_type)
 
-def __insert_job_into_ax_client__complete_trial_if_result(trial_idx: int, result: dict, __status: Optional[Any], base_str: Optional[str]) -> None:
+def complete_trial_if_result(trial_idx: int, result: dict, __status: Optional[Any], base_str: Optional[str]) -> None:
+    if ax_client is None:
+        raise RuntimeError("complete_trial_if_result: ax_client was empty")
+
     if f"{result}" != "":
-        __insert_job_into_ax_client__update_status(__status, base_str, "Completing trial")
+        update_status(__status, base_str, "Completing trial")
         is_ok = True
 
         for keyname in result.keys():
@@ -6573,20 +7286,20 @@ def __insert_job_into_ax_client__complete_trial_if_result(trial_idx: int, result
                 is_ok = False
 
         if is_ok:
-            ax_client.complete_trial(trial_index=trial_idx, raw_data=result)
-            __insert_job_into_ax_client__update_status(__status, base_str, "Completed trial")
+            complete_ax_client_trial(trial_idx, result)
+            update_status(__status, base_str, "Completed trial")
         else:
             print_debug("Empty job encountered")
     else:
-        __insert_job_into_ax_client__update_status(__status, base_str, "Found trial without result. Not adding it.")
+        update_status(__status, base_str, "Found trial without result. Not adding it.")
 
-def __insert_job_into_ax_client__save_results_if_needed(__status: Optional[Any], base_str: Optional[str]) -> None:
+def save_results_if_needed(__status: Optional[Any], base_str: Optional[str]) -> None:
     if not args.worker_generator_path:
-        __insert_job_into_ax_client__update_status(__status, base_str, f"Saving {RESULTS_CSV_FILENAME}")
+        update_status(__status, base_str, f"Saving {RESULTS_CSV_FILENAME}")
         save_results_csv()
-        __insert_job_into_ax_client__update_status(__status, base_str, f"Saved {RESULTS_CSV_FILENAME}")
+        update_status(__status, base_str, f"Saved {RESULTS_CSV_FILENAME}")
 
-def __insert_job_into_ax_client__handle_type_error(e: Exception, arm_params: dict) -> bool:
+def handle_insert_job_error(e: Exception, arm_params: dict) -> bool:
     parsed_error = parse_parameter_type_error(e)
     if parsed_error is not None:
         param = parsed_error["parameter_name"]
@@ -6609,37 +7322,37 @@ def insert_job_into_ax_client(
     result: dict,
     new_job_type: str = "MANUAL",
     __status: Optional[Any] = None,
-    base_str: str = None
+    base_str: Optional[str] = None
 ) -> bool:
-    __insert_job_into_ax_client__check_ax_client()
+    check_ax_client()
 
     done_converting = False
     while not done_converting:
         try:
-            __insert_job_into_ax_client__update_status(__status, base_str, "Checking ax client")
+            update_status(__status, base_str, "Checking ax client")
             if ax_client is None:
                 return False
 
-            __insert_job_into_ax_client__update_status(__status, base_str, "Attaching new trial")
-            _, new_trial_idx = __insert_job_into_ax_client__attach_trial(arm_params)
+            update_status(__status, base_str, "Attaching new trial")
+            _, new_trial_idx = attach_trial(arm_params)
 
-            __insert_job_into_ax_client__update_status(__status, base_str, "Getting new trial")
-            trial = __insert_job_into_ax_client__get_trial(new_trial_idx)
-            __insert_job_into_ax_client__update_status(__status, base_str, "Got new trial")
+            update_status(__status, base_str, "Getting new trial")
+            trial = get_trial_by_index(new_trial_idx)
+            update_status(__status, base_str, "Got new trial")
 
-            __insert_job_into_ax_client__update_status(__status, base_str, "Creating new arm")
-            manual_generator_run = __insert_job_into_ax_client__create_generator_run(arm_params, new_trial_idx, new_job_type)
+            update_status(__status, base_str, "Creating new arm")
+            manual_generator_run = create_generator_run(arm_params, new_trial_idx, new_job_type)
             trial._generator_run = manual_generator_run
             fool_linter(trial._generator_run)
 
-            __insert_job_into_ax_client__complete_trial_if_result(new_trial_idx, result, __status, base_str)
+            complete_trial_if_result(new_trial_idx, result, __status, base_str)
             done_converting = True
 
-            __insert_job_into_ax_client__save_results_if_needed(__status, base_str)
+            save_results_if_needed(__status, base_str)
             return True
 
         except ax.exceptions.core.UnsupportedError as e:
-            if not __insert_job_into_ax_client__handle_type_error(e, arm_params):
+            if not handle_insert_job_error(e, arm_params):
                 break
 
     return False
@@ -6970,7 +7683,7 @@ def get_parameters_from_outfile(stdout_path: str) -> Union[None, dict, str]:
         if not args.tests:
             original_print(f"get_parameters_from_outfile: The file '{stdout_path}' was not found.")
     except Exception as e:
-        print(f"get_parameters_from_outfile: There was an error: {e}")
+        print_red(f"get_parameters_from_outfile: There was an error: {e}")
 
     return None
 
@@ -6988,7 +7701,7 @@ def get_hostname_from_outfile(stdout_path: Optional[str]) -> Optional[str]:
         original_print(f"The file '{stdout_path}' was not found.")
         return None
     except Exception as e:
-        print(f"There was an error: {e}")
+        print_red(f"There was an error: {e}")
         return None
 
 def add_to_global_error_list(msg: str) -> None:
@@ -7024,22 +7737,70 @@ def mark_trial_as_failed(trial_index: int, _trial: Any) -> None:
 
             return None
 
-        ax_client.log_trial_failure(trial_index=trial_index)
+        log_ax_client_trial_failure(trial_index)
         _trial.mark_failed(unsafe=True)
     except ValueError as e:
         print_debug(f"mark_trial_as_failed error: {e}")
 
     return None
 
-def _finish_job_core_helper_check_valid_result(result: Union[None, list, int, float, tuple]) -> bool:
+def check_valid_result(result: Union[None, dict]) -> bool:
     possible_val_not_found_values = [
         VAL_IF_NOTHING_FOUND,
         -VAL_IF_NOTHING_FOUND,
         -99999999999999997168788049560464200849936328366177157906432,
         99999999999999997168788049560464200849936328366177157906432
     ]
-    values_to_check = result if isinstance(result, list) else [result]
-    return result is not None and all(r not in possible_val_not_found_values for r in values_to_check)
+
+    def flatten_values(obj: Any) -> Any:
+        values = []
+        try:
+            if isinstance(obj, dict):
+                for v in obj.values():
+                    values.extend(flatten_values(v))
+            elif isinstance(obj, (list, tuple, set)):
+                for v in obj:
+                    values.extend(flatten_values(v))
+            else:
+                values.append(obj)
+        except Exception as e:
+            print_red(f"Error while flattening values: {e}")
+        return values
+
+    if result is None:
+        return False
+
+    try:
+        all_values = flatten_values(result)
+        for val in all_values:
+            if val in possible_val_not_found_values:
+                return False
+        return True
+    except Exception as e:
+        print_red(f"Error while checking result validity: {e}")
+        return False
+
+def update_ax_client_trial(trial_idx: int, result: Union[list, dict]) -> None:
+    if not ax_client:
+        my_exit(101)
+
+        return None
+
+    trial = get_trial_by_index(trial_idx)
+
+    trial.update_trial_data(raw_data=result)
+
+    return None
+
+def complete_ax_client_trial(trial_idx: int, result: Union[list, dict]) -> None:
+    if not ax_client:
+        my_exit(101)
+
+        return None
+
+    ax_client.complete_trial(trial_index=trial_idx, raw_data=result)
+
+    return None
 
 def _finish_job_core_helper_complete_trial(trial_index: int, raw_result: dict) -> None:
     if ax_client is None:
@@ -7048,25 +7809,64 @@ def _finish_job_core_helper_complete_trial(trial_index: int, raw_result: dict) -
 
     try:
         print_debug(f"Completing trial: {trial_index} with result: {raw_result}...")
-        ax_client.complete_trial(trial_index=trial_index, raw_data=raw_result)
+        complete_ax_client_trial(trial_index, raw_result)
         print_debug(f"Completing trial: {trial_index} with result: {raw_result}... Done!")
     except ax.exceptions.core.UnsupportedError as e:
         if f"{e}":
             print_debug(f"Completing trial: {trial_index} with result: {raw_result} after failure. Trying to update trial...")
-            ax_client.update_trial_data(trial_index=trial_index, raw_data=raw_result)
+            update_ax_client_trial(trial_index, raw_result)
             print_debug(f"Completing trial: {trial_index} with result: {raw_result} after failure... Done!")
         else:
             _fatal_error(f"Error completing trial: {e}", 234)
 
     return None
 
-def _finish_job_core_helper_mark_success(_trial: ax.core.trial.Trial, result: Union[float, int, tuple]) -> None:
+def format_result_for_display(result: dict) -> str:
+    def safe_float(v: Any) -> str:
+        try:
+            if v is None:
+                return "None"
+            if isinstance(v, (int, float)):
+                if math.isnan(v):
+                    return "NaN"
+                if math.isinf(v):
+                    return "∞" if v > 0 else "-∞"
+                return f"{v:.6f}"
+            return str(v)
+        except Exception as e:
+            return f"<error: {e}>"
+
+    try:
+        if not isinstance(result, dict):
+            return safe_float(result)
+
+        parts = []
+        for key, val in result.items():
+            try:
+                if isinstance(val, (list, tuple)) and len(val) == 2:
+                    main, sem = val
+                    main_str = safe_float(main)
+                    if sem is not None:
+                        sem_str = safe_float(sem)
+                        parts.append(f"{key}: {main_str} (SEM: {sem_str})")
+                    else:
+                        parts.append(f"{key}: {main_str}")
+                else:
+                    parts.append(f"{key}: {safe_float(val)}")
+            except Exception as e:
+                parts.append(f"{key}: <error: {e}>")
+
+        return ", ".join(parts)
+    except Exception as e:
+        return f"<error formatting result: {e}>"
+
+def _finish_job_core_helper_mark_success(_trial: ax.core.trial.Trial, result: dict) -> None:
     print_debug(f"Marking trial {_trial} as completed")
     _trial.mark_completed(unsafe=True)
 
     succeeded_jobs(1)
 
-    progressbar_description(f"new result: {result}")
+    progressbar_description(f"new result: {format_result_for_display(result)}")
     update_progress_bar(1)
 
     save_results_csv()
@@ -7080,8 +7880,8 @@ def _finish_job_core_helper_mark_failure(job: Any, trial_index: int, _trial: Any
     if job:
         try:
             progressbar_description("job_failed")
-            ax_client.log_trial_failure(trial_index=trial_index)
-            _trial.mark_failed(unsafe=True)
+            log_ax_client_trial_failure(trial_index)
+            mark_trial_as_failed(trial_index, _trial)
         except Exception as e:
             print_red(f"\nERROR while trying to mark job as failure: {e}")
         job.cancel()
@@ -7098,16 +7898,16 @@ def finish_job_core(job: Any, trial_index: int, this_jobs_finished: int) -> int:
     result = job.result()
     print_debug(f"finish_job_core: trial-index: {trial_index}, job.result(): {result}, state: {state_from_job(job)}")
 
-    raw_result = result
-    result_keys = list(result.keys())
-    result = result[result_keys[0]]
     this_jobs_finished += 1
 
     if ax_client:
-        _trial = ax_client.get_trial(trial_index)
+        _trial = get_ax_client_trial(trial_index)
+
+        if _trial is None:
+            return 0
 
-        if _finish_job_core_helper_check_valid_result(result):
-            _finish_job_core_helper_complete_trial(trial_index, raw_result)
+        if check_valid_result(result):
+            _finish_job_core_helper_complete_trial(trial_index, result)
 
             try:
                 _finish_job_core_helper_mark_success(_trial, result)
@@ -7115,15 +7915,17 @@ def finish_job_core(job: Any, trial_index: int, this_jobs_finished: int) -> int:
                 if len(arg_result_names) > 1 and count_done_jobs() > 1 and not job_calculate_pareto_front(get_current_run_folder(), True):
                     print_red("job_calculate_pareto_front post job failed")
             except Exception as e:
-                print(f"ERROR in line {get_line_info()}: {e}")
+                print_red(f"ERROR in line {get_line_info()}: {e}")
         else:
             _finish_job_core_helper_mark_failure(job, trial_index, _trial)
     else:
-        _fatal_error("ax_client could not be found or used", 9)
+        _fatal_error("ax_client could not be found or used", 101)
 
     print_debug(f"finish_job_core: removing job {job}, trial_index: {trial_index}")
     global_vars["jobs"].remove((job, trial_index))
 
+    log_data()
+
     force_live_share()
 
     return this_jobs_finished
@@ -7136,11 +7938,14 @@ def _finish_previous_jobs_helper_handle_failed_job(job: Any, trial_index: int) -
     if job:
         try:
             progressbar_description("job_failed")
-            _trial = ax_client.get_trial(trial_index)
-            ax_client.log_trial_failure(trial_index=trial_index)
+            _trial = get_ax_client_trial(trial_index)
+            if _trial is None:
+                return None
+
+            log_ax_client_trial_failure(trial_index)
             mark_trial_as_failed(trial_index, _trial)
         except Exception as e:
-            print(f"ERROR in line {get_line_info()}: {e}")
+            print_debug(f"ERROR in line {get_line_info()}: {e}")
         job.cancel()
         orchestrate_job(job, trial_index)
 
@@ -7155,10 +7960,12 @@ def _finish_previous_jobs_helper_handle_failed_job(job: Any, trial_index: int) -
 
 def _finish_previous_jobs_helper_handle_exception(job: Any, trial_index: int, error: Exception) -> int:
     if "None for metric" in str(error):
-        print_red(
-            f"\n⚠ It seems like the program that was about to be run didn't have 'RESULT: <FLOAT>' in it's output string."
-            f"\nError: {error}\nJob-result: {job.result()}"
-        )
+        err_msg = f"\n⚠ It seems like the program that was about to be run didn't have 'RESULT: <FLOAT>' in it's output string.\nError: {error}\nJob-result: {job.result()}"
+
+        if count_done_jobs() == 0:
+            print_red(err_msg)
+        else:
+            print_debug(err_msg)
     else:
         print_red(f"\n⚠ {error}")
 
@@ -7177,7 +7984,10 @@ def _finish_previous_jobs_helper_process_job(job: Any, trial_index: int, this_jo
         this_jobs_finished += _finish_previous_jobs_helper_handle_exception(job, trial_index, error)
     return this_jobs_finished
 
-def _finish_previous_jobs_helper_check_and_process(job: Any, trial_index: int, this_jobs_finished: int) -> int:
+def _finish_previous_jobs_helper_check_and_process(__args: Tuple[Any, int]) -> int:
+    job, trial_index = __args
+
+    this_jobs_finished = 0
     if job is None:
         print_debug(f"finish_previous_jobs: job {job} is None")
         return this_jobs_finished
@@ -7190,10 +8000,6 @@ def _finish_previous_jobs_helper_check_and_process(job: Any, trial_index: int, t
 
     return this_jobs_finished
 
-def _finish_previous_jobs_helper_wrapper(__args: Tuple[Any, int]) -> int:
-    job, trial_index = __args
-    return _finish_previous_jobs_helper_check_and_process(job, trial_index, 0)
-
 def finish_previous_jobs(new_msgs: List[str] = []) -> None:
     global JOBS_FINISHED
 
@@ -7206,13 +8012,10 @@ def finish_previous_jobs(new_msgs: List[str] = []) -> None:
 
     jobs_copy = global_vars["jobs"][:]
 
-    if len(jobs_copy) > 0:
-        print_debug(f"jobs in finish_previous_jobs: {jobs_copy}")
-
-    finishing_jobs_start_time = time.time()
+    #finishing_jobs_start_time = time.time()
 
     with ThreadPoolExecutor() as finish_job_executor:
-        futures = [finish_job_executor.submit(_finish_previous_jobs_helper_wrapper, (job, trial_index)) for job, trial_index in jobs_copy]
+        futures = [finish_job_executor.submit(_finish_previous_jobs_helper_check_and_process, (job, trial_index)) for job, trial_index in jobs_copy]
 
         for future in as_completed(futures):
             try:
@@ -7220,17 +8023,15 @@ def finish_previous_jobs(new_msgs: List[str] = []) -> None:
             except Exception as e:
                 print_red(f"⚠ Exception in parallel job handling: {e}")
 
-    finishing_jobs_end_time = time.time()
-
-    finishing_jobs_runtime = finishing_jobs_end_time - finishing_jobs_start_time
+    #finishing_jobs_end_time = time.time()
 
-    print_debug(f"Finishing jobs took {finishing_jobs_runtime} second(s)")
-
-    save_results_csv()
+    #finishing_jobs_runtime = finishing_jobs_end_time - finishing_jobs_start_time
 
-    save_checkpoint()
+    #print_debug(f"Finishing jobs took {finishing_jobs_runtime} second(s)")
 
     if this_jobs_finished > 0:
+        save_results_csv()
+        save_checkpoint()
         progressbar_description([*new_msgs, f"finished {this_jobs_finished} {'job' if this_jobs_finished == 1 else 'jobs'}"])
 
     JOBS_FINISHED += this_jobs_finished
@@ -7373,11 +8174,15 @@ def is_already_in_defective_nodes(hostname: str) -> bool:
                     return True
     except Exception as e:
         print_red(f"is_already_in_defective_nodes: Error reading the file {file_path}: {e}")
-        return False
 
     return False
 
 def submit_new_job(parameters: Union[dict, str], trial_index: int) -> Any:
+    if submitit_executor is None:
+        print_red("submit_new_job: submitit_executor was None")
+
+        return None
+
     print_debug(f"Submitting new job for trial_index {trial_index}, parameters {parameters}")
 
     start = time.time()
@@ -7388,25 +8193,49 @@ def submit_new_job(parameters: Union[dict, str], trial_index: int) -> Any:
 
     print_debug(f"Done submitting new job, took {elapsed} seconds")
 
+    log_data()
+
     return new_job
 
+def get_ax_client_trial(trial_index: int) -> Optional[ax.core.trial.Trial]:
+    if not ax_client:
+        my_exit(101)
+
+        return None
+
+    try:
+        log_data()
+
+        return ax_client.get_trial(trial_index)
+    except KeyError:
+        error_without_print(f"get_ax_client_trial: trial_index {trial_index} failed")
+        return None
+
 def orchestrator_start_trial(parameters: Union[dict, str], trial_index: int) -> None:
     if submitit_executor and ax_client:
         new_job = submit_new_job(parameters, trial_index)
-        submitted_jobs(1)
+        if new_job:
+            submitted_jobs(1)
 
-        _trial = ax_client.get_trial(trial_index)
+            _trial = get_ax_client_trial(trial_index)
 
-        try:
-            _trial.mark_staged(unsafe=True)
-        except Exception as e:
-            print_debug(f"orchestrator_start_trial: error {e}")
-        _trial.mark_running(unsafe=True, no_runner_required=True)
+            if _trial is not None:
+                try:
+                    _trial.mark_staged(unsafe=True)
+                except Exception as e:
+                    print_debug(f"orchestrator_start_trial: error {e}")
+                _trial.mark_running(unsafe=True, no_runner_required=True)
 
-        print_debug(f"orchestrator_start_trial: appending job {new_job} to global_vars['jobs'], trial_index: {trial_index}")
-        global_vars["jobs"].append((new_job, trial_index))
+                print_debug(f"orchestrator_start_trial: appending job {new_job} to global_vars['jobs'], trial_index: {trial_index}")
+                global_vars["jobs"].append((new_job, trial_index))
+            else:
+                print_red("Trial was none in orchestrator_start_trial")
+        else:
+            print_red("orchestrator_start_trial: Failed to start new job")
+    elif ax_client:
+        _fatal_error("submitit_executor could not be found properly", 9)
     else:
-        _fatal_error("submitit_executor or ax_client could not be found properly", 9)
+        _fatal_error("ax_client could not be found properly", 101)
 
 def handle_exclude_node(stdout_path: str, hostname_from_out_file: Union[None, str]) -> None:
     stdout_path = check_alternate_path(stdout_path)
@@ -7425,7 +8254,7 @@ def handle_restart(stdout_path: str, trial_index: int) -> None:
     if parameters:
         orchestrator_start_trial(parameters, trial_index)
     else:
-        print(f"Could not determine parameters from outfile {stdout_path} for restarting job")
+        print_red(f"Could not determine parameters from outfile {stdout_path} for restarting job")
 
 def check_alternate_path(path: str) -> str:
     if os.path.exists(path):
@@ -7512,7 +8341,7 @@ def execute_evaluation(_params: list) -> Optional[int]:
     print_debug(f"execute_evaluation({_params})")
     trial_index, parameters, trial_counter, phase = _params
     if not ax_client:
-        _fatal_error("Failed to get ax_client", 9)
+        _fatal_error("Failed to get ax_client", 101)
 
         return None
 
@@ -7521,7 +8350,11 @@ def execute_evaluation(_params: list) -> Optional[int]:
 
         return None
 
-    _trial = ax_client.get_trial(trial_index)
+    _trial = get_ax_client_trial(trial_index)
+
+    if _trial is None:
+        error_without_print(f"execute_evaluation: _trial was not in execute_evaluation for params {_params}")
+        return None
 
     def mark_trial_stage(stage: str, error_msg: str) -> None:
         try:
@@ -7536,17 +8369,18 @@ def execute_evaluation(_params: list) -> Optional[int]:
     try:
         initialize_job_environment()
         new_job = submit_new_job(parameters, trial_index)
-        submitted_jobs(1)
-
-        print_debug(f"execute_evaluation: appending job {new_job} to global_vars['jobs'], trial_index: {trial_index}")
-        global_vars["jobs"].append((new_job, trial_index))
+        if new_job:
+            submitted_jobs(1)
 
-        mark_trial_stage("mark_running", "Marking the trial as running failed")
-        trial_counter += 1
+            print_debug(f"execute_evaluation: appending job {new_job} to global_vars['jobs'], trial_index: {trial_index}")
+            global_vars["jobs"].append((new_job, trial_index))
 
-        progressbar_description("started new job")
+            mark_trial_stage("mark_running", "Marking the trial as running failed")
+            trial_counter += 1
 
-        save_results_csv()
+            progressbar_description("started new job")
+        else:
+            progressbar_description("Failed to start new job")
     except submitit.core.utils.FailedJobError as error:
         handle_failed_job(error, trial_index, new_job)
         trial_counter += 1
@@ -7588,7 +8422,7 @@ def handle_failed_job(error: Union[None, Exception, str], trial_index: int, new_
             my_exit(144)
 
     if new_job is None:
-        print_red("handle_failed_job: job is None")
+        print_debug("handle_failed_job: job is None")
 
         return None
 
@@ -7599,16 +8433,24 @@ def handle_failed_job(error: Union[None, Exception, str], trial_index: int, new_
 
     return None
 
+def log_ax_client_trial_failure(trial_index: int) -> None:
+    if not ax_client:
+        my_exit(101)
+
+        return
+
+    ax_client.log_trial_failure(trial_index=trial_index)
+
 def cancel_failed_job(trial_index: int, new_job: Job) -> None:
     print_debug("Trying to cancel job that failed")
     if new_job:
         try:
             if ax_client:
-                ax_client.log_trial_failure(trial_index=trial_index)
+                log_ax_client_trial_failure(trial_index)
             else:
                 _fatal_error("ax_client not defined", 101)
         except Exception as e:
-            print(f"ERROR in line {get_line_info()}: {e}")
+            print_red(f"ERROR in line {get_line_info()}: {e}")
         new_job.cancel()
 
         print_debug(f"cancel_failed_job: removing job {new_job}, trial_index: {trial_index}")
@@ -7644,10 +8486,12 @@ def show_debug_table_for_break_run_search(_name: str, _max_eval: Optional[int])
         ("failed_jobs()", failed_jobs()),
         ("count_done_jobs()", count_done_jobs()),
         ("_max_eval", _max_eval),
-        ("progress_bar.total", progress_bar.total),
         ("NR_INSERTED_JOBS", NR_INSERTED_JOBS)
     ]
 
+    if progress_bar is not None:
+        rows.append(("progress_bar.total", progress_bar.total))
+
     for row in rows:
         table.add_row(str(row[0]), str(row[1]))
 
@@ -7660,6 +8504,8 @@ def break_run_search(_name: str, _max_eval: Optional[int]) -> bool:
     _submitted_jobs = submitted_jobs()
     _failed_jobs = failed_jobs()
 
+    log_data()
+
     max_failed_jobs = max_eval
 
     if args.max_failed_jobs is not None and args.max_failed_jobs > 0:
@@ -7687,11 +8533,11 @@ def break_run_search(_name: str, _max_eval: Optional[int]) -> bool:
             _ret = True
 
     if args.verbose_break_run_search_table:
-        show_debug_table_for_break_run_search(_name, _max_eval, _ret)
+        show_debug_table_for_break_run_search(_name, _max_eval)
 
     return _ret
 
-def _calculate_nr_of_jobs_to_get(simulated_jobs: int, currently_running_jobs: int) -> int:
+def calculate_nr_of_jobs_to_get(simulated_jobs: int, currently_running_jobs: int) -> int:
     """Calculates the number of jobs to retrieve."""
     return min(
         max_eval + simulated_jobs - count_done_jobs(),
@@ -7704,7 +8550,7 @@ def remove_extra_spaces(text: str) -> str:
         raise ValueError("Input must be a string")
     return re.sub(r'\s+', ' ', text).strip()
 
-def _get_trials_message(nr_of_jobs_to_get: int, full_nr_of_jobs_to_get: int, trial_durations: List[float]) -> str:
+def get_trials_message(nr_of_jobs_to_get: int, full_nr_of_jobs_to_get: int, trial_durations: List[float]) -> str:
     """Generates the appropriate message for the number of trials being retrieved."""
     ret = ""
     if full_nr_of_jobs_to_get > 1:
@@ -7789,70 +8635,70 @@ def get_batched_arms(nr_of_jobs_to_get: int) -> list:
         print_red("get_batched_arms: ax_client was None")
         return []
 
-    # Experiment-Status laden
     load_experiment_state()
 
-    while len(batched_arms) != nr_of_jobs_to_get:
+    while len(batched_arms) < nr_of_jobs_to_get:
         if attempts > args.max_attempts_for_generation:
             print_debug(f"get_batched_arms: Stopped after {attempts} attempts: could not generate enough arms "
                         f"(got {len(batched_arms)} out of {nr_of_jobs_to_get}).")
             break
 
-        remaining = nr_of_jobs_to_get - len(batched_arms)
-        print_debug(f"get_batched_arms: Attempt {attempts + 1}: requesting {remaining} more arm(s).")
-
-        #print("get pending observations")
-        pending_observations = get_pending_observation_features(experiment=ax_client.experiment)
-        #print("got pending observations")
+        #print_debug(f"get_batched_arms: Attempt {attempts + 1}: requesting 1 more arm")
 
-        #print("getting global_gs.gen()")
-        batched_generator_run = global_gs.gen(
+        pending_observations = get_pending_observation_features(
             experiment=ax_client.experiment,
-            n=remaining,
-            pending_observations=pending_observations
+            include_out_of_design_points=True
         )
-        #print(f"got global_gs.gen(): {batched_generator_run}")
 
-        # Inline rekursiv entpacken bis flach
+        try:
+            #print_debug("getting global_gs.gen() with n=1")
+
+            batched_generator_run: Any = global_gs.gen(
+                experiment=ax_client.experiment,
+                n=1,
+                pending_observations=pending_observations,
+            )
+            print_debug(f"got global_gs.gen(): {batched_generator_run}")
+        except Exception as e:
+            print_debug(f"global_gs.gen failed: {e}")
+            traceback.print_exception(type(e), e, e.__traceback__, file=sys.stderr)
+            break
+
         depth = 0
         path = "batched_generator_run"
-        while isinstance(batched_generator_run, (list, tuple)) and len(batched_generator_run) > 0:
-            #print(f"Depth {depth}, path {path}, type {type(batched_generator_run).__name__}, length {len(batched_generator_run)}: {batched_generator_run}")
+        while isinstance(batched_generator_run, (list, tuple)) and len(batched_generator_run) == 1:
+            #print_debug(f"Depth {depth}, path {path}, type {type(batched_generator_run).__name__}, length {len(batched_generator_run)}: {batched_generator_run}")
             batched_generator_run = batched_generator_run[0]
             path += "[0]"
             depth += 1
 
-        #print(f"Final flat object at depth {depth}, path {path}: {batched_generator_run} (type {type(batched_generator_run).__name__})")
+        #print_debug(f"Final flat object at depth {depth}, path {path}: {batched_generator_run} (type {type(batched_generator_run).__name__})")
 
-        #print("got new arms")
-        new_arms = batched_generator_run.arms
-        #print(f"new_arms: {new_arms}")
+        new_arms = getattr(batched_generator_run, "arms", [])
         if not new_arms:
             print_debug("get_batched_arms: No new arms were generated in this attempt.")
         else:
-            print_debug(f"get_batched_arms: Generated {len(new_arms)} new arm(s), wanted {nr_of_jobs_to_get}.")
+            print_debug(f"get_batched_arms: Generated {len(new_arms)} new arm(s), now at {len(batched_arms) + len(new_arms)} of {nr_of_jobs_to_get}.")
+            batched_arms.extend(new_arms)
 
-        batched_arms.extend(new_arms)
         attempts += 1
 
     print_debug(f"get_batched_arms: Finished with {len(batched_arms)} arm(s) after {attempts} attempt(s).")
 
-    save_results_csv()
-
     return batched_arms
 
-def _fetch_next_trials(nr_of_jobs_to_get: int, recursion: bool = False) -> Tuple[Dict[int, Any], bool]:
+def fetch_next_trials(nr_of_jobs_to_get: int, recursion: bool = False) -> Tuple[Dict[int, Any], bool]:
     die_101_if_no_ax_client_or_experiment_or_gs()
 
     if not ax_client:
-        _fatal_error("ax_client was not defined", 9)
+        _fatal_error("ax_client was not defined", 101)
 
     if global_gs is None:
         _fatal_error("Global generation strategy is not set. This is a bug in OmniOpt2.", 107)
 
-    return _generate_trials(nr_of_jobs_to_get, recursion)
+    return generate_trials(nr_of_jobs_to_get, recursion)
 
-def _generate_trials(n: int, recursion: bool) -> Tuple[Dict[int, Any], bool]:
+def generate_trials(n: int, recursion: bool) -> Tuple[Dict[int, Any], bool]:
     trials_dict: Dict[int, Any] = {}
     trial_durations: List[float] = []
 
@@ -7866,7 +8712,6 @@ def _generate_trials(n: int, recursion: bool) -> Tuple[Dict[int, Any], bool]:
                 if cnt >= n:
                     break
 
-                # 🔹 Erzeuge einen komplett neuen Arm, damit Ax den Namen vergibt
                 try:
                     arm = Arm(parameters=arm.parameters)
                 except Exception as arm_err:
@@ -7874,16 +8719,14 @@ def _generate_trials(n: int, recursion: bool) -> Tuple[Dict[int, Any], bool]:
                     retries += 1
                     continue
 
-                print_debug(f"Fetching trial {cnt + 1}/{n}...")
-                progressbar_description(_get_trials_message(cnt + 1, n, trial_durations))
+                progressbar_description(get_trials_message(cnt + 1, n, trial_durations))
 
                 try:
-                    result = _create_and_handle_trial(arm)
+                    result = create_and_handle_trial(arm)
                     if result is not None:
                         trial_index, trial_duration, trial_successful = result
-
                 except TrialRejected as e:
-                    print_debug(f"Trial rejected: {e}")
+                    print_debug(f"generate_trials: Trial rejected, error: {e}")
                     retries += 1
                     continue
 
@@ -7893,19 +8736,26 @@ def _generate_trials(n: int, recursion: bool) -> Tuple[Dict[int, Any], bool]:
                     cnt += 1
                     trials_dict[trial_index] = arm.parameters
 
-        save_results_csv()
+        finalized = finalize_generation(trials_dict, cnt, n, start_time)
 
-        return _finalize_generation(trials_dict, cnt, n, start_time)
+        return finalized
 
     except Exception as e:
-        return _handle_generation_failure(e, n, recursion)
+        return handle_generation_failure(e, n, recursion)
 
 class TrialRejected(Exception):
     pass
 
-def _create_and_handle_trial(arm: Any) -> Optional[Tuple[int, float, bool]]:
+def mark_abandoned(trial: Any, reason: str, trial_index: int) -> None:
+    try:
+        print_debug(f"[INFO] Marking trial {trial.index} ({trial.arm.name}) as abandoned, trial-index: {trial_index}. Reason: {reason}")
+        trial.mark_abandoned(reason)
+    except Exception as e:
+        print_red(f"[ERROR] Could not mark trial as abandoned: {e}")
+
+def create_and_handle_trial(arm: Any) -> Optional[Tuple[int, float, bool]]:
     if ax_client is None:
-        print_red("ax_client is None in _create_and_handle_trial")
+        print_red("ax_client is None in create_and_handle_trial")
         return None
 
     start = time.time()
@@ -7928,7 +8778,7 @@ def _create_and_handle_trial(arm: Any) -> Optional[Tuple[int, float, bool]]:
     arm = trial.arms[0]
     if deduplicated_arm(arm):
         print_debug(f"Duplicated arm: {arm}")
-        trial.mark_abandoned(reason="Duplication detected")
+        mark_abandoned(trial, "Duplication detected", trial_index)
         raise TrialRejected("Duplicate arm.")
 
     arms_by_name_for_deduplication[arm.name] = arm
@@ -7936,8 +8786,8 @@ def _create_and_handle_trial(arm: Any) -> Optional[Tuple[int, float, bool]]:
     params = arm.parameters
 
     if not has_no_post_generation_constraints_or_matches_constraints(post_generation_constraints, params):
-        print_debug(f"Trial {trial_index} does not meet post-generation constraints. Marking abandoned.")
-        trial.mark_abandoned(reason="Post-Generation-Constraint failed")
+        print_debug(f"Trial {trial_index} does not meet post-generation constraints. Marking abandoned. Params: {params}, constraints: {post_generation_constraints}")
+        mark_abandoned(trial, "Post-Generation-Constraint failed", trial_index)
         abandoned_trial_indices.append(trial_index)
         raise TrialRejected("Post-generation constraints not met.")
 
@@ -7945,7 +8795,7 @@ def _create_and_handle_trial(arm: Any) -> Optional[Tuple[int, float, bool]]:
     end = time.time()
     return trial_index, float(end - start), True
 
-def _finalize_generation(trials_dict: Dict[int, Any], cnt: int, requested: int, start_time: float) -> Tuple[Dict[int, Any], bool]:
+def finalize_generation(trials_dict: Dict[int, Any], cnt: int, requested: int, start_time: float) -> Tuple[Dict[int, Any], bool]:
     total_time = time.time() - start_time
 
     log_gen_times.append(total_time)
@@ -7956,7 +8806,7 @@ def _finalize_generation(trials_dict: Dict[int, Any], cnt: int, requested: int,
 
     return trials_dict, False
 
-def _handle_generation_failure(
+def handle_generation_failure(
     e: Exception,
     requested: int,
     recursion: bool
@@ -7972,19 +8822,19 @@ def _handle_generation_failure(
     )):
         msg = str(e)
         if msg not in error_8_saved:
-            _print_exhaustion_warning(e, recursion)
+            print_exhaustion_warning(e, recursion)
             error_8_saved.append(msg)
 
         if not recursion and args.revert_to_random_when_seemingly_exhausted:
             print_debug("Switching to random search strategy.")
-            set_global_gs_to_random()
-            return _fetch_next_trials(requested, True)
+            set_global_gs_to_sobol()
+            return fetch_next_trials(requested, True)
 
-    print_red(f"_handle_generation_failure: General Exception: {e}")
+    print_red(f"handle_generation_failure: General Exception: {e}")
 
     return {}, True
 
-def _print_exhaustion_warning(e: Exception, recursion: bool) -> None:
+def print_exhaustion_warning(e: Exception, recursion: bool) -> None:
     if not recursion and args.revert_to_random_when_seemingly_exhausted:
         print_yellow(f"\n⚠Error 8: {e} From now (done jobs: {count_done_jobs()}) on, random points will be generated.")
     else:
@@ -8029,21 +8879,24 @@ def get_model_gen_kwargs() -> dict:
         "fit_out_of_design": args.fit_out_of_design
     }
 
-def set_global_gs_to_random() -> None:
+def set_global_gs_to_sobol() -> None:
     global global_gs
     global overwritten_to_random
 
+    print("Reverting to SOBOL")
+
     global_gs = GenerationStrategy(
         name="Random*",
         nodes=[
             GenerationNode(
-                node_name="Sobol",
-                generator_specs=[
-                    GeneratorSpec(
-                        Models.SOBOL,
-                        model_gen_kwargs=get_model_gen_kwargs()
-                    )
-                ]
+                name="Sobol",
+                should_deduplicate=True,
+                generator_specs=[ # type: ignore[arg-type]
+                    GeneratorSpec( # type: ignore[arg-type]
+                        Generators.SOBOL, # type: ignore[arg-type]
+                        model_gen_kwargs=get_model_gen_kwargs() # type: ignore[arg-type]
+                    ) # type: ignore[arg-type]
+                ] # type: ignore[arg-type]
             )
         ]
     )
@@ -8261,14 +9114,14 @@ def _handle_linalg_error(error: Union[None, str, Exception]) -> None:
     """Handles the np.linalg.LinAlgError based on the model being used."""
     print_red(f"Error: {error}")
 
-def _get_next_trials(nr_of_jobs_to_get: int) -> Tuple[Union[None, dict], bool]:
-    finish_previous_jobs(["finishing jobs (_get_next_trials)"])
+def get_next_trials(nr_of_jobs_to_get: int) -> Tuple[Union[None, dict], bool]:
+    finish_previous_jobs(["finishing jobs (get_next_trials)"])
 
-    if break_run_search("_get_next_trials", max_eval) or nr_of_jobs_to_get == 0:
+    if break_run_search("get_next_trials", max_eval) or nr_of_jobs_to_get == 0:
         return {}, True
 
     try:
-        trial_index_to_param, optimization_complete = _fetch_next_trials(nr_of_jobs_to_get)
+        trial_index_to_param, optimization_complete = fetch_next_trials(nr_of_jobs_to_get)
 
         cf = currentframe()
         if cf:
@@ -8403,16 +9256,21 @@ def get_model_from_name(name: str) -> Any:
             return gen
     raise ValueError(f"Unknown or unsupported model: {name}")
 
-def get_name_from_model(model) -> Optional[str]:
+def get_name_from_model(model: Any) -> str:
     if not isinstance(SUPPORTED_MODELS, (list, set, tuple)):
-        return None
+        raise RuntimeError("get_model_from_name: SUPPORTED_MODELS was not a list, set or tuple. Cannot continue")
 
     model_str = model.value if hasattr(model, "value") else str(model)
 
     model_str_lower = model_str.lower()
     model_map = {m.lower(): m for m in SUPPORTED_MODELS}
 
-    return model_map.get(model_str_lower, None)
+    ret = model_map.get(model_str_lower, None)
+
+    if ret is None:
+        raise RuntimeError("get_name_from_model: failed to get Model")
+
+    return ret
 
 def parse_generation_strategy_string(gen_strat_str: str) -> tuple[list[dict[str, int]], int]:
     gen_strat_list = []
@@ -8423,10 +9281,10 @@ def parse_generation_strategy_string(gen_strat_str: str) -> tuple[list[dict[str,
 
     for s in splitted_by_comma:
         if "=" not in s:
-            print(f"'{s}' does not contain '='")
+            print_red(f"'{s}' does not contain '='")
             my_exit(123)
         if s.count("=") != 1:
-            print(f"There can only be one '=' in the gen_strat_str's element '{s}'")
+            print_red(f"There can only be one '=' in the gen_strat_str's element '{s}'")
             my_exit(123)
 
         model_name, nr_str = s.split("=")
@@ -8436,13 +9294,13 @@ def parse_generation_strategy_string(gen_strat_str: str) -> tuple[list[dict[str,
             _fatal_error(f"Model {matching_model} is not valid for custom generation strategy.", 56)
 
         if not matching_model:
-            print(f"'{model_name}' not found in SUPPORTED_MODELS")
+            print_red(f"'{model_name}' not found in SUPPORTED_MODELS")
             my_exit(123)
 
         try:
             nr = int(nr_str)
         except ValueError:
-            print(f"Invalid number of generations '{nr_str}' for model '{model_name}'")
+            print_red(f"Invalid number of generations '{nr_str}' for model '{model_name}'")
             my_exit(123)
 
         gen_strat_list.append({matching_model: nr})
@@ -8566,7 +9424,7 @@ def create_node(model_name: str, threshold: int, next_model_name: Optional[str])
     if model_name == "TPE":
         if len(arg_result_names) != 1:
             _fatal_error(f"Has {len(arg_result_names)} results. TPE currently only supports single-objective-optimization.", 108)
-        return ExternalProgramGenerationNode(external_generator=f"python3 {script_dir}/.tpe.py", node_name="EXTERNAL_GENERATOR")
+        return ExternalProgramGenerationNode(external_generator=f"python3 {script_dir}/.tpe.py", name="EXTERNAL_GENERATOR")
 
     external_generators = {
         "PSEUDORANDOM": f"python3 {script_dir}/.random_generator.py",
@@ -8580,10 +9438,10 @@ def create_node(model_name: str, threshold: int, next_model_name: Optional[str])
         cmd = external_generators[model_name]
         if model_name == "EXTERNAL_GENERATOR" and not cmd:
             _fatal_error("--external_generator is missing. Cannot create points for EXTERNAL_GENERATOR without it.", 204)
-        return ExternalProgramGenerationNode(external_generator=cmd, node_name="EXTERNAL_GENERATOR")
+        return ExternalProgramGenerationNode(external_generator=cmd, name="EXTERNAL_GENERATOR")
 
     trans_crit = [
-        MaxTrials(
+        MinTrials(
             threshold=threshold,
             block_transition_if_unmet=True,
             transition_to=target_model,
@@ -8599,11 +9457,12 @@ def create_node(model_name: str, threshold: int, next_model_name: Optional[str])
     if model_name.lower() != "sobol":
         kwargs["model_kwargs"] = get_model_kwargs()
 
-    model_spec = [GeneratorSpec(selected_model, **kwargs)]
+    model_spec = [GeneratorSpec(selected_model, **kwargs)] # type: ignore[arg-type]
 
     res = GenerationNode(
-        node_name=model_name,
+        name=model_name,
         generator_specs=model_spec,
+        should_deduplicate=True,
         transition_criteria=trans_crit
     )
 
@@ -8614,11 +9473,11 @@ def get_optimizer_kwargs() -> dict:
         "sequential": False
     }
 
-def create_step(model_name: str, _num_trials: int = -1, index: Optional[int] = None) -> GenerationStep:
+def create_step(model_name: str, _num_trials: int, index: int) -> GenerationStep:
     model_enum = get_model_from_name(model_name)
 
     return GenerationStep(
-        generator=model_enum,   # ✅ neue API
+        generator=model_enum,
         num_trials=_num_trials,
         max_parallelism=1000 * max_eval + 1000,
         model_kwargs=get_model_kwargs(),
@@ -8629,17 +9488,16 @@ def create_step(model_name: str, _num_trials: int = -1, index: Optional[int] = N
     )
 
 def set_global_generation_strategy() -> None:
-    with spinner("Setting global generation strategy"):
-        continue_not_supported_on_custom_generation_strategy()
+    continue_not_supported_on_custom_generation_strategy()
 
-        try:
-            if args.generation_strategy is None:
-                setup_default_generation_strategy()
-            else:
-                setup_custom_generation_strategy()
-        except Exception as e:
-            print_red(f"Unexpected error in generation strategy setup: {e}")
-            my_exit(111)
+    try:
+        if args.generation_strategy is None:
+            setup_default_generation_strategy()
+        else:
+            setup_custom_generation_strategy()
+    except Exception as e:
+        print_red(f"Unexpected error in generation strategy setup: {e}")
+        my_exit(111)
 
     if global_gs is None:
         print_red("global_gs is None after setup!")
@@ -8775,10 +9633,6 @@ def execute_trials(
         index_param_list.append(_args)
         i += 1
 
-        save_results_csv()
-
-    save_results_csv()
-
     start_time = time.time()
 
     cnt = 0
@@ -8794,10 +9648,14 @@ def execute_trials(
                 result = future.result()
                 print_debug(f"result in execute_trials: {result}")
             except Exception as exc:
-                print_red(f"execute_trials: Error at executing a trial: {exc}")
+                failed_args = future_to_args[future]
+                print_red(f"execute_trials: Error at executing a trial with args {failed_args}: {exc}")
+                traceback.print_exc()
 
     end_time = time.time()
 
+    log_data()
+
     duration = float(end_time - start_time)
     job_submit_durations.append(duration)
     job_submit_nrs.append(cnt)
@@ -8831,20 +9689,20 @@ def create_and_execute_next_runs(next_nr_steps: int, phase: Optional[str], _max_
     done_optimizing: bool = False
 
     try:
-        done_optimizing, trial_index_to_param = _create_and_execute_next_runs_run_loop(_max_eval, phase)
-        _create_and_execute_next_runs_finish(done_optimizing)
+        done_optimizing, trial_index_to_param = create_and_execute_next_runs_run_loop(_max_eval, phase)
+        create_and_execute_next_runs_finish(done_optimizing)
     except Exception as e:
         stacktrace = traceback.format_exc()
         print_debug(f"Warning: create_and_execute_next_runs encountered an exception: {e}\n{stacktrace}")
         return handle_exceptions_create_and_execute_next_runs(e)
 
-    return _create_and_execute_next_runs_return_value(trial_index_to_param)
+    return create_and_execute_next_runs_return_value(trial_index_to_param)
 
-def _create_and_execute_next_runs_run_loop(_max_eval: Optional[int], phase: Optional[str]) -> Tuple[bool, Optional[Dict]]:
+def create_and_execute_next_runs_run_loop(_max_eval: Optional[int], phase: Optional[str]) -> Tuple[bool, Optional[Dict]]:
     done_optimizing = False
     trial_index_to_param: Optional[Dict] = None
 
-    nr_of_jobs_to_get = _calculate_nr_of_jobs_to_get(get_nr_of_imported_jobs(), len(global_vars["jobs"]))
+    nr_of_jobs_to_get = calculate_nr_of_jobs_to_get(get_nr_of_imported_jobs(), len(global_vars["jobs"]))
 
     __max_eval = _max_eval if _max_eval is not None else 0
     new_nr_of_jobs_to_get = min(__max_eval - (submitted_jobs() - failed_jobs()), nr_of_jobs_to_get)
@@ -8857,7 +9715,8 @@ def _create_and_execute_next_runs_run_loop(_max_eval: Optional[int], phase: Opti
         get_next_trials_nr = new_nr_of_jobs_to_get
 
     for _ in range(range_nr):
-        trial_index_to_param, done_optimizing = _get_next_trials(get_next_trials_nr)
+        trial_index_to_param, done_optimizing = get_next_trials(get_next_trials_nr)
+        log_data()
         if done_optimizing:
             continue
 
@@ -8882,13 +9741,13 @@ def _create_and_execute_next_runs_run_loop(_max_eval: Optional[int], phase: Opti
 
     return done_optimizing, trial_index_to_param
 
-def _create_and_execute_next_runs_finish(done_optimizing: bool) -> None:
+def create_and_execute_next_runs_finish(done_optimizing: bool) -> None:
     finish_previous_jobs(["finishing jobs"])
 
     if done_optimizing:
         end_program(False, 0)
 
-def _create_and_execute_next_runs_return_value(trial_index_to_param: Optional[Dict]) -> int:
+def create_and_execute_next_runs_return_value(trial_index_to_param: Optional[Dict]) -> int:
     try:
         if trial_index_to_param:
             res = len(trial_index_to_param.keys())
@@ -8999,7 +9858,7 @@ def execute_nvidia_smi() -> None:
                 if not host:
                     print_debug("host not defined")
         except Exception as e:
-            print(f"execute_nvidia_smi: An error occurred: {e}")
+            print_red(f"execute_nvidia_smi: An error occurred: {e}")
         if is_slurm_job() and not args.force_local_execution:
             _sleep(30)
 
@@ -9037,11 +9896,6 @@ def run_search() -> bool:
 
     return False
 
-async def start_logging_daemon() -> None:
-    while True:
-        log_data()
-        time.sleep(30)
-
 def should_break_search() -> bool:
     ret = False
 
@@ -9105,7 +9959,7 @@ def finalize_jobs() -> None:
         handle_slurm_execution()
 
 def go_through_jobs_that_are_not_completed_yet() -> None:
-    print_debug(f"Waiting for jobs to finish (currently, len(global_vars['jobs']) = {len(global_vars['jobs'])}")
+    #print_debug(f"Waiting for jobs to finish (currently, len(global_vars['jobs']) = {len(global_vars['jobs'])}")
 
     nr_jobs_left = len(global_vars['jobs'])
     if nr_jobs_left == 1:
@@ -9175,7 +10029,7 @@ def parse_orchestrator_file(_f: str, _test: bool = False) -> Union[dict, None]:
 
                 return data
             except Exception as e:
-                print(f"Error while parse_experiment_parameters({_f}): {e}")
+                print_red(f"Error while parse_experiment_parameters({_f}): {e}")
     else:
         print_red(f"{_f} could not be found")
 
@@ -9192,355 +10046,43 @@ def set_orchestrator() -> None:
                 print_yellow("--orchestrator_file will be ignored on non-sbatch-systems.")
 
 def check_if_has_random_steps() -> None:
-    if (not args.continue_previous_job and "--continue" not in sys.argv) and (args.num_random_steps == 0 or not args.num_random_steps) and args.model not in ["EXTERNAL_GENERATOR", "SOBOL", "PSEUDORANDOM"]:
-        _fatal_error("You have no random steps set. This is only allowed in continued jobs. To start, you need either some random steps, or a continued run.", 233)
-
-def add_exclude_to_defective_nodes() -> None:
-    with spinner("Adding excluded nodes..."):
-        if args.exclude:
-            entries = [entry.strip() for entry in args.exclude.split(',')]
-
-            for entry in entries:
-                count_defective_nodes(None, entry)
-
-def check_max_eval(_max_eval: int) -> None:
-    with spinner("Checking max_eval..."):
-        if not _max_eval:
-            _fatal_error("--max_eval needs to be set!", 19)
-
-def parse_parameters() -> Any:
-    cli_params_experiment_parameters = None
-    if args.parameter:
-        parse_experiment_parameters()
-        cli_params_experiment_parameters = experiment_parameters
-
-    return cli_params_experiment_parameters
-
-def create_pareto_front_table(idxs: List[int], metric_x: str, metric_y: str) -> Table:
-    table = Table(title=f"Pareto-Front for {metric_y}/{metric_x}:", show_lines=True)
-
-    rows = _pareto_front_table_read_csv()
-    if not rows:
-        table.add_column("No data found")
-        return table
-
-    filtered_rows = _pareto_front_table_filter_rows(rows, idxs)
-    if not filtered_rows:
-        table.add_column("No matching entries")
-        return table
-
-    param_cols, result_cols = _pareto_front_table_get_columns(filtered_rows[0])
-
-    _pareto_front_table_add_headers(table, param_cols, result_cols)
-    _pareto_front_table_add_rows(table, filtered_rows, param_cols, result_cols)
-
-    return table
-
-def _pareto_front_table_read_csv() -> List[Dict[str, str]]:
-    with open(RESULT_CSV_FILE, mode="r", encoding="utf-8", newline="") as f:
-        return list(csv.DictReader(f))
-
-def _pareto_front_table_filter_rows(rows: List[Dict[str, str]], idxs: List[int]) -> List[Dict[str, str]]:
-    result = []
-    for row in rows:
-        try:
-            trial_index = int(row["trial_index"])
-        except (KeyError, ValueError):
-            continue
-
-        if row.get("trial_status", "").strip().upper() == "COMPLETED" and trial_index in idxs:
-            result.append(row)
-    return result
-
-def _pareto_front_table_get_columns(first_row: Dict[str, str]) -> Tuple[List[str], List[str]]:
-    all_columns = list(first_row.keys())
-    ignored_cols = set(special_col_names) - {"trial_index"}
-
-    param_cols = [col for col in all_columns if col not in ignored_cols and col not in arg_result_names and not col.startswith("OO_Info_")]
-    result_cols = [col for col in arg_result_names if col in all_columns]
-    return param_cols, result_cols
-
-def _pareto_front_table_add_headers(table: Table, param_cols: List[str], result_cols: List[str]) -> None:
-    for col in param_cols:
-        table.add_column(col, justify="center")
-    for col in result_cols:
-        table.add_column(Text(f"{col}", style="cyan"), justify="center")
-
-def _pareto_front_table_add_rows(table: Table, rows: List[Dict[str, str]], param_cols: List[str], result_cols: List[str]) -> None:
-    for row in rows:
-        values = [str(helpers.to_int_when_possible(row[col])) for col in param_cols]
-        result_values = [Text(str(helpers.to_int_when_possible(row[col])), style="cyan") for col in result_cols]
-        table.add_row(*values, *result_values, style="bold green")
-
-def pareto_front_as_rich_table(idxs: list, metric_x: str, metric_y: str) -> Optional[Table]:
-    if not os.path.exists(RESULT_CSV_FILE):
-        print_debug(f"pareto_front_as_rich_table: File '{RESULT_CSV_FILE}' not found")
-        return None
-
-    return create_pareto_front_table(idxs, metric_x, metric_y)
-
-def supports_sixel() -> bool:
-    term = os.environ.get("TERM", "").lower()
-    if "xterm" in term or "mlterm" in term:
-        return True
-
-    try:
-        output = subprocess.run(["tput", "setab", "256"], capture_output=True, text=True, check=True)
-        if output.returncode == 0 and "sixel" in output.stdout.lower():
-            return True
-    except (subprocess.CalledProcessError, FileNotFoundError):
-        pass
-
-    return False
-
-def plot_pareto_frontier_sixel(data: Any, x_metric: str, y_metric: str) -> None:
-    if data is None:
-        print("[italic yellow]The data seems to be empty. Cannot plot pareto frontier.[/]")
-        return
-
-    if not supports_sixel():
-        print(f"[italic yellow]Your console does not support sixel-images. Will not print Pareto-frontier as a matplotlib-sixel-plot for {x_metric}/{y_metric}.[/]")
-        return
-
-    import matplotlib.pyplot as plt
-
-    means = data[x_metric][y_metric]["means"]
-
-    x_values = means[x_metric]
-    y_values = means[y_metric]
-
-    fig, _ax = plt.subplots()
-
-    _ax.scatter(x_values, y_values, s=50, marker='x', c='blue', label='Data Points')
-
-    _ax.set_xlabel(x_metric)
-    _ax.set_ylabel(y_metric)
-
-    _ax.set_title(f'Pareto-Front {x_metric}/{y_metric}')
-
-    _ax.ticklabel_format(style='plain', axis='both', useOffset=False)
-
-    with tempfile.NamedTemporaryFile(suffix=".png", delete=True) as tmp_file:
-        plt.savefig(tmp_file.name, dpi=300)
-
-        print_image_to_cli(tmp_file.name, 1000)
-
-    plt.close(fig)
-
-def _pareto_front_general_validate_shapes(x: np.ndarray, y: np.ndarray) -> None:
-    if x.shape != y.shape:
-        raise ValueError("Input arrays x and y must have the same shape.")
-
-def _pareto_front_general_compare(
-    xi: float, yi: float, xj: float, yj: float,
-    x_minimize: bool, y_minimize: bool
-) -> bool:
-    x_better_eq = xj <= xi if x_minimize else xj >= xi
-    y_better_eq = yj <= yi if y_minimize else yj >= yi
-    x_strictly_better = xj < xi if x_minimize else xj > xi
-    y_strictly_better = yj < yi if y_minimize else yj > yi
-
-    return bool(x_better_eq and y_better_eq and (x_strictly_better or y_strictly_better))
-
-def _pareto_front_general_find_dominated(
-    x: np.ndarray, y: np.ndarray, x_minimize: bool, y_minimize: bool
-) -> np.ndarray:
-    num_points = len(x)
-    is_dominated = np.zeros(num_points, dtype=bool)
-
-    for i in range(num_points):
-        for j in range(num_points):
-            if i == j:
-                continue
-
-            if _pareto_front_general_compare(x[i], y[i], x[j], y[j], x_minimize, y_minimize):
-                is_dominated[i] = True
-                break
-
-    return is_dominated
-
-def pareto_front_general(
-    x: np.ndarray,
-    y: np.ndarray,
-    x_minimize: bool = True,
-    y_minimize: bool = True
-) -> np.ndarray:
-    try:
-        _pareto_front_general_validate_shapes(x, y)
-        is_dominated = _pareto_front_general_find_dominated(x, y, x_minimize, y_minimize)
-        return np.where(~is_dominated)[0]
-    except Exception as e:
-        print("Error in pareto_front_general:", str(e))
-        return np.array([], dtype=int)
-
-def _pareto_front_aggregate_data(path_to_calculate: str) -> Optional[Dict[Tuple[int, str], Dict[str, Dict[str, float]]]]:
-    results_csv_file = f"{path_to_calculate}/{RESULTS_CSV_FILENAME}"
-    result_names_file = f"{path_to_calculate}/result_names.txt"
-
-    if not os.path.exists(results_csv_file) or not os.path.exists(result_names_file):
-        return None
-
-    with open(result_names_file, mode="r", encoding="utf-8") as f:
-        result_names = [line.strip() for line in f if line.strip()]
-
-    records: dict = defaultdict(lambda: {'means': {}})
-
-    with open(results_csv_file, encoding="utf-8", mode="r", newline='') as csvfile:
-        reader = csv.DictReader(csvfile)
-        for row in reader:
-            trial_index = int(row['trial_index'])
-            arm_name = row['arm_name']
-            key = (trial_index, arm_name)
-
-            for metric in result_names:
-                if metric in row:
-                    try:
-                        records[key]['means'][metric] = float(row[metric])
-                    except ValueError:
-                        continue
-
-    return records
-
-def _pareto_front_filter_complete_points(
-    path_to_calculate: str,
-    records: Dict[Tuple[int, str], Dict[str, Dict[str, float]]],
-    primary_name: str,
-    secondary_name: str
-) -> List[Tuple[Tuple[int, str], float, float]]:
-    points = []
-    for key, metrics in records.items():
-        means = metrics['means']
-        if primary_name in means and secondary_name in means:
-            x_val = means[primary_name]
-            y_val = means[secondary_name]
-            points.append((key, x_val, y_val))
-    if len(points) == 0:
-        raise ValueError(f"No full data points with both objectives found in {path_to_calculate}.")
-    return points
-
-def _pareto_front_transform_objectives(
-    points: List[Tuple[Any, float, float]],
-    primary_name: str,
-    secondary_name: str
-) -> Tuple[np.ndarray, np.ndarray]:
-    primary_idx = arg_result_names.index(primary_name)
-    secondary_idx = arg_result_names.index(secondary_name)
-
-    x = np.array([p[1] for p in points])
-    y = np.array([p[2] for p in points])
-
-    if arg_result_min_or_max[primary_idx] == "max":
-        x = -x
-    elif arg_result_min_or_max[primary_idx] != "min":
-        raise ValueError(f"Unknown mode for {primary_name}: {arg_result_min_or_max[primary_idx]}")
-
-    if arg_result_min_or_max[secondary_idx] == "max":
-        y = -y
-    elif arg_result_min_or_max[secondary_idx] != "min":
-        raise ValueError(f"Unknown mode for {secondary_name}: {arg_result_min_or_max[secondary_idx]}")
-
-    return x, y
-
-def _pareto_front_select_pareto_points(
-    x: np.ndarray,
-    y: np.ndarray,
-    x_minimize: bool,
-    y_minimize: bool,
-    points: List[Tuple[Any, float, float]],
-    num_points: int
-) -> List[Tuple[Any, float, float]]:
-    indices = pareto_front_general(x, y, x_minimize, y_minimize)
-    sorted_indices = indices[np.argsort(x[indices])]
-    sorted_indices = sorted_indices[:num_points]
-    selected_points = [points[i] for i in sorted_indices]
-    return selected_points
-
-def _pareto_front_build_return_structure(
-    path_to_calculate: str,
-    selected_points: List[Tuple[Any, float, float]],
-    records: Dict[Tuple[int, str], Dict[str, Dict[str, float]]],
-    absolute_metrics: List[str],
-    primary_name: str,
-    secondary_name: str
-) -> dict:
-    results_csv_file = f"{path_to_calculate}/{RESULTS_CSV_FILENAME}"
-    result_names_file = f"{path_to_calculate}/result_names.txt"
-
-    with open(result_names_file, mode="r", encoding="utf-8") as f:
-        result_names = [line.strip() for line in f if line.strip()]
-
-    csv_rows = {}
-    with open(results_csv_file, mode="r", encoding="utf-8", newline='') as csvfile:
-        reader = csv.DictReader(csvfile)
-        for row in reader:
-            trial_index = int(row['trial_index'])
-            csv_rows[trial_index] = row
-
-    ignored_columns = {'trial_index', 'arm_name', 'trial_status', 'generation_node'}
-    ignored_columns.update(result_names)
-
-    param_dicts = []
-    idxs = []
-    means_dict = defaultdict(list)
-
-    for (trial_index, arm_name), _, _ in selected_points:
-        row = csv_rows.get(trial_index, {})
-        if row == {} or row is None or row['arm_name'] != arm_name:
-            print_debug(f"_pareto_front_build_return_structure: trial_index '{trial_index}' could not be found and row returned as None")
-            continue
-
-        idxs.append(int(row["trial_index"]))
-
-        param_dict: dict[str, int | float | str] = {}
-        for key, value in row.items():
-            if key not in ignored_columns:
-                try:
-                    param_dict[key] = int(value)
-                except ValueError:
-                    try:
-                        param_dict[key] = float(value)
-                    except ValueError:
-                        param_dict[key] = value
+    if (not args.continue_previous_job and "--continue" not in sys.argv) and (args.num_random_steps == 0 or not args.num_random_steps) and args.model not in ["EXTERNAL_GENERATOR", "SOBOL", "PSEUDORANDOM"]:
+        _fatal_error("You have no random steps set. This is only allowed in continued jobs. To start, you need either some random steps, or a continued run.", 233)
 
-        param_dicts.append(param_dict)
+def add_exclude_to_defective_nodes() -> None:
+    with spinner("Adding excluded nodes..."):
+        if args.exclude:
+            entries = [entry.strip() for entry in args.exclude.split(',')]
 
-        for metric in absolute_metrics:
-            means_dict[metric].append(records[(trial_index, arm_name)]['means'].get(metric, float("nan")))
+            for entry in entries:
+                count_defective_nodes(None, entry)
 
-    ret = {
-        primary_name: {
-            secondary_name: {
-                "absolute_metrics": absolute_metrics,
-                "param_dicts": param_dicts,
-                "means": dict(means_dict),
-                "idxs": idxs
-            },
-            "absolute_metrics": absolute_metrics
-        }
-    }
+def check_max_eval(_max_eval: int) -> None:
+    with spinner("Checking max_eval..."):
+        if not _max_eval:
+            _fatal_error("--max_eval needs to be set!", 19)
 
-    return ret
+def parse_parameters() -> Any:
+    cli_params_experiment_parameters = None
+    if args.parameter:
+        parse_experiment_parameters()
+        cli_params_experiment_parameters = experiment_parameters
 
-def get_pareto_frontier_points(
-    path_to_calculate: str,
-    primary_objective: str,
-    secondary_objective: str,
-    x_minimize: bool,
-    y_minimize: bool,
-    absolute_metrics: List[str],
-    num_points: int
-) -> Optional[dict]:
-    records = _pareto_front_aggregate_data(path_to_calculate)
+    return cli_params_experiment_parameters
 
-    if records is None:
-        return None
+def supports_sixel() -> bool:
+    term = os.environ.get("TERM", "").lower()
+    if "xterm" in term or "mlterm" in term:
+        return True
 
-    points = _pareto_front_filter_complete_points(path_to_calculate, records, primary_objective, secondary_objective)
-    x, y = _pareto_front_transform_objectives(points, primary_objective, secondary_objective)
-    selected_points = _pareto_front_select_pareto_points(x, y, x_minimize, y_minimize, points, num_points)
-    result = _pareto_front_build_return_structure(path_to_calculate, selected_points, records, absolute_metrics, primary_objective, secondary_objective)
+    try:
+        output = subprocess.run(["tput", "setab", "256"], capture_output=True, text=True, check=True)
+        if output.returncode == 0 and "sixel" in output.stdout.lower():
+            return True
+    except (subprocess.CalledProcessError, FileNotFoundError):
+        pass
 
-    return result
+    return False
 
 def save_experiment_state() -> None:
     try:
@@ -9548,14 +10090,14 @@ def save_experiment_state() -> None:
             print_red("save_experiment_state: ax_client or ax_client.experiment is None, cannot save.")
             return
         state_path = get_current_run_folder("experiment_state.json")
-        ax_client.save_to_json_file(state_path)
+        save_ax_client_to_json_file(state_path)
     except Exception as e:
-        print(f"Error saving experiment state: {e}")
+        print_debug(f"Error saving experiment state: {e}")
 
 def wait_for_state_file(state_path: str, min_size: int = 5, max_wait_seconds: int = 60) -> bool:
     try:
         if not os.path.exists(state_path):
-            print(f"[ERROR] File '{state_path}' does not exist.")
+            print_debug(f"[ERROR] File '{state_path}' does not exist.")
             return False
 
         i = 0
@@ -9610,7 +10152,6 @@ def load_experiment_state() -> None:
     state_path = get_current_run_folder("experiment_state.json")
 
     if not os.path.exists(state_path):
-        print(f"State file {state_path} does not exist, starting fresh")
         return
 
     if args.worker_generator_path:
@@ -9624,10 +10165,10 @@ def load_experiment_state() -> None:
         return
 
     try:
-        arms_seen = {}
+        arms_seen: dict = {}
         for arm in data.get("arms", []):
             name = arm.get("name")
-            sig = arm.get("parameters")  # grobe Signatur
+            sig = arm.get("parameters")
             if not name:
                 continue
             if name in arms_seen and arms_seen[name] != sig:
@@ -9636,7 +10177,6 @@ def load_experiment_state() -> None:
                 arm["name"] = new_name
             arms_seen[name] = sig
 
-        # Gefilterten Zustand speichern und laden
         temp_path = state_path + ".no_conflicts.json"
         with open(temp_path, encoding="utf-8", mode="w") as f:
             json.dump(data, f)
@@ -9776,38 +10316,47 @@ def get_result_minimize_flag(path_to_calculate: str, resname: str) -> bool:
 
     return minmax[index] == "min"
 
-def get_pareto_front_data(path_to_calculate: str, res_names: list) -> dict:
-    pareto_front_data: dict = {}
+def post_job_calculate_pareto_front() -> None:
+    if not args.calculate_pareto_front_of_job:
+        return
 
-    all_combinations = list(combinations(range(len(arg_result_names)), 2))
+    failure = False
 
-    skip = False
+    _paths_to_calculate = []
 
-    for i, j in all_combinations:
-        if not skip:
-            metric_x = arg_result_names[i]
-            metric_y = arg_result_names[j]
+    for _path_to_calculate in list(set(args.calculate_pareto_front_of_job)):
+        try:
+            found_paths = find_results_paths(_path_to_calculate)
 
-            x_minimize = get_result_minimize_flag(path_to_calculate, metric_x)
-            y_minimize = get_result_minimize_flag(path_to_calculate, metric_y)
+            for _fp in found_paths:
+                if _fp not in _paths_to_calculate:
+                    _paths_to_calculate.append(_fp)
+        except (FileNotFoundError, NotADirectoryError) as e:
+            print_red(f"post_job_calculate_pareto_front: find_results_paths('{_path_to_calculate}') failed with {e}")
 
-            try:
-                if metric_x not in pareto_front_data:
-                    pareto_front_data[metric_x] = {}
+            failure = True
 
-                pareto_front_data[metric_x][metric_y] = get_calculated_frontier(path_to_calculate, metric_x, metric_y, x_minimize, y_minimize, res_names)
-            except ax.exceptions.core.DataRequiredError as e:
-                print_red(f"Error computing Pareto frontier for {metric_x} and {metric_y}: {e}")
-            except SignalINT:
-                print_red("Calculating Pareto-fronts was cancelled by pressing CTRL-c")
-                skip = True
+    for _path_to_calculate in _paths_to_calculate:
+        for path_to_calculate in found_paths:
+            if not job_calculate_pareto_front(path_to_calculate):
+                failure = True
 
-    return pareto_front_data
+    if failure:
+        my_exit(24)
+
+    my_exit(0)
+
+def pareto_front_as_rich_table(idxs: list, metric_x: str, metric_y: str) -> Optional[Table]:
+    if not os.path.exists(RESULT_CSV_FILE):
+        print_debug(f"pareto_front_as_rich_table: File '{RESULT_CSV_FILE}' not found")
+        return None
+
+    return create_pareto_front_table(idxs, metric_x, metric_y)
 
 def show_pareto_frontier_data(path_to_calculate: str, res_names: list, disable_sixel_and_table: bool = False) -> None:
     if len(res_names) <= 1:
         print_debug(f"--result_names (has {len(res_names)} entries) must be at least 2.")
-        return
+        return None
 
     pareto_front_data: dict = get_pareto_front_data(path_to_calculate, res_names)
 
@@ -9828,8 +10377,16 @@ def show_pareto_frontier_data(path_to_calculate: str, res_names: list, disable_s
                 else:
                     print(f"Not showing Pareto-front-sixel for {path_to_calculate}")
 
-            if len(calculated_frontier[metric_x][metric_y]["idxs"]):
-                pareto_points[metric_x][metric_y] = sorted(calculated_frontier[metric_x][metric_y]["idxs"])
+            if calculated_frontier is None:
+                print_debug("ERROR: calculated_frontier is None")
+                return None
+
+            try:
+                if len(calculated_frontier[metric_x][metric_y]["idxs"]):
+                    pareto_points[metric_x][metric_y] = sorted(calculated_frontier[metric_x][metric_y]["idxs"])
+            except AttributeError:
+                print_debug(f"ERROR: calculated_frontier structure invalid for ({metric_x}, {metric_y})")
+                return None
 
             rich_table = pareto_front_as_rich_table(
                 calculated_frontier[metric_x][metric_y]["idxs"],
@@ -9854,6 +10411,8 @@ def show_pareto_frontier_data(path_to_calculate: str, res_names: list, disable_s
 
     live_share_after_pareto()
 
+    return None
+
 def show_available_hardware_and_generation_strategy_string(gpu_string: str, gpu_color: str) -> None:
     cpu_count = os.cpu_count()
 
@@ -9865,9 +10424,11 @@ def show_available_hardware_and_generation_strategy_string(gpu_string: str, gpu_
         pass
 
     if gpu_string:
-        console.print(f"[green]You have {cpu_count} CPUs available for the main process.[/green] [{gpu_color}]{gpu_string}[/{gpu_color}] [green]{gs_string}[/green]")
+        console.print(f"[green]You have {cpu_count} CPUs available for the main process.[/green] [{gpu_color}]{gpu_string}[/{gpu_color}]")
     else:
-        print_green(f"You have {cpu_count} CPUs available for the main process. {gs_string}")
+        print_green(f"You have {cpu_count} CPUs available for the main process.")
+
+    print_green(gs_string)
 
 def write_args_overview_table() -> None:
     table = Table(title="Arguments Overview")
@@ -10134,112 +10695,6 @@ def find_results_paths(base_path: str) -> list:
 
     return list(set(found_paths))
 
-def post_job_calculate_pareto_front() -> None:
-    if not args.calculate_pareto_front_of_job:
-        return
-
-    failure = False
-
-    _paths_to_calculate = []
-
-    for _path_to_calculate in list(set(args.calculate_pareto_front_of_job)):
-        try:
-            found_paths = find_results_paths(_path_to_calculate)
-
-            for _fp in found_paths:
-                if _fp not in _paths_to_calculate:
-                    _paths_to_calculate.append(_fp)
-        except (FileNotFoundError, NotADirectoryError) as e:
-            print_red(f"post_job_calculate_pareto_front: find_results_paths('{_path_to_calculate}') failed with {e}")
-
-            failure = True
-
-    for _path_to_calculate in _paths_to_calculate:
-        for path_to_calculate in found_paths:
-            if not job_calculate_pareto_front(path_to_calculate):
-                failure = True
-
-    if failure:
-        my_exit(24)
-
-    my_exit(0)
-
-def job_calculate_pareto_front(path_to_calculate: str, disable_sixel_and_table: bool = False) -> bool:
-    pf_start_time = time.time()
-
-    if not path_to_calculate:
-        return False
-
-    global CURRENT_RUN_FOLDER
-    global RESULT_CSV_FILE
-    global arg_result_names
-
-    if not path_to_calculate:
-        print_red("Can only calculate pareto front of previous job when --calculate_pareto_front_of_job is set")
-        return False
-
-    if not os.path.exists(path_to_calculate):
-        print_red(f"Path '{path_to_calculate}' does not exist")
-        return False
-
-    ax_client_json = f"{path_to_calculate}/state_files/ax_client.experiment.json"
-
-    if not os.path.exists(ax_client_json):
-        print_red(f"Path '{ax_client_json}' not found")
-        return False
-
-    checkpoint_file: str = f"{path_to_calculate}/state_files/checkpoint.json"
-    if not os.path.exists(checkpoint_file):
-        print_red(f"The checkpoint file '{checkpoint_file}' does not exist")
-        return False
-
-    RESULT_CSV_FILE = f"{path_to_calculate}/{RESULTS_CSV_FILENAME}"
-    if not os.path.exists(RESULT_CSV_FILE):
-        print_red(f"{RESULT_CSV_FILE} not found")
-        return False
-
-    res_names = []
-
-    res_names_file = f"{path_to_calculate}/result_names.txt"
-    if not os.path.exists(res_names_file):
-        print_red(f"File '{res_names_file}' does not exist")
-        return False
-
-    try:
-        with open(res_names_file, "r", encoding="utf-8") as file:
-            lines = file.readlines()
-    except Exception as e:
-        print_red(f"Error reading file '{res_names_file}': {e}")
-        return False
-
-    for line in lines:
-        entry = line.strip()
-        if entry != "":
-            res_names.append(entry)
-
-    if len(res_names) < 2:
-        print_red(f"Error: There are less than 2 result names (is: {len(res_names)}, {', '.join(res_names)}) in {path_to_calculate}. Cannot continue calculating the pareto front.")
-        return False
-
-    load_username_to_args(path_to_calculate)
-
-    CURRENT_RUN_FOLDER = path_to_calculate
-
-    arg_result_names = res_names
-
-    load_experiment_parameters_from_checkpoint_file(checkpoint_file, False)
-
-    if experiment_parameters is None:
-        return False
-
-    show_pareto_or_error_msg(path_to_calculate, res_names, disable_sixel_and_table)
-
-    pf_end_time = time.time()
-
-    print_debug(f"Calculating the Pareto-front took {pf_end_time - pf_start_time} seconds")
-
-    return True
-
 def set_arg_states_from_continue() -> None:
     if args.continue_previous_job and not args.num_random_steps:
         num_random_steps_file = f"{args.continue_previous_job}/state_files/num_random_steps"
@@ -10273,7 +10728,7 @@ def write_result_names_file() -> None:
         except Exception as e:
             print_red(f"Error trying to open file '{fn}': {e}")
 
-def run_program_once(params=None) -> None:
+def run_program_once(params: Optional[dict] = None) -> None:
     if not args.run_program_once:
         print_debug("[yellow]No setup script specified (run_program_once). Skipping setup.[/yellow]")
         return
@@ -10282,27 +10737,27 @@ def run_program_once(params=None) -> None:
         params = {}
 
     if isinstance(args.run_program_once, str):
-        command_str = args.run_program_once
+        command_str = decode_if_base64(args.run_program_once)
         for k, v in params.items():
             placeholder = f"%({k})"
             command_str = command_str.replace(placeholder, str(v))
 
-        with spinner(f"Executing command: [cyan]{command_str}[/cyan]"):
-            result = subprocess.run(command_str, shell=True, check=True)
-            if result.returncode == 0:
-                console.log("[bold green]Setup script completed successfully ✅[/bold green]")
-            else:
-                console.log(f"[bold red]Setup script failed with exit code {result.returncode} ❌[/bold red]")
+        print(f"Executing command: [cyan]{command_str}[/cyan]")
+        result = subprocess.run(command_str, shell=True, check=True)
+        if result.returncode == 0:
+            print("[bold green]Setup script completed successfully ✅[/bold green]")
+        else:
+            print(f"[bold red]Setup script failed with exit code {result.returncode} ❌[/bold red]")
 
-                my_exit(57)
+            my_exit(57)
 
     elif isinstance(args.run_program_once, (list, tuple)):
         with spinner("run_program_once: Executing command list: [cyan]{args.run_program_once}[/cyan]"):
             result = subprocess.run(args.run_program_once, check=True)
             if result.returncode == 0:
-                console.log("[bold green]Setup script completed successfully ✅[/bold green]")
+                print("[bold green]Setup script completed successfully ✅[/bold green]")
             else:
-                console.log(f"[bold red]Setup script failed with exit code {result.returncode} ❌[/bold red]")
+                print(f"[bold red]Setup script failed with exit code {result.returncode} ❌[/bold red]")
 
                 my_exit(57)
 
@@ -10312,7 +10767,7 @@ def run_program_once(params=None) -> None:
         my_exit(57)
 
 def show_omniopt_call() -> None:
-    def remove_ui_url(arg_str) -> str:
+    def remove_ui_url(arg_str: str) -> str:
         return re.sub(r'(?:--ui_url(?:=\S+)?(?:\s+\S+)?)', '', arg_str).strip()
 
     original_argv = " ".join(sys.argv[1:])
@@ -10320,8 +10775,14 @@ def show_omniopt_call() -> None:
 
     original_print(oo_call + " " + cleaned)
 
+    if args.dependency is not None and args.dependency != "":
+        print(f"Dependency: {args.dependency}")
+
+    if args.ui_url is not None and args.ui_url != "":
+        print_yellow("--ui_url is deprecated. Do not use it anymore. It will be ignored and one day be removed.")
+
 def main() -> None:
-    global RESULT_CSV_FILE, ax_client, LOGFILE_DEBUG_GET_NEXT_TRIALS
+    global RESULT_CSV_FILE, LOGFILE_DEBUG_GET_NEXT_TRIALS
 
     check_if_has_random_steps()
 
@@ -10372,7 +10833,7 @@ def main() -> None:
     print_run_info()
 
     initialize_nvidia_logs()
-    write_ui_url_if_present()
+    write_ui_url()
 
     LOGFILE_DEBUG_GET_NEXT_TRIALS = get_current_run_folder('get_next_trials.csv')
     cli_params_experiment_parameters = parse_parameters()
@@ -10403,15 +10864,13 @@ def main() -> None:
     exp_params = get_experiment_parameters(cli_params_experiment_parameters)
 
     if exp_params is not None:
-        ax_client, experiment_args, gpu_string, gpu_color = exp_params
+        experiment_args, gpu_string, gpu_color = exp_params
         print_debug(f"experiment_parameters: {experiment_parameters}")
 
         set_orchestrator()
 
         init_live_share()
 
-        start_periodic_live_share()
-
         show_available_hardware_and_generation_strategy_string(gpu_string, gpu_color)
 
         original_print(f"Run-Program: {global_vars['joined_run_program']}")
@@ -10426,6 +10885,8 @@ def main() -> None:
 
         write_files_and_show_overviews()
 
+        live_share()
+
         #if args.continue_previous_job:
         #    insert_jobs_from_csv(f"{args.continue_previous_job}/{RESULTS_CSV_FILENAME}")
 
@@ -10434,7 +10895,7 @@ def main() -> None:
 
             set_global_generation_strategy()
 
-        start_worker_generators()
+        #start_worker_generators()
 
         try:
             run_search_with_progress_bar()
@@ -10500,10 +10961,93 @@ def initialize_nvidia_logs() -> None:
     global NVIDIA_SMI_LOGS_BASE
     NVIDIA_SMI_LOGS_BASE = get_current_run_folder('gpu_usage_')
 
-def write_ui_url_if_present() -> None:
-    if args.ui_url:
-        with open(get_current_run_folder("ui_url.txt"), mode="a", encoding="utf-8") as myfile:
-            myfile.write(decode_if_base64(args.ui_url))
+def build_gui_url(config: argparse.Namespace) -> str:
+    base_url = get_base_url()
+    params = collect_params(config)
+    ret = f"{base_url}?{urlencode(params, doseq=True)}"
+
+    return ret
+
+def get_result_names_for_url(value: List) -> str:
+    d = dict(v.split("=", 1) if "=" in v else (v, "min") for v in value)
+    s = " ".join(f"{k}={v}" for k, v in d.items())
+
+    return s
+
+def collect_params(config: argparse.Namespace) -> dict:
+    params = {}
+    user_home = os.path.expanduser("~")
+
+    for attr, value in vars(config).items():
+        if attr == "run_program":
+            params[attr] = global_vars["joined_run_program"]
+        elif attr == "result_names" and value:
+            params[attr] = get_result_names_for_url(value)
+        elif attr == "parameter" and value is not None:
+            params.update(process_parameters(config.parameter))
+        elif attr == "root_venv_dir":
+            if value is not None and os.path.abspath(value) != os.path.abspath(user_home):
+                params[attr] = value
+        elif isinstance(value, bool):
+            params[attr] = int(value)
+        elif isinstance(value, list):
+            params[attr] = value
+        elif value is not None:
+            params[attr] = value
+
+    return params
+
+def process_parameters(parameters: list) -> dict:
+    params = {}
+    for i, param in enumerate(parameters):
+        if isinstance(param, dict):
+            name = param.get("name", f"param_{i}")
+            ptype = param.get("type", "unknown")
+        else:
+            name = param[0] if len(param) > 0 else f"param_{i}"
+            ptype = param[1] if len(param) > 1 else "unknown"
+
+        params[f"parameter_{i}_name"] = name
+        params[f"parameter_{i}_type"] = ptype
+
+        if ptype == "range":
+            params.update(process_range_parameter(i, param))
+        elif ptype == "choice":
+            params.update(process_choice_parameter(i, param))
+        elif ptype == "fixed":
+            params.update(process_fixed_parameter(i, param))
+
+    params["num_parameters"] = len(parameters)
+    return params
+
+def process_range_parameter(i: int, param: list) -> dict:
+    return {
+        f"parameter_{i}_min": param[2] if len(param) > 3 else 0,
+        f"parameter_{i}_max": param[3] if len(param) > 3 else 1,
+        f"parameter_{i}_number_type": param[4] if len(param) > 4 else "float",
+        f"parameter_{i}_log_scale": "false",
+    }
+
+def process_choice_parameter(i: int, param: list) -> dict:
+    choices = ""
+    if len(param) > 2 and param[2]:
+        choices = ",".join([c.strip() for c in str(param[2]).split(",")])
+    return {f"parameter_{i}_values": choices}
+
+def process_fixed_parameter(i: int, param: list) -> dict:
+    return {f"parameter_{i}_value": param[2] if len(param) > 2 else ""}
+
+def get_base_url() -> str:
+    file_path = Path.home() / ".oo_base_url"
+    if file_path.exists():
+        return file_path.read_text().strip()
+
+    return "https://imageseg.scads.de/omniax/"
+
+def write_ui_url() -> None:
+    url = build_gui_url(args)
+    with open(get_current_run_folder("ui_url.txt"), mode="a", encoding="utf-8") as myfile:
+        myfile.write(url)
 
 def handle_random_steps() -> None:
     if args.parameter and args.continue_previous_job and random_steps <= 0:
@@ -10562,8 +11106,6 @@ def run_search_with_progress_bar() -> None:
     wait_for_jobs_to_complete()
 
 def complex_tests(_program_name: str, wanted_stderr: str, wanted_exit_code: int, wanted_signal: Union[int, None], res_is_none: bool = False) -> int:
-    #print_yellow(f"Test suite: {_program_name}")
-
     nr_errors: int = 0
 
     program_path: str = f"./.tests/test_wronggoing_stuff.bin/bin/{_program_name}"
@@ -10637,7 +11179,7 @@ def test_find_paths(program_code: str) -> int:
     for i in files:
         if i not in string:
             if os.path.exists(i):
-                print("Missing {i} in find_file_paths string!")
+                print(f"Missing {i} in find_file_paths string!")
                 nr_errors += 1
 
     return nr_errors
@@ -11018,17 +11560,16 @@ Exit-Code: 159
 
     my_exit(nr_errors)
 
-def main_outside() -> None:
+def main_wrapper() -> None:
     print(f"Run-UUID: {run_uuid}")
 
     auto_wrap_namespace(globals())
 
     print_logo()
 
-    start_logging_daemon()
-
     fool_linter(args.num_cpus_main_job)
     fool_linter(args.flame_graph)
+    fool_linter(args.memray)
 
     with warnings.catch_warnings():
         warnings.simplefilter("ignore")
@@ -11060,9 +11601,34 @@ def main_outside() -> None:
             else:
                 end_program(True)
 
+def stack_trace_wrapper(func: Any, regex: Any = None) -> Any:
+    pattern = re.compile(regex) if regex else None
+
+    def wrapped(*args: Any, **kwargs: Any) -> None:
+        if pattern and not pattern.search(func.__name__):
+            return func(*args, **kwargs)
+
+        stack = inspect.stack()
+        chain = []
+        for frame in stack[1:]:
+            fn = frame.function
+            if fn in ("wrapped", "<module>"):
+                continue
+            chain.append(fn)
+
+        if chain:
+            sys.stderr.write(" ⇒ ".join(reversed(chain)) + "\n")
+
+        return func(*args, **kwargs)
+
+    return wrapped
+
 def auto_wrap_namespace(namespace: Any) -> Any:
     enable_beartype = any(os.getenv(v) for v in ("ENABLE_BEARTYPE", "CI"))
 
+    if args.beartype:
+        enable_beartype = True
+
     excluded_functions = {
         "log_time_and_memory_wrapper",
         "collect_runtime_stats",
@@ -11071,8 +11637,6 @@ def auto_wrap_namespace(namespace: Any) -> Any:
         "_record_stats",
         "_open",
         "_check_memory_leak",
-        "start_periodic_live_share",
-        "start_logging_daemon",
         "get_current_run_folder",
         "show_func_name_wrapper"
     }
@@ -11089,14 +11653,16 @@ def auto_wrap_namespace(namespace: Any) -> Any:
             if args.show_func_name:
                 wrapped = show_func_name_wrapper(wrapped)
 
+            if args.debug_stack_trace_regex:
+                wrapped = stack_trace_wrapper(wrapped, args.debug_stack_trace_regex)
+
             namespace[name] = wrapped
 
     return namespace
 
 if __name__ == "__main__":
-
     try:
-        main_outside()
+        main_wrapper()
     except (SignalUSR, SignalINT, SignalCONT) as e:
-        print_red(f"main_outside failed with exception {e}")
+        print_red(f"main_wrapper failed with exception {e}")
         end_program(True)