omniopt2 8701__tar.gz → 8712__tar.gz

{omniopt2-8701 → omniopt2-8712}/.omniopt.py

@@ -32,6 +32,8 @@ import psutil
 FORCE_EXIT: bool = False
 
 last_msg_progressbar = ""
+last_msg_raw = None
+last_lock = threading.Lock()
 
 def force_exit(signal_number: Any, frame: Any) -> Any:
     global FORCE_EXIT
@@ -545,6 +547,18 @@ logfile_worker_creation_logs: str = f'{log_uuid_dir}_worker_creation_logs'
 logfile_trial_index_to_param_logs: str = f'{log_uuid_dir}_trial_index_to_param_logs'
 LOGFILE_DEBUG_GET_NEXT_TRIALS: Union[str, None] = None
 
+def error_without_print(text: str) -> None:
+    print_debug(text)
+
+    if get_current_run_folder():
+        try:
+            with open(get_current_run_folder("oo_errors.txt"), mode="a", encoding="utf-8") as myfile:
+                myfile.write(text + "\n\n")
+        except (OSError, FileNotFoundError) as e:
+            helpers.print_color("red", f"Error: {e}. This may mean that the {get_current_run_folder()} was deleted during the run. Could not write '{text} to {get_current_run_folder()}/oo_errors.txt'")
+            sys.exit(99)
+
+
 def print_red(text: str) -> None:
     helpers.print_color("red", text)
 
@@ -2827,15 +2841,20 @@ def print_debug_get_next_trials(got: int, requested: int, _line: int) -> None:
     log_message_to_file(LOGFILE_DEBUG_GET_NEXT_TRIALS, msg, 0, "")
 
 def print_debug_progressbar(msg: str) -> None:
-    global last_msg_progressbar
+    global last_msg_progressbar, last_msg_raw
 
-    if msg != last_msg_progressbar:
-        time_str = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        msg = f"{time_str} ({worker_generator_uuid}): {msg}"
+    try:
+        with last_lock:
+            if msg != last_msg_raw:
+                time_str = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+                full_msg = f"{time_str} ({worker_generator_uuid}): {msg}"
 
-        _debug_progressbar(msg)
+                _debug_progressbar(full_msg)
 
-        last_msg_progressbar = msg
+                last_msg_raw = msg
+                last_msg_progressbar = full_msg
+    except Exception as e:
+        print(f"Error in print_debug_progressbar: {e}", flush=True)
 
 def get_process_info(pid: Any) -> str:
     try:
@@ -5009,7 +5028,8 @@ def show_pareto_or_error_msg(path_to_calculate: str, res_names: list = arg_resul
         try:
             show_pareto_frontier_data(path_to_calculate, res_names, disable_sixel_and_table)
         except Exception as e:
-            print_red(f"show_pareto_frontier_data() failed with exception '{e}'")
+            inner_tb = ''.join(traceback.format_exception(type(e), e, e.__traceback__))
+            print_red(f"show_pareto_frontier_data() failed with exception '{e}':\n{inner_tb}")
     else:
         print_debug(f"show_pareto_frontier_data will NOT be executed because len(arg_result_names) is {len(arg_result_names)}")
     return None
@@ -7496,7 +7516,7 @@ def finish_previous_jobs(new_msgs: List[str] = []) -> None:
 
     jobs_copy = global_vars["jobs"][:]
 
-    finishing_jobs_start_time = time.time()
+    #finishing_jobs_start_time = time.time()
 
     with ThreadPoolExecutor() as finish_job_executor:
         futures = [finish_job_executor.submit(_finish_previous_jobs_helper_check_and_process, (job, trial_index)) for job, trial_index in jobs_copy]
@@ -7507,11 +7527,11 @@ def finish_previous_jobs(new_msgs: List[str] = []) -> None:
             except Exception as e:
                 print_red(f"⚠ Exception in parallel job handling: {e}")
 
-    finishing_jobs_end_time = time.time()
+    #finishing_jobs_end_time = time.time()
 
-    finishing_jobs_runtime = finishing_jobs_end_time - finishing_jobs_start_time
+    #finishing_jobs_runtime = finishing_jobs_end_time - finishing_jobs_start_time
 
-    print_debug(f"Finishing jobs took {finishing_jobs_runtime} second(s)")
+    #print_debug(f"Finishing jobs took {finishing_jobs_runtime} second(s)")
 
     if this_jobs_finished > 0:
         save_results_csv()
@@ -7688,7 +7708,7 @@ def get_ax_client_trial(trial_index: int) -> Optional[ax.core.trial.Trial]:
     try:
         return ax_client.get_trial(trial_index)
     except KeyError as e:
-        print_red(f"get_ax_client_trial: trial_index {trial_index} failed, error: {e}")
+        error_without_print(f"get_ax_client_trial: trial_index {trial_index} failed, error: {e}")
         return None
 
 def orchestrator_start_trial(parameters: Union[dict, str], trial_index: int) -> None:
@@ -7833,7 +7853,7 @@ def execute_evaluation(_params: list) -> Optional[int]:
     _trial = get_ax_client_trial(trial_index)
 
     if _trial is None:
-        print_red(f"execute_evaluation: _trial was not in execute_evaluation for params {_params}")
+        error_without_print(f"execute_evaluation: _trial was not in execute_evaluation for params {_params}")
         return None
 
     def mark_trial_stage(stage: str, error_msg: str) -> None:
@@ -8123,13 +8143,13 @@ def get_batched_arms(nr_of_jobs_to_get: int) -> list:
 
         print_debug(f"get_batched_arms: Attempt {attempts + 1}: requesting 1 more arm")
 
-        t0 = time.time()
+        #t0 = time.time()
         pending_observations = get_pending_observation_features(experiment=ax_client.experiment)
-        dt = time.time() - t0
-        print_debug(f"got pending observations: {pending_observations} (took {dt:.2f} seconds)")
+        #dt = time.time() - t0
+        #print_debug(f"got pending observations: {pending_observations} (took {dt:.2f} seconds)")
 
         try:
-            print_debug("getting global_gs.gen() with n=1")
+            #print_debug("getting global_gs.gen() with n=1")
             batched_generator_run: Any = global_gs.gen(
                 experiment=ax_client.experiment,
                 n=1,
@@ -8144,12 +8164,12 @@ def get_batched_arms(nr_of_jobs_to_get: int) -> list:
         depth = 0
         path = "batched_generator_run"
         while isinstance(batched_generator_run, (list, tuple)) and len(batched_generator_run) == 1:
-            print_debug(f"Depth {depth}, path {path}, type {type(batched_generator_run).__name__}, length {len(batched_generator_run)}: {batched_generator_run}")
+            #print_debug(f"Depth {depth}, path {path}, type {type(batched_generator_run).__name__}, length {len(batched_generator_run)}: {batched_generator_run}")
             batched_generator_run = batched_generator_run[0]
             path += "[0]"
             depth += 1
 
-        print_debug(f"Final flat object at depth {depth}, path {path}: {batched_generator_run} (type {type(batched_generator_run).__name__})")
+        #print_debug(f"Final flat object at depth {depth}, path {path}: {batched_generator_run} (type {type(batched_generator_run).__name__})")
 
         new_arms = getattr(batched_generator_run, "arms", [])
         if not new_arms:
@@ -9428,7 +9448,7 @@ def finalize_jobs() -> None:
         handle_slurm_execution()
 
 def go_through_jobs_that_are_not_completed_yet() -> None:
-    print_debug(f"Waiting for jobs to finish (currently, len(global_vars['jobs']) = {len(global_vars['jobs'])}")
+    #print_debug(f"Waiting for jobs to finish (currently, len(global_vars['jobs']) = {len(global_vars['jobs'])}")
 
     nr_jobs_left = len(global_vars['jobs'])
     if nr_jobs_left == 1:

{omniopt2-8701 → omniopt2-8712}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: omniopt2
-Version: 8701
+Version: 8712
 Summary: Automatic highly parallelized hyperparameter optimizer based on Ax/Botorch
 Home-page: https://scads.ai/transfer-2/verfuegbare-software-dienste-en/omniopt/
 Author: Norman Koch

{omniopt2-8701 → omniopt2-8712}/omniopt2.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: omniopt2
-Version: 8701
+Version: 8712
 Summary: Automatic highly parallelized hyperparameter optimizer based on Ax/Botorch
 Home-page: https://scads.ai/transfer-2/verfuegbare-software-dienste-en/omniopt/
 Author: Norman Koch

{omniopt2-8701 → omniopt2-8712}/pyproject.toml

@@ -5,7 +5,7 @@ authors = [
   {email = "norman.koch@tu-dresden.de"},
   {name = "Norman Koch"}
 ]
-version = "8701"
+version = "8712"
 
 readme = "README.md"
 dynamic = ["dependencies"]