omniopt2 7137__py3-none-any.whl → 7140__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- .omniopt.py +183 -153
- {omniopt2-7137.data → omniopt2-7140.data}/data/bin/.omniopt.py +183 -153
- {omniopt2-7137.dist-info → omniopt2-7140.dist-info}/METADATA +1 -1
- {omniopt2-7137.dist-info → omniopt2-7140.dist-info}/RECORD +35 -35
- omniopt2.egg-info/PKG-INFO +1 -1
- pyproject.toml +1 -1
- {omniopt2-7137.data → omniopt2-7140.data}/data/bin/.colorfunctions.sh +0 -0
- {omniopt2-7137.data → omniopt2-7140.data}/data/bin/.general.sh +0 -0
- {omniopt2-7137.data → omniopt2-7140.data}/data/bin/.helpers.py +0 -0
- {omniopt2-7137.data → omniopt2-7140.data}/data/bin/.omniopt_plot_cpu_ram_usage.py +0 -0
- {omniopt2-7137.data → omniopt2-7140.data}/data/bin/.omniopt_plot_general.py +0 -0
- {omniopt2-7137.data → omniopt2-7140.data}/data/bin/.omniopt_plot_gpu_usage.py +0 -0
- {omniopt2-7137.data → omniopt2-7140.data}/data/bin/.omniopt_plot_kde.py +0 -0
- {omniopt2-7137.data → omniopt2-7140.data}/data/bin/.omniopt_plot_scatter.py +0 -0
- {omniopt2-7137.data → omniopt2-7140.data}/data/bin/.omniopt_plot_scatter_generation_method.py +0 -0
- {omniopt2-7137.data → omniopt2-7140.data}/data/bin/.omniopt_plot_scatter_hex.py +0 -0
- {omniopt2-7137.data → omniopt2-7140.data}/data/bin/.omniopt_plot_time_and_exit_code.py +0 -0
- {omniopt2-7137.data → omniopt2-7140.data}/data/bin/.omniopt_plot_trial_index_result.py +0 -0
- {omniopt2-7137.data → omniopt2-7140.data}/data/bin/.omniopt_plot_worker.py +0 -0
- {omniopt2-7137.data → omniopt2-7140.data}/data/bin/.random_generator.py +0 -0
- {omniopt2-7137.data → omniopt2-7140.data}/data/bin/.shellscript_functions +0 -0
- {omniopt2-7137.data → omniopt2-7140.data}/data/bin/.tpe.py +0 -0
- {omniopt2-7137.data → omniopt2-7140.data}/data/bin/LICENSE +0 -0
- {omniopt2-7137.data → omniopt2-7140.data}/data/bin/apt-dependencies.txt +0 -0
- {omniopt2-7137.data → omniopt2-7140.data}/data/bin/omniopt +0 -0
- {omniopt2-7137.data → omniopt2-7140.data}/data/bin/omniopt_docker +0 -0
- {omniopt2-7137.data → omniopt2-7140.data}/data/bin/omniopt_evaluate +0 -0
- {omniopt2-7137.data → omniopt2-7140.data}/data/bin/omniopt_plot +0 -0
- {omniopt2-7137.data → omniopt2-7140.data}/data/bin/omniopt_share +0 -0
- {omniopt2-7137.data → omniopt2-7140.data}/data/bin/requirements.txt +0 -0
- {omniopt2-7137.data → omniopt2-7140.data}/data/bin/setup.py +0 -0
- {omniopt2-7137.data → omniopt2-7140.data}/data/bin/test_requirements.txt +0 -0
- {omniopt2-7137.dist-info → omniopt2-7140.dist-info}/WHEEL +0 -0
- {omniopt2-7137.dist-info → omniopt2-7140.dist-info}/licenses/LICENSE +0 -0
- {omniopt2-7137.dist-info → omniopt2-7140.dist-info}/top_level.txt +0 -0
.omniopt.py
CHANGED
@@ -2301,13 +2301,14 @@ def check_slurm_job_id() -> None:
|
|
2301
2301
|
|
2302
2302
|
@beartype
|
2303
2303
|
def create_folder_and_file(folder: str) -> str:
|
2304
|
-
|
2304
|
+
with console.status(f"[bold green]Creating folder {folder}..."):
|
2305
|
+
print_debug(f"create_folder_and_file({folder})")
|
2305
2306
|
|
2306
|
-
|
2307
|
+
makedirs(folder)
|
2307
2308
|
|
2308
|
-
|
2309
|
+
file_path = os.path.join(folder, "results.csv")
|
2309
2310
|
|
2310
|
-
|
2311
|
+
return file_path
|
2311
2312
|
|
2312
2313
|
@beartype
|
2313
2314
|
def get_program_code_from_out_file(f: str) -> str:
|
@@ -7738,87 +7739,88 @@ def create_systematic_step(model: Any, _num_trials: int = -1, index: Optional[in
|
|
7738
7739
|
|
7739
7740
|
@beartype
|
7740
7741
|
def set_global_generation_strategy() -> None:
|
7741
|
-
|
7742
|
+
with console.status("[bold green]Setting generation strategy..."):
|
7743
|
+
global global_gs, generation_strategy_human_readable
|
7742
7744
|
|
7743
|
-
|
7745
|
+
args_generation_strategy = args.generation_strategy
|
7744
7746
|
|
7745
|
-
|
7747
|
+
continue_not_supported_on_custom_generation_strategy()
|
7746
7748
|
|
7747
|
-
|
7748
|
-
|
7749
|
+
gs_names: list = []
|
7750
|
+
gs_nodes: list = []
|
7749
7751
|
|
7750
|
-
|
7751
|
-
|
7752
|
-
|
7753
|
-
|
7752
|
+
if args_generation_strategy is None:
|
7753
|
+
num_imported_jobs: int = get_nr_of_imported_jobs()
|
7754
|
+
set_max_eval(max_eval + num_imported_jobs)
|
7755
|
+
set_random_steps(random_steps or 0)
|
7754
7756
|
|
7755
|
-
|
7756
|
-
|
7757
|
+
if max_eval is None:
|
7758
|
+
set_max_eval(max(1, random_steps))
|
7757
7759
|
|
7758
|
-
|
7760
|
+
chosen_model = get_chosen_model()
|
7759
7761
|
|
7760
|
-
|
7761
|
-
|
7762
|
+
if chosen_model == "SOBOL":
|
7763
|
+
set_random_steps(max_eval)
|
7762
7764
|
|
7763
|
-
|
7764
|
-
|
7765
|
-
|
7766
|
-
|
7765
|
+
if random_steps >= 1:
|
7766
|
+
next_node_name = None
|
7767
|
+
if max_eval - random_steps and chosen_model:
|
7768
|
+
next_node_name = chosen_model
|
7767
7769
|
|
7768
|
-
|
7769
|
-
|
7770
|
+
gs_names.append(get_step_name("SOBOL", random_steps))
|
7771
|
+
gs_nodes.append(create_node("SOBOL", random_steps, next_node_name))
|
7770
7772
|
|
7771
|
-
|
7773
|
+
write_state_file("model", str(chosen_model))
|
7772
7774
|
|
7773
|
-
|
7774
|
-
|
7775
|
+
if chosen_model != "SOBOL" and max_eval > random_steps:
|
7776
|
+
this_node = create_node(chosen_model, max_eval - random_steps, None)
|
7775
7777
|
|
7776
|
-
|
7777
|
-
|
7778
|
+
gs_names.append(get_step_name(chosen_model, max_eval - random_steps))
|
7779
|
+
gs_nodes.append(this_node)
|
7778
7780
|
|
7779
|
-
|
7781
|
+
generation_strategy_human_readable = join_with_comma_and_then(gs_names)
|
7780
7782
|
|
7781
|
-
|
7782
|
-
|
7783
|
-
|
7784
|
-
|
7785
|
-
|
7786
|
-
|
7787
|
-
|
7783
|
+
try:
|
7784
|
+
global_gs = GenerationStrategy(
|
7785
|
+
name="+".join(gs_names),
|
7786
|
+
nodes=gs_nodes
|
7787
|
+
)
|
7788
|
+
except ax.exceptions.generation_strategy.GenerationStrategyMisconfiguredException as e:
|
7789
|
+
print_red(f"Error: {e}\ngs_names: {gs_names}\ngs_nodes: {gs_nodes}")
|
7788
7790
|
|
7789
|
-
|
7790
|
-
|
7791
|
-
|
7791
|
+
my_exit(55)
|
7792
|
+
else:
|
7793
|
+
generation_strategy_array, new_max_eval = parse_generation_strategy_string(args_generation_strategy)
|
7792
7794
|
|
7793
|
-
|
7795
|
+
new_max_eval_plus_inserted_jobs = new_max_eval + get_nr_of_imported_jobs()
|
7794
7796
|
|
7795
|
-
|
7796
|
-
|
7797
|
-
|
7797
|
+
if max_eval < new_max_eval_plus_inserted_jobs:
|
7798
|
+
print_yellow(f"--generation_strategy {args_generation_strategy.upper()} has, in sum, more tasks than --max_eval {max_eval}. max_eval will be set to {new_max_eval_plus_inserted_jobs}.")
|
7799
|
+
set_max_eval(new_max_eval_plus_inserted_jobs)
|
7798
7800
|
|
7799
|
-
|
7801
|
+
print_generation_strategy(generation_strategy_array)
|
7800
7802
|
|
7801
|
-
|
7803
|
+
start_index = int(len(generation_strategy_array) / 2)
|
7802
7804
|
|
7803
|
-
|
7805
|
+
steps: list = []
|
7804
7806
|
|
7805
|
-
|
7806
|
-
|
7807
|
+
for gs_element in generation_strategy_array:
|
7808
|
+
model_name = list(gs_element.keys())[0]
|
7807
7809
|
|
7808
|
-
|
7810
|
+
nr = int(gs_element[model_name])
|
7809
7811
|
|
7810
|
-
|
7811
|
-
|
7812
|
+
gs_elem = create_systematic_step(select_model(model_name), nr, start_index)
|
7813
|
+
steps.append(gs_elem)
|
7812
7814
|
|
7813
|
-
|
7815
|
+
gs_names.append(get_step_name(model_name, nr))
|
7814
7816
|
|
7815
|
-
|
7817
|
+
start_index = start_index + 1
|
7816
7818
|
|
7817
|
-
|
7819
|
+
write_state_file("custom_generation_strategy", args_generation_strategy)
|
7818
7820
|
|
7819
|
-
|
7821
|
+
global_gs = GenerationStrategy(steps=steps)
|
7820
7822
|
|
7821
|
-
|
7823
|
+
generation_strategy_human_readable = join_with_comma_and_then(gs_names)
|
7822
7824
|
|
7823
7825
|
@beartype
|
7824
7826
|
def wait_for_jobs_or_break(_max_eval: Optional[int], _progress_bar: Any) -> bool:
|
@@ -8000,35 +8002,36 @@ def _create_and_execute_next_runs_return_value(trial_index_to_param: Optional[Di
|
|
8000
8002
|
|
8001
8003
|
@beartype
|
8002
8004
|
def get_number_of_steps(_max_eval: int) -> Tuple[int, int]:
|
8003
|
-
|
8005
|
+
with console.status("[bold green]Calculating number of steps..."):
|
8006
|
+
_random_steps = args.num_random_steps
|
8004
8007
|
|
8005
|
-
|
8008
|
+
already_done_random_steps = get_random_steps_from_prev_job()
|
8006
8009
|
|
8007
|
-
|
8010
|
+
_random_steps = _random_steps - already_done_random_steps
|
8008
8011
|
|
8009
|
-
|
8010
|
-
|
8011
|
-
|
8012
|
+
if _random_steps > _max_eval:
|
8013
|
+
print_yellow(f"You have less --max_eval {_max_eval} than --num_random_steps {_random_steps}. Switched both.")
|
8014
|
+
_random_steps, _max_eval = _max_eval, _random_steps
|
8012
8015
|
|
8013
|
-
|
8014
|
-
|
8016
|
+
if _random_steps < num_parallel_jobs and SYSTEM_HAS_SBATCH:
|
8017
|
+
print_yellow(f"Warning: --num_random_steps {_random_steps} is smaller than --num_parallel_jobs {num_parallel_jobs}. It's recommended that --num_parallel_jobs is the same as or a multiple of --num_random_steps")
|
8015
8018
|
|
8016
|
-
|
8017
|
-
|
8019
|
+
if _random_steps > _max_eval:
|
8020
|
+
set_max_eval(_random_steps)
|
8018
8021
|
|
8019
|
-
|
8020
|
-
|
8021
|
-
|
8022
|
-
|
8023
|
-
|
8024
|
-
|
8022
|
+
original_second_steps = _max_eval - _random_steps
|
8023
|
+
second_step_steps = max(0, original_second_steps)
|
8024
|
+
if second_step_steps != original_second_steps:
|
8025
|
+
original_print(f"? original_second_steps: {original_second_steps} = max_eval {_max_eval} - _random_steps {_random_steps}")
|
8026
|
+
if second_step_steps == 0:
|
8027
|
+
print_yellow("This is basically a random search. Increase --max_eval or reduce --num_random_steps")
|
8025
8028
|
|
8026
|
-
|
8029
|
+
second_step_steps = second_step_steps - already_done_random_steps
|
8027
8030
|
|
8028
|
-
|
8029
|
-
|
8031
|
+
if args.continue_previous_job:
|
8032
|
+
second_step_steps = _max_eval
|
8030
8033
|
|
8031
|
-
|
8034
|
+
return _random_steps, second_step_steps
|
8032
8035
|
|
8033
8036
|
@beartype
|
8034
8037
|
def _set_global_executor() -> None:
|
@@ -8334,16 +8337,18 @@ def check_if_has_random_steps() -> None:
|
|
8334
8337
|
|
8335
8338
|
@beartype
|
8336
8339
|
def add_exclude_to_defective_nodes() -> None:
|
8337
|
-
|
8338
|
-
|
8340
|
+
with console.status("[bold green]Adding excluded nodes..."):
|
8341
|
+
if args.exclude:
|
8342
|
+
entries = [entry.strip() for entry in args.exclude.split(',')]
|
8339
8343
|
|
8340
|
-
|
8341
|
-
|
8344
|
+
for entry in entries:
|
8345
|
+
count_defective_nodes(None, entry)
|
8342
8346
|
|
8343
8347
|
@beartype
|
8344
8348
|
def check_max_eval(_max_eval: int) -> None:
|
8345
|
-
|
8346
|
-
|
8349
|
+
with console.status("[bold green]Checking max_eval..."):
|
8350
|
+
if not _max_eval:
|
8351
|
+
_fatal_error("--max_eval needs to be set!", 19)
|
8347
8352
|
|
8348
8353
|
@beartype
|
8349
8354
|
def parse_parameters() -> Any:
|
@@ -9009,58 +9014,72 @@ def write_files_and_show_overviews() -> None:
|
|
9009
9014
|
|
9010
9015
|
@beartype
|
9011
9016
|
def write_git_version() -> None:
|
9012
|
-
|
9013
|
-
|
9014
|
-
|
9017
|
+
with console.status("[bold green]Writing git info file..."):
|
9018
|
+
folder = f"{get_current_run_folder()}/"
|
9019
|
+
os.makedirs(folder, exist_ok=True)
|
9020
|
+
file_path = os.path.join(folder, "git_version")
|
9015
9021
|
|
9016
|
-
|
9017
|
-
|
9022
|
+
try:
|
9023
|
+
commit_hash = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True, stderr=subprocess.DEVNULL).strip()
|
9018
9024
|
|
9019
|
-
|
9025
|
+
git_tag = ""
|
9026
|
+
|
9027
|
+
try:
|
9028
|
+
git_tag = subprocess.check_output(["git", "describe", "--tags"], text=True, stderr=subprocess.DEVNULL).strip()
|
9029
|
+
git_tag = f" ({git_tag})"
|
9030
|
+
except subprocess.CalledProcessError:
|
9031
|
+
pass
|
9032
|
+
|
9033
|
+
if commit_hash:
|
9034
|
+
with open(file_path, mode="w", encoding="utf-8") as f:
|
9035
|
+
f.write(f"Commit: {commit_hash}{git_tag}\n")
|
9020
9036
|
|
9021
|
-
try:
|
9022
|
-
git_tag = subprocess.check_output(["git", "describe", "--tags"], text=True, stderr=subprocess.DEVNULL).strip()
|
9023
|
-
git_tag = f" ({git_tag})"
|
9024
9037
|
except subprocess.CalledProcessError:
|
9025
9038
|
pass
|
9026
9039
|
|
9027
|
-
|
9028
|
-
|
9029
|
-
|
9030
|
-
|
9031
|
-
|
9032
|
-
|
9040
|
+
@beartype
|
9041
|
+
def write_job_start_file() -> None:
|
9042
|
+
with console.status("[bold green]Writing job_start_time file..."):
|
9043
|
+
fn = f'{get_current_run_folder()}/job_start_time.txt'
|
9044
|
+
try:
|
9045
|
+
with open(fn, mode='w', encoding="utf-8") as f:
|
9046
|
+
f.write(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
|
9047
|
+
except Exception as e:
|
9048
|
+
print_red(f"Error trying to write {fn}: {e}")
|
9033
9049
|
|
9034
9050
|
@beartype
|
9035
9051
|
def write_live_share_file_if_needed() -> None:
|
9036
|
-
|
9037
|
-
|
9052
|
+
with console.status("[bold green]Writing live_share file if it is present..."):
|
9053
|
+
if args.live_share:
|
9054
|
+
write_state_file("live_share", "1\n")
|
9038
9055
|
|
9039
9056
|
@beartype
|
9040
9057
|
def write_username_statefile() -> None:
|
9041
|
-
|
9042
|
-
|
9043
|
-
|
9058
|
+
with console.status("[bold green]Writing username state file..."):
|
9059
|
+
_path = get_current_run_folder()
|
9060
|
+
if args.username:
|
9061
|
+
file_path = f"{_path}/state_files/username"
|
9062
|
+
|
9063
|
+
try:
|
9064
|
+
os.makedirs(os.path.dirname(file_path), exist_ok=True)
|
9065
|
+
with open(file_path, mode="w", encoding="utf-8") as f:
|
9066
|
+
f.write(args.username)
|
9067
|
+
except Exception as e:
|
9068
|
+
print_red(f"Error writing to file: {e}")
|
9069
|
+
|
9070
|
+
@beartype
|
9071
|
+
def write_revert_to_random_when_seemingly_exhausted_file() -> None:
|
9072
|
+
with console.status("[bold green]Writing revert_to_random_when_seemingly_exhausted file ..."):
|
9073
|
+
_path = get_current_run_folder()
|
9074
|
+
file_path = f"{_path}/state_files/revert_to_random_when_seemingly_exhausted"
|
9044
9075
|
|
9045
9076
|
try:
|
9046
9077
|
os.makedirs(os.path.dirname(file_path), exist_ok=True)
|
9047
9078
|
with open(file_path, mode="w", encoding="utf-8") as f:
|
9048
|
-
f.write(
|
9079
|
+
f.write("1\n")
|
9049
9080
|
except Exception as e:
|
9050
9081
|
print_red(f"Error writing to file: {e}")
|
9051
9082
|
|
9052
|
-
@beartype
|
9053
|
-
def write_revert_to_random_when_seemingly_exhausted_file() -> None:
|
9054
|
-
_path = get_current_run_folder()
|
9055
|
-
file_path = f"{_path}/state_files/revert_to_random_when_seemingly_exhausted"
|
9056
|
-
|
9057
|
-
try:
|
9058
|
-
os.makedirs(os.path.dirname(file_path), exist_ok=True)
|
9059
|
-
with open(file_path, mode="w", encoding="utf-8") as f:
|
9060
|
-
f.write("1\n")
|
9061
|
-
except Exception as e:
|
9062
|
-
print_red(f"Error writing to file: {e}")
|
9063
|
-
|
9064
9083
|
@beartype
|
9065
9084
|
def debug_vars_unused_by_python_for_linter() -> None:
|
9066
9085
|
print_debug(
|
@@ -9327,6 +9346,28 @@ def set_arg_states_from_continue() -> None:
|
|
9327
9346
|
if os.path.exists(f"{args.continue_previous_job}/state_files/revert_to_random_when_seemingly_exhausted"):
|
9328
9347
|
args.revert_to_random_when_seemingly_exhausted = True
|
9329
9348
|
|
9349
|
+
@beartype
|
9350
|
+
def write_result_min_max_file() -> None:
|
9351
|
+
with console.status("[bold green]Writing result min/max file..."):
|
9352
|
+
try:
|
9353
|
+
fn = f"{get_current_run_folder()}/result_min_max.txt"
|
9354
|
+
with open(fn, mode="a", encoding="utf-8") as myfile:
|
9355
|
+
for rarg in arg_result_min_or_max:
|
9356
|
+
original_print(rarg, file=myfile)
|
9357
|
+
except Exception as e:
|
9358
|
+
print_red(f"Error trying to open file '{fn}': {e}")
|
9359
|
+
|
9360
|
+
@beartype
|
9361
|
+
def write_result_names_file() -> None:
|
9362
|
+
with console.status("[bold green]Writing result names file..."):
|
9363
|
+
try:
|
9364
|
+
fn = f"{get_current_run_folder()}/result_names.txt"
|
9365
|
+
with open(fn, mode="a", encoding="utf-8") as myfile:
|
9366
|
+
for rarg in arg_result_names:
|
9367
|
+
original_print(rarg, file=myfile)
|
9368
|
+
except Exception as e:
|
9369
|
+
print_red(f"Error trying to open file '{fn}': {e}")
|
9370
|
+
|
9330
9371
|
@beartype
|
9331
9372
|
def main() -> None:
|
9332
9373
|
global RESULT_CSV_FILE, ax_client, LOGFILE_DEBUG_GET_NEXT_TRIALS
|
@@ -9354,21 +9395,9 @@ def main() -> None:
|
|
9354
9395
|
|
9355
9396
|
write_username_statefile()
|
9356
9397
|
|
9357
|
-
|
9358
|
-
fn = f"{get_current_run_folder()}/result_names.txt"
|
9359
|
-
with open(fn, mode="a", encoding="utf-8") as myfile:
|
9360
|
-
for rarg in arg_result_names:
|
9361
|
-
original_print(rarg, file=myfile)
|
9362
|
-
except Exception as e:
|
9363
|
-
print_red(f"Error trying to open file '{fn}': {e}")
|
9398
|
+
write_result_names_file()
|
9364
9399
|
|
9365
|
-
|
9366
|
-
fn = f"{get_current_run_folder()}/result_min_max.txt"
|
9367
|
-
with open(fn, mode="a", encoding="utf-8") as myfile:
|
9368
|
-
for rarg in arg_result_min_or_max:
|
9369
|
-
original_print(rarg, file=myfile)
|
9370
|
-
except Exception as e:
|
9371
|
-
print_red(f"Error trying to open file '{fn}': {e}")
|
9400
|
+
write_result_min_max_file()
|
9372
9401
|
|
9373
9402
|
if os.getenv("CI"):
|
9374
9403
|
data_dict: dict = {
|
@@ -9394,12 +9423,7 @@ def main() -> None:
|
|
9394
9423
|
|
9395
9424
|
write_live_share_file_if_needed()
|
9396
9425
|
|
9397
|
-
|
9398
|
-
try:
|
9399
|
-
with open(fn, mode='w', encoding="utf-8") as f:
|
9400
|
-
f.write(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
|
9401
|
-
except Exception as e:
|
9402
|
-
print_red(f"Error trying to write {fn}: {e}")
|
9426
|
+
write_job_start_file()
|
9403
9427
|
|
9404
9428
|
write_git_version()
|
9405
9429
|
|
@@ -9410,6 +9434,7 @@ def main() -> None:
|
|
9410
9434
|
set_random_steps(_random_steps)
|
9411
9435
|
|
9412
9436
|
add_exclude_to_defective_nodes()
|
9437
|
+
|
9413
9438
|
handle_random_steps()
|
9414
9439
|
|
9415
9440
|
set_global_generation_strategy()
|
@@ -9463,30 +9488,34 @@ def log_worker_creation() -> None:
|
|
9463
9488
|
|
9464
9489
|
@beartype
|
9465
9490
|
def set_run_folder() -> None:
|
9466
|
-
|
9467
|
-
|
9468
|
-
|
9469
|
-
|
9470
|
-
while os.path.exists(f"{CURRENT_RUN_FOLDER}"):
|
9471
|
-
RUN_FOLDER_NUMBER += 1
|
9491
|
+
with console.status("[bold green]Setting run folder..."):
|
9492
|
+
global CURRENT_RUN_FOLDER
|
9493
|
+
RUN_FOLDER_NUMBER: int = 0
|
9472
9494
|
CURRENT_RUN_FOLDER = f"{args.run_dir}/{global_vars['experiment_name']}/{RUN_FOLDER_NUMBER}"
|
9473
9495
|
|
9496
|
+
while os.path.exists(f"{CURRENT_RUN_FOLDER}"):
|
9497
|
+
RUN_FOLDER_NUMBER += 1
|
9498
|
+
CURRENT_RUN_FOLDER = f"{args.run_dir}/{global_vars['experiment_name']}/{RUN_FOLDER_NUMBER}"
|
9499
|
+
|
9474
9500
|
@beartype
|
9475
9501
|
def print_run_info() -> None:
|
9476
|
-
|
9477
|
-
|
9478
|
-
|
9502
|
+
with console.status("[bold green]Printing run info..."):
|
9503
|
+
original_print(f"Run-folder: {get_current_run_folder()}")
|
9504
|
+
if args.continue_previous_job:
|
9505
|
+
original_print(f"Continuation from {args.continue_previous_job}")
|
9479
9506
|
|
9480
9507
|
@beartype
|
9481
9508
|
def initialize_nvidia_logs() -> None:
|
9482
|
-
|
9483
|
-
|
9509
|
+
with console.status("[bold green]Initializing NVIDIA-Logs..."):
|
9510
|
+
global NVIDIA_SMI_LOGS_BASE
|
9511
|
+
NVIDIA_SMI_LOGS_BASE = f'{get_current_run_folder()}/gpu_usage_'
|
9484
9512
|
|
9485
9513
|
@beartype
|
9486
9514
|
def write_ui_url_if_present() -> None:
|
9487
|
-
|
9488
|
-
|
9489
|
-
|
9515
|
+
with console.status("[bold green]Writing ui_url file if it is present..."):
|
9516
|
+
if args.ui_url:
|
9517
|
+
with open(f"{get_current_run_folder()}/ui_url.txt", mode="a", encoding="utf-8") as myfile:
|
9518
|
+
myfile.write(decode_if_base64(args.ui_url))
|
9490
9519
|
|
9491
9520
|
@beartype
|
9492
9521
|
def set_random_steps(new_steps: int) -> None:
|
@@ -9498,9 +9527,10 @@ def set_random_steps(new_steps: int) -> None:
|
|
9498
9527
|
|
9499
9528
|
@beartype
|
9500
9529
|
def handle_random_steps() -> None:
|
9501
|
-
|
9502
|
-
|
9503
|
-
|
9530
|
+
with console.status("[bold green]Handling random steps..."):
|
9531
|
+
if args.parameter and args.continue_previous_job and random_steps <= 0:
|
9532
|
+
print(f"A parameter has been reset, but the earlier job already had its random phase. To look at the new search space, {args.num_random_steps} random steps will be executed.")
|
9533
|
+
set_random_steps(args.num_random_steps)
|
9504
9534
|
|
9505
9535
|
@beartype
|
9506
9536
|
def initialize_ax_client() -> None:
|
@@ -2301,13 +2301,14 @@ def check_slurm_job_id() -> None:
|
|
2301
2301
|
|
2302
2302
|
@beartype
|
2303
2303
|
def create_folder_and_file(folder: str) -> str:
|
2304
|
-
|
2304
|
+
with console.status(f"[bold green]Creating folder {folder}..."):
|
2305
|
+
print_debug(f"create_folder_and_file({folder})")
|
2305
2306
|
|
2306
|
-
|
2307
|
+
makedirs(folder)
|
2307
2308
|
|
2308
|
-
|
2309
|
+
file_path = os.path.join(folder, "results.csv")
|
2309
2310
|
|
2310
|
-
|
2311
|
+
return file_path
|
2311
2312
|
|
2312
2313
|
@beartype
|
2313
2314
|
def get_program_code_from_out_file(f: str) -> str:
|
@@ -7738,87 +7739,88 @@ def create_systematic_step(model: Any, _num_trials: int = -1, index: Optional[in
|
|
7738
7739
|
|
7739
7740
|
@beartype
|
7740
7741
|
def set_global_generation_strategy() -> None:
|
7741
|
-
|
7742
|
+
with console.status("[bold green]Setting generation strategy..."):
|
7743
|
+
global global_gs, generation_strategy_human_readable
|
7742
7744
|
|
7743
|
-
|
7745
|
+
args_generation_strategy = args.generation_strategy
|
7744
7746
|
|
7745
|
-
|
7747
|
+
continue_not_supported_on_custom_generation_strategy()
|
7746
7748
|
|
7747
|
-
|
7748
|
-
|
7749
|
+
gs_names: list = []
|
7750
|
+
gs_nodes: list = []
|
7749
7751
|
|
7750
|
-
|
7751
|
-
|
7752
|
-
|
7753
|
-
|
7752
|
+
if args_generation_strategy is None:
|
7753
|
+
num_imported_jobs: int = get_nr_of_imported_jobs()
|
7754
|
+
set_max_eval(max_eval + num_imported_jobs)
|
7755
|
+
set_random_steps(random_steps or 0)
|
7754
7756
|
|
7755
|
-
|
7756
|
-
|
7757
|
+
if max_eval is None:
|
7758
|
+
set_max_eval(max(1, random_steps))
|
7757
7759
|
|
7758
|
-
|
7760
|
+
chosen_model = get_chosen_model()
|
7759
7761
|
|
7760
|
-
|
7761
|
-
|
7762
|
+
if chosen_model == "SOBOL":
|
7763
|
+
set_random_steps(max_eval)
|
7762
7764
|
|
7763
|
-
|
7764
|
-
|
7765
|
-
|
7766
|
-
|
7765
|
+
if random_steps >= 1:
|
7766
|
+
next_node_name = None
|
7767
|
+
if max_eval - random_steps and chosen_model:
|
7768
|
+
next_node_name = chosen_model
|
7767
7769
|
|
7768
|
-
|
7769
|
-
|
7770
|
+
gs_names.append(get_step_name("SOBOL", random_steps))
|
7771
|
+
gs_nodes.append(create_node("SOBOL", random_steps, next_node_name))
|
7770
7772
|
|
7771
|
-
|
7773
|
+
write_state_file("model", str(chosen_model))
|
7772
7774
|
|
7773
|
-
|
7774
|
-
|
7775
|
+
if chosen_model != "SOBOL" and max_eval > random_steps:
|
7776
|
+
this_node = create_node(chosen_model, max_eval - random_steps, None)
|
7775
7777
|
|
7776
|
-
|
7777
|
-
|
7778
|
+
gs_names.append(get_step_name(chosen_model, max_eval - random_steps))
|
7779
|
+
gs_nodes.append(this_node)
|
7778
7780
|
|
7779
|
-
|
7781
|
+
generation_strategy_human_readable = join_with_comma_and_then(gs_names)
|
7780
7782
|
|
7781
|
-
|
7782
|
-
|
7783
|
-
|
7784
|
-
|
7785
|
-
|
7786
|
-
|
7787
|
-
|
7783
|
+
try:
|
7784
|
+
global_gs = GenerationStrategy(
|
7785
|
+
name="+".join(gs_names),
|
7786
|
+
nodes=gs_nodes
|
7787
|
+
)
|
7788
|
+
except ax.exceptions.generation_strategy.GenerationStrategyMisconfiguredException as e:
|
7789
|
+
print_red(f"Error: {e}\ngs_names: {gs_names}\ngs_nodes: {gs_nodes}")
|
7788
7790
|
|
7789
|
-
|
7790
|
-
|
7791
|
-
|
7791
|
+
my_exit(55)
|
7792
|
+
else:
|
7793
|
+
generation_strategy_array, new_max_eval = parse_generation_strategy_string(args_generation_strategy)
|
7792
7794
|
|
7793
|
-
|
7795
|
+
new_max_eval_plus_inserted_jobs = new_max_eval + get_nr_of_imported_jobs()
|
7794
7796
|
|
7795
|
-
|
7796
|
-
|
7797
|
-
|
7797
|
+
if max_eval < new_max_eval_plus_inserted_jobs:
|
7798
|
+
print_yellow(f"--generation_strategy {args_generation_strategy.upper()} has, in sum, more tasks than --max_eval {max_eval}. max_eval will be set to {new_max_eval_plus_inserted_jobs}.")
|
7799
|
+
set_max_eval(new_max_eval_plus_inserted_jobs)
|
7798
7800
|
|
7799
|
-
|
7801
|
+
print_generation_strategy(generation_strategy_array)
|
7800
7802
|
|
7801
|
-
|
7803
|
+
start_index = int(len(generation_strategy_array) / 2)
|
7802
7804
|
|
7803
|
-
|
7805
|
+
steps: list = []
|
7804
7806
|
|
7805
|
-
|
7806
|
-
|
7807
|
+
for gs_element in generation_strategy_array:
|
7808
|
+
model_name = list(gs_element.keys())[0]
|
7807
7809
|
|
7808
|
-
|
7810
|
+
nr = int(gs_element[model_name])
|
7809
7811
|
|
7810
|
-
|
7811
|
-
|
7812
|
+
gs_elem = create_systematic_step(select_model(model_name), nr, start_index)
|
7813
|
+
steps.append(gs_elem)
|
7812
7814
|
|
7813
|
-
|
7815
|
+
gs_names.append(get_step_name(model_name, nr))
|
7814
7816
|
|
7815
|
-
|
7817
|
+
start_index = start_index + 1
|
7816
7818
|
|
7817
|
-
|
7819
|
+
write_state_file("custom_generation_strategy", args_generation_strategy)
|
7818
7820
|
|
7819
|
-
|
7821
|
+
global_gs = GenerationStrategy(steps=steps)
|
7820
7822
|
|
7821
|
-
|
7823
|
+
generation_strategy_human_readable = join_with_comma_and_then(gs_names)
|
7822
7824
|
|
7823
7825
|
@beartype
|
7824
7826
|
def wait_for_jobs_or_break(_max_eval: Optional[int], _progress_bar: Any) -> bool:
|
@@ -8000,35 +8002,36 @@ def _create_and_execute_next_runs_return_value(trial_index_to_param: Optional[Di
|
|
8000
8002
|
|
8001
8003
|
@beartype
|
8002
8004
|
def get_number_of_steps(_max_eval: int) -> Tuple[int, int]:
|
8003
|
-
|
8005
|
+
with console.status("[bold green]Calculating number of steps..."):
|
8006
|
+
_random_steps = args.num_random_steps
|
8004
8007
|
|
8005
|
-
|
8008
|
+
already_done_random_steps = get_random_steps_from_prev_job()
|
8006
8009
|
|
8007
|
-
|
8010
|
+
_random_steps = _random_steps - already_done_random_steps
|
8008
8011
|
|
8009
|
-
|
8010
|
-
|
8011
|
-
|
8012
|
+
if _random_steps > _max_eval:
|
8013
|
+
print_yellow(f"You have less --max_eval {_max_eval} than --num_random_steps {_random_steps}. Switched both.")
|
8014
|
+
_random_steps, _max_eval = _max_eval, _random_steps
|
8012
8015
|
|
8013
|
-
|
8014
|
-
|
8016
|
+
if _random_steps < num_parallel_jobs and SYSTEM_HAS_SBATCH:
|
8017
|
+
print_yellow(f"Warning: --num_random_steps {_random_steps} is smaller than --num_parallel_jobs {num_parallel_jobs}. It's recommended that --num_parallel_jobs is the same as or a multiple of --num_random_steps")
|
8015
8018
|
|
8016
|
-
|
8017
|
-
|
8019
|
+
if _random_steps > _max_eval:
|
8020
|
+
set_max_eval(_random_steps)
|
8018
8021
|
|
8019
|
-
|
8020
|
-
|
8021
|
-
|
8022
|
-
|
8023
|
-
|
8024
|
-
|
8022
|
+
original_second_steps = _max_eval - _random_steps
|
8023
|
+
second_step_steps = max(0, original_second_steps)
|
8024
|
+
if second_step_steps != original_second_steps:
|
8025
|
+
original_print(f"? original_second_steps: {original_second_steps} = max_eval {_max_eval} - _random_steps {_random_steps}")
|
8026
|
+
if second_step_steps == 0:
|
8027
|
+
print_yellow("This is basically a random search. Increase --max_eval or reduce --num_random_steps")
|
8025
8028
|
|
8026
|
-
|
8029
|
+
second_step_steps = second_step_steps - already_done_random_steps
|
8027
8030
|
|
8028
|
-
|
8029
|
-
|
8031
|
+
if args.continue_previous_job:
|
8032
|
+
second_step_steps = _max_eval
|
8030
8033
|
|
8031
|
-
|
8034
|
+
return _random_steps, second_step_steps
|
8032
8035
|
|
8033
8036
|
@beartype
|
8034
8037
|
def _set_global_executor() -> None:
|
@@ -8334,16 +8337,18 @@ def check_if_has_random_steps() -> None:
|
|
8334
8337
|
|
8335
8338
|
@beartype
|
8336
8339
|
def add_exclude_to_defective_nodes() -> None:
|
8337
|
-
|
8338
|
-
|
8340
|
+
with console.status("[bold green]Adding excluded nodes..."):
|
8341
|
+
if args.exclude:
|
8342
|
+
entries = [entry.strip() for entry in args.exclude.split(',')]
|
8339
8343
|
|
8340
|
-
|
8341
|
-
|
8344
|
+
for entry in entries:
|
8345
|
+
count_defective_nodes(None, entry)
|
8342
8346
|
|
8343
8347
|
@beartype
|
8344
8348
|
def check_max_eval(_max_eval: int) -> None:
|
8345
|
-
|
8346
|
-
|
8349
|
+
with console.status("[bold green]Checking max_eval..."):
|
8350
|
+
if not _max_eval:
|
8351
|
+
_fatal_error("--max_eval needs to be set!", 19)
|
8347
8352
|
|
8348
8353
|
@beartype
|
8349
8354
|
def parse_parameters() -> Any:
|
@@ -9009,58 +9014,72 @@ def write_files_and_show_overviews() -> None:
|
|
9009
9014
|
|
9010
9015
|
@beartype
|
9011
9016
|
def write_git_version() -> None:
|
9012
|
-
|
9013
|
-
|
9014
|
-
|
9017
|
+
with console.status("[bold green]Writing git info file..."):
|
9018
|
+
folder = f"{get_current_run_folder()}/"
|
9019
|
+
os.makedirs(folder, exist_ok=True)
|
9020
|
+
file_path = os.path.join(folder, "git_version")
|
9015
9021
|
|
9016
|
-
|
9017
|
-
|
9022
|
+
try:
|
9023
|
+
commit_hash = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True, stderr=subprocess.DEVNULL).strip()
|
9018
9024
|
|
9019
|
-
|
9025
|
+
git_tag = ""
|
9026
|
+
|
9027
|
+
try:
|
9028
|
+
git_tag = subprocess.check_output(["git", "describe", "--tags"], text=True, stderr=subprocess.DEVNULL).strip()
|
9029
|
+
git_tag = f" ({git_tag})"
|
9030
|
+
except subprocess.CalledProcessError:
|
9031
|
+
pass
|
9032
|
+
|
9033
|
+
if commit_hash:
|
9034
|
+
with open(file_path, mode="w", encoding="utf-8") as f:
|
9035
|
+
f.write(f"Commit: {commit_hash}{git_tag}\n")
|
9020
9036
|
|
9021
|
-
try:
|
9022
|
-
git_tag = subprocess.check_output(["git", "describe", "--tags"], text=True, stderr=subprocess.DEVNULL).strip()
|
9023
|
-
git_tag = f" ({git_tag})"
|
9024
9037
|
except subprocess.CalledProcessError:
|
9025
9038
|
pass
|
9026
9039
|
|
9027
|
-
|
9028
|
-
|
9029
|
-
|
9030
|
-
|
9031
|
-
|
9032
|
-
|
9040
|
+
@beartype
|
9041
|
+
def write_job_start_file() -> None:
|
9042
|
+
with console.status("[bold green]Writing job_start_time file..."):
|
9043
|
+
fn = f'{get_current_run_folder()}/job_start_time.txt'
|
9044
|
+
try:
|
9045
|
+
with open(fn, mode='w', encoding="utf-8") as f:
|
9046
|
+
f.write(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
|
9047
|
+
except Exception as e:
|
9048
|
+
print_red(f"Error trying to write {fn}: {e}")
|
9033
9049
|
|
9034
9050
|
@beartype
|
9035
9051
|
def write_live_share_file_if_needed() -> None:
|
9036
|
-
|
9037
|
-
|
9052
|
+
with console.status("[bold green]Writing live_share file if it is present..."):
|
9053
|
+
if args.live_share:
|
9054
|
+
write_state_file("live_share", "1\n")
|
9038
9055
|
|
9039
9056
|
@beartype
|
9040
9057
|
def write_username_statefile() -> None:
|
9041
|
-
|
9042
|
-
|
9043
|
-
|
9058
|
+
with console.status("[bold green]Writing username state file..."):
|
9059
|
+
_path = get_current_run_folder()
|
9060
|
+
if args.username:
|
9061
|
+
file_path = f"{_path}/state_files/username"
|
9062
|
+
|
9063
|
+
try:
|
9064
|
+
os.makedirs(os.path.dirname(file_path), exist_ok=True)
|
9065
|
+
with open(file_path, mode="w", encoding="utf-8") as f:
|
9066
|
+
f.write(args.username)
|
9067
|
+
except Exception as e:
|
9068
|
+
print_red(f"Error writing to file: {e}")
|
9069
|
+
|
9070
|
+
@beartype
|
9071
|
+
def write_revert_to_random_when_seemingly_exhausted_file() -> None:
|
9072
|
+
with console.status("[bold green]Writing revert_to_random_when_seemingly_exhausted file ..."):
|
9073
|
+
_path = get_current_run_folder()
|
9074
|
+
file_path = f"{_path}/state_files/revert_to_random_when_seemingly_exhausted"
|
9044
9075
|
|
9045
9076
|
try:
|
9046
9077
|
os.makedirs(os.path.dirname(file_path), exist_ok=True)
|
9047
9078
|
with open(file_path, mode="w", encoding="utf-8") as f:
|
9048
|
-
f.write(
|
9079
|
+
f.write("1\n")
|
9049
9080
|
except Exception as e:
|
9050
9081
|
print_red(f"Error writing to file: {e}")
|
9051
9082
|
|
9052
|
-
@beartype
|
9053
|
-
def write_revert_to_random_when_seemingly_exhausted_file() -> None:
|
9054
|
-
_path = get_current_run_folder()
|
9055
|
-
file_path = f"{_path}/state_files/revert_to_random_when_seemingly_exhausted"
|
9056
|
-
|
9057
|
-
try:
|
9058
|
-
os.makedirs(os.path.dirname(file_path), exist_ok=True)
|
9059
|
-
with open(file_path, mode="w", encoding="utf-8") as f:
|
9060
|
-
f.write("1\n")
|
9061
|
-
except Exception as e:
|
9062
|
-
print_red(f"Error writing to file: {e}")
|
9063
|
-
|
9064
9083
|
@beartype
|
9065
9084
|
def debug_vars_unused_by_python_for_linter() -> None:
|
9066
9085
|
print_debug(
|
@@ -9327,6 +9346,28 @@ def set_arg_states_from_continue() -> None:
|
|
9327
9346
|
if os.path.exists(f"{args.continue_previous_job}/state_files/revert_to_random_when_seemingly_exhausted"):
|
9328
9347
|
args.revert_to_random_when_seemingly_exhausted = True
|
9329
9348
|
|
9349
|
+
@beartype
|
9350
|
+
def write_result_min_max_file() -> None:
|
9351
|
+
with console.status("[bold green]Writing result min/max file..."):
|
9352
|
+
try:
|
9353
|
+
fn = f"{get_current_run_folder()}/result_min_max.txt"
|
9354
|
+
with open(fn, mode="a", encoding="utf-8") as myfile:
|
9355
|
+
for rarg in arg_result_min_or_max:
|
9356
|
+
original_print(rarg, file=myfile)
|
9357
|
+
except Exception as e:
|
9358
|
+
print_red(f"Error trying to open file '{fn}': {e}")
|
9359
|
+
|
9360
|
+
@beartype
|
9361
|
+
def write_result_names_file() -> None:
|
9362
|
+
with console.status("[bold green]Writing result names file..."):
|
9363
|
+
try:
|
9364
|
+
fn = f"{get_current_run_folder()}/result_names.txt"
|
9365
|
+
with open(fn, mode="a", encoding="utf-8") as myfile:
|
9366
|
+
for rarg in arg_result_names:
|
9367
|
+
original_print(rarg, file=myfile)
|
9368
|
+
except Exception as e:
|
9369
|
+
print_red(f"Error trying to open file '{fn}': {e}")
|
9370
|
+
|
9330
9371
|
@beartype
|
9331
9372
|
def main() -> None:
|
9332
9373
|
global RESULT_CSV_FILE, ax_client, LOGFILE_DEBUG_GET_NEXT_TRIALS
|
@@ -9354,21 +9395,9 @@ def main() -> None:
|
|
9354
9395
|
|
9355
9396
|
write_username_statefile()
|
9356
9397
|
|
9357
|
-
|
9358
|
-
fn = f"{get_current_run_folder()}/result_names.txt"
|
9359
|
-
with open(fn, mode="a", encoding="utf-8") as myfile:
|
9360
|
-
for rarg in arg_result_names:
|
9361
|
-
original_print(rarg, file=myfile)
|
9362
|
-
except Exception as e:
|
9363
|
-
print_red(f"Error trying to open file '{fn}': {e}")
|
9398
|
+
write_result_names_file()
|
9364
9399
|
|
9365
|
-
|
9366
|
-
fn = f"{get_current_run_folder()}/result_min_max.txt"
|
9367
|
-
with open(fn, mode="a", encoding="utf-8") as myfile:
|
9368
|
-
for rarg in arg_result_min_or_max:
|
9369
|
-
original_print(rarg, file=myfile)
|
9370
|
-
except Exception as e:
|
9371
|
-
print_red(f"Error trying to open file '{fn}': {e}")
|
9400
|
+
write_result_min_max_file()
|
9372
9401
|
|
9373
9402
|
if os.getenv("CI"):
|
9374
9403
|
data_dict: dict = {
|
@@ -9394,12 +9423,7 @@ def main() -> None:
|
|
9394
9423
|
|
9395
9424
|
write_live_share_file_if_needed()
|
9396
9425
|
|
9397
|
-
|
9398
|
-
try:
|
9399
|
-
with open(fn, mode='w', encoding="utf-8") as f:
|
9400
|
-
f.write(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
|
9401
|
-
except Exception as e:
|
9402
|
-
print_red(f"Error trying to write {fn}: {e}")
|
9426
|
+
write_job_start_file()
|
9403
9427
|
|
9404
9428
|
write_git_version()
|
9405
9429
|
|
@@ -9410,6 +9434,7 @@ def main() -> None:
|
|
9410
9434
|
set_random_steps(_random_steps)
|
9411
9435
|
|
9412
9436
|
add_exclude_to_defective_nodes()
|
9437
|
+
|
9413
9438
|
handle_random_steps()
|
9414
9439
|
|
9415
9440
|
set_global_generation_strategy()
|
@@ -9463,30 +9488,34 @@ def log_worker_creation() -> None:
|
|
9463
9488
|
|
9464
9489
|
@beartype
|
9465
9490
|
def set_run_folder() -> None:
|
9466
|
-
|
9467
|
-
|
9468
|
-
|
9469
|
-
|
9470
|
-
while os.path.exists(f"{CURRENT_RUN_FOLDER}"):
|
9471
|
-
RUN_FOLDER_NUMBER += 1
|
9491
|
+
with console.status("[bold green]Setting run folder..."):
|
9492
|
+
global CURRENT_RUN_FOLDER
|
9493
|
+
RUN_FOLDER_NUMBER: int = 0
|
9472
9494
|
CURRENT_RUN_FOLDER = f"{args.run_dir}/{global_vars['experiment_name']}/{RUN_FOLDER_NUMBER}"
|
9473
9495
|
|
9496
|
+
while os.path.exists(f"{CURRENT_RUN_FOLDER}"):
|
9497
|
+
RUN_FOLDER_NUMBER += 1
|
9498
|
+
CURRENT_RUN_FOLDER = f"{args.run_dir}/{global_vars['experiment_name']}/{RUN_FOLDER_NUMBER}"
|
9499
|
+
|
9474
9500
|
@beartype
|
9475
9501
|
def print_run_info() -> None:
|
9476
|
-
|
9477
|
-
|
9478
|
-
|
9502
|
+
with console.status("[bold green]Printing run info..."):
|
9503
|
+
original_print(f"Run-folder: {get_current_run_folder()}")
|
9504
|
+
if args.continue_previous_job:
|
9505
|
+
original_print(f"Continuation from {args.continue_previous_job}")
|
9479
9506
|
|
9480
9507
|
@beartype
|
9481
9508
|
def initialize_nvidia_logs() -> None:
|
9482
|
-
|
9483
|
-
|
9509
|
+
with console.status("[bold green]Initializing NVIDIA-Logs..."):
|
9510
|
+
global NVIDIA_SMI_LOGS_BASE
|
9511
|
+
NVIDIA_SMI_LOGS_BASE = f'{get_current_run_folder()}/gpu_usage_'
|
9484
9512
|
|
9485
9513
|
@beartype
|
9486
9514
|
def write_ui_url_if_present() -> None:
|
9487
|
-
|
9488
|
-
|
9489
|
-
|
9515
|
+
with console.status("[bold green]Writing ui_url file if it is present..."):
|
9516
|
+
if args.ui_url:
|
9517
|
+
with open(f"{get_current_run_folder()}/ui_url.txt", mode="a", encoding="utf-8") as myfile:
|
9518
|
+
myfile.write(decode_if_base64(args.ui_url))
|
9490
9519
|
|
9491
9520
|
@beartype
|
9492
9521
|
def set_random_steps(new_steps: int) -> None:
|
@@ -9498,9 +9527,10 @@ def set_random_steps(new_steps: int) -> None:
|
|
9498
9527
|
|
9499
9528
|
@beartype
|
9500
9529
|
def handle_random_steps() -> None:
|
9501
|
-
|
9502
|
-
|
9503
|
-
|
9530
|
+
with console.status("[bold green]Handling random steps..."):
|
9531
|
+
if args.parameter and args.continue_previous_job and random_steps <= 0:
|
9532
|
+
print(f"A parameter has been reset, but the earlier job already had its random phase. To look at the new search space, {args.num_random_steps} random steps will be executed.")
|
9533
|
+
set_random_steps(args.num_random_steps)
|
9504
9534
|
|
9505
9535
|
@beartype
|
9506
9536
|
def initialize_ax_client() -> None:
|
@@ -3,7 +3,7 @@
|
|
3
3
|
.general.sh,sha256=uyGMN8xNToQ0v50KoiYxm6jRmgf0orroOaodM_Nuq30,2107
|
4
4
|
.gitignore,sha256=OMaFWOR6wxjAlI85rF3euQcjQFFAl1F34abZkltKnaU,3714
|
5
5
|
.helpers.py,sha256=6uoxzpi8zfvvQy16LpGkIC1OzvDC8TR8luAOX1iW1OM,30679
|
6
|
-
.omniopt.py,sha256=
|
6
|
+
.omniopt.py,sha256=yqEJa8aRVFHT1onvUjm4b7OGFhoKj6qk2UvRCS3djjI,383154
|
7
7
|
.omniopt_plot_cpu_ram_usage.py,sha256=DbOAmdrbcZtsMnHJgHfeRngjtv6zX5J0axyua_dYezc,3932
|
8
8
|
.omniopt_plot_general.py,sha256=ZERZJkvVOoJhi7SszmTF1Iln-_08_0Aki48u3LHUW-k,6809
|
9
9
|
.omniopt_plot_gpu_usage.py,sha256=ojxVicwSoiyl7f3c-6lLuT2EpyPcSJKEcbp75LgDY2k,5107
|
@@ -26,44 +26,44 @@ omniopt_docker,sha256=LWVUeyvmA5AKqAHiH9jBUkR5uZ6AHMnSy0eET7mK6E4,3602
|
|
26
26
|
omniopt_evaluate,sha256=9oBh0_ikCuIz_aJQZrN0j39NDiIDYoSvEFmSVIoFjJE,23842
|
27
27
|
omniopt_plot,sha256=Z8ZR10p-ZRSgMeVPO-wVCJ8lk-LQtntjZ9Bk9RifCIs,13360
|
28
28
|
omniopt_share,sha256=7g5I7YdoWcA6_GDwWwq0xPf23qiVc_VDrm9ySLH7SH0,14051
|
29
|
-
pyproject.toml,sha256=
|
29
|
+
pyproject.toml,sha256=3cKwHcmY431gR4HuDICNtKZZmcgZcq01qf2dgTqU3X8,397
|
30
30
|
requirements.txt,sha256=jqvp-P5m33fwsODnA_FFVVujUwIt9z_nhBRIxAtahcQ,302
|
31
31
|
setup.cfg,sha256=HEc8uu6NpfxG5_AVh5SvXOpEFMNKPPPxgMIAH144vT4,38
|
32
32
|
test_requirements.txt,sha256=dnCbKmKalrVzNZ_-iQWf1xCxcnDsdGuhbDAr9XlGm-U,477
|
33
|
-
omniopt2-
|
34
|
-
omniopt2-
|
35
|
-
omniopt2-
|
36
|
-
omniopt2-
|
37
|
-
omniopt2-
|
38
|
-
omniopt2-
|
39
|
-
omniopt2-
|
40
|
-
omniopt2-
|
41
|
-
omniopt2-
|
42
|
-
omniopt2-
|
43
|
-
omniopt2-
|
44
|
-
omniopt2-
|
45
|
-
omniopt2-
|
46
|
-
omniopt2-
|
47
|
-
omniopt2-
|
48
|
-
omniopt2-
|
49
|
-
omniopt2-
|
50
|
-
omniopt2-
|
51
|
-
omniopt2-
|
52
|
-
omniopt2-
|
53
|
-
omniopt2-
|
54
|
-
omniopt2-
|
55
|
-
omniopt2-
|
56
|
-
omniopt2-
|
57
|
-
omniopt2-
|
58
|
-
omniopt2-
|
59
|
-
omniopt2-
|
60
|
-
omniopt2-
|
61
|
-
omniopt2.egg-info/PKG-INFO,sha256=
|
33
|
+
omniopt2-7140.data/data/bin/.colorfunctions.sh,sha256=x6RGjZAkQR24oS-0uatyX55KS-UVcPQWTCC8egI5Y9o,1227
|
34
|
+
omniopt2-7140.data/data/bin/.general.sh,sha256=uyGMN8xNToQ0v50KoiYxm6jRmgf0orroOaodM_Nuq30,2107
|
35
|
+
omniopt2-7140.data/data/bin/.helpers.py,sha256=6uoxzpi8zfvvQy16LpGkIC1OzvDC8TR8luAOX1iW1OM,30679
|
36
|
+
omniopt2-7140.data/data/bin/.omniopt.py,sha256=yqEJa8aRVFHT1onvUjm4b7OGFhoKj6qk2UvRCS3djjI,383154
|
37
|
+
omniopt2-7140.data/data/bin/.omniopt_plot_cpu_ram_usage.py,sha256=DbOAmdrbcZtsMnHJgHfeRngjtv6zX5J0axyua_dYezc,3932
|
38
|
+
omniopt2-7140.data/data/bin/.omniopt_plot_general.py,sha256=ZERZJkvVOoJhi7SszmTF1Iln-_08_0Aki48u3LHUW-k,6809
|
39
|
+
omniopt2-7140.data/data/bin/.omniopt_plot_gpu_usage.py,sha256=ojxVicwSoiyl7f3c-6lLuT2EpyPcSJKEcbp75LgDY2k,5107
|
40
|
+
omniopt2-7140.data/data/bin/.omniopt_plot_kde.py,sha256=uRLWr72TDKvj3AqJ0O0AvkKZ1ok1O1QpXnbfQQdo0nA,6873
|
41
|
+
omniopt2-7140.data/data/bin/.omniopt_plot_scatter.py,sha256=-amqmLR_YLCbHImLPh6gwVFv8iiVnXME544XHsRVCuw,8422
|
42
|
+
omniopt2-7140.data/data/bin/.omniopt_plot_scatter_generation_method.py,sha256=rgKY_w1E516c9UucVaEvaKd8tCnoUq9xg-RrYSDzYEQ,4289
|
43
|
+
omniopt2-7140.data/data/bin/.omniopt_plot_scatter_hex.py,sha256=w1L5gL6Bc_QudoSJi2lxEVvF17Apjjb3l2T-lXGnsUg,10279
|
44
|
+
omniopt2-7140.data/data/bin/.omniopt_plot_time_and_exit_code.py,sha256=hC4RFDiJN_UImezFR6M5uVF-QKDqMDpq6R5DIg7dDDc,6463
|
45
|
+
omniopt2-7140.data/data/bin/.omniopt_plot_trial_index_result.py,sha256=5DmqZAQO_PFmzdap-TIhSMAshRXpOHQacnHAtjwnzN4,4629
|
46
|
+
omniopt2-7140.data/data/bin/.omniopt_plot_worker.py,sha256=VuluQq4W6KRR5RU08dxmDSFk5mbfDRkRJQFwwcLgAGw,4524
|
47
|
+
omniopt2-7140.data/data/bin/.random_generator.py,sha256=ezBBUXpez_QaGdpCglMcJ0KZPdQP0XdX5gnLzO1xhwU,2987
|
48
|
+
omniopt2-7140.data/data/bin/.shellscript_functions,sha256=IRbIX0-BBxkQGxWBFBc-RdaGj6Rs_zu3gkxAWd5fL2k,14839
|
49
|
+
omniopt2-7140.data/data/bin/.tpe.py,sha256=Yd9s-ixCbWxTd3x0O1M1sok9QfM2mBEfsDwXhx50-Nc,6464
|
50
|
+
omniopt2-7140.data/data/bin/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
|
51
|
+
omniopt2-7140.data/data/bin/apt-dependencies.txt,sha256=X5tBB8ZLW9XaFtRh8B7C2pIkSoxNNawqioDr0QZAtuM,149
|
52
|
+
omniopt2-7140.data/data/bin/omniopt,sha256=3JRBCx_MjkcRlc0y80VNxV0cH0Zs4Va95pU4ojk9dlU,47447
|
53
|
+
omniopt2-7140.data/data/bin/omniopt_docker,sha256=LWVUeyvmA5AKqAHiH9jBUkR5uZ6AHMnSy0eET7mK6E4,3602
|
54
|
+
omniopt2-7140.data/data/bin/omniopt_evaluate,sha256=9oBh0_ikCuIz_aJQZrN0j39NDiIDYoSvEFmSVIoFjJE,23842
|
55
|
+
omniopt2-7140.data/data/bin/omniopt_plot,sha256=Z8ZR10p-ZRSgMeVPO-wVCJ8lk-LQtntjZ9Bk9RifCIs,13360
|
56
|
+
omniopt2-7140.data/data/bin/omniopt_share,sha256=7g5I7YdoWcA6_GDwWwq0xPf23qiVc_VDrm9ySLH7SH0,14051
|
57
|
+
omniopt2-7140.data/data/bin/requirements.txt,sha256=jqvp-P5m33fwsODnA_FFVVujUwIt9z_nhBRIxAtahcQ,302
|
58
|
+
omniopt2-7140.data/data/bin/setup.py,sha256=g3uEqJHXhggXwgLYoxOjsXg9Z6IV1ubh-Og59AZ264Q,4648
|
59
|
+
omniopt2-7140.data/data/bin/test_requirements.txt,sha256=dnCbKmKalrVzNZ_-iQWf1xCxcnDsdGuhbDAr9XlGm-U,477
|
60
|
+
omniopt2-7140.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
|
61
|
+
omniopt2.egg-info/PKG-INFO,sha256=CJ6UQg1NCjmltwxD1Tw0iZHsYU1VQkevBP2oSsv8G74,5844
|
62
62
|
omniopt2.egg-info/SOURCES.txt,sha256=N-HtSaaqFRsd4XqAfeWVSp__3I-sw0d7cknJgyewRwQ,778
|
63
63
|
omniopt2.egg-info/dependency_links.txt,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
64
64
|
omniopt2.egg-info/requires.txt,sha256=AoHFBcKgH-fRh6E2OhIgWce1ao2Bkk8LK0FWTsGhAIk,779
|
65
65
|
omniopt2.egg-info/top_level.txt,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
66
|
-
omniopt2-
|
67
|
-
omniopt2-
|
68
|
-
omniopt2-
|
69
|
-
omniopt2-
|
66
|
+
omniopt2-7140.dist-info/METADATA,sha256=CJ6UQg1NCjmltwxD1Tw0iZHsYU1VQkevBP2oSsv8G74,5844
|
67
|
+
omniopt2-7140.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
68
|
+
omniopt2-7140.dist-info/top_level.txt,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
69
|
+
omniopt2-7140.dist-info/RECORD,,
|
omniopt2.egg-info/PKG-INFO
CHANGED
pyproject.toml
CHANGED
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
{omniopt2-7137.data → omniopt2-7140.data}/data/bin/.omniopt_plot_scatter_generation_method.py
RENAMED
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|