omniopt2 7773__py3-none-any.whl → 8167__py3-none-any.whl

omniopt

@@ -181,46 +181,13 @@
 		trap - DEBUG
 	}
 
-	MAIN_PID=$$
-
-	log_cpu_usage() {
-		if [[ -n "$RUN_UUID" ]]; then
-			if command -v ps 2>/dev/null >/dev/null; then
-				mkdir -p logs
-
-				LOG_FILE="logs/${RUN_UUID}_cpu_log"
-
-				[[ ! -f "$LOG_FILE" ]] && echo "timestamp,process,command,cpu_usage" > "$LOG_FILE"
-				
-				_continue=1
-
-				while [[ $_continue -eq 1 ]]; do
-					if ! kill -0 "$MAIN_PID" 2>/dev/null; then
-						_continue=0
-					fi
-
-					TIMESTAMP=$(date '+%Y-%m-%d %H:%M:%S')
-
-					if [[ -e "$LOG_FILE" ]]; then
-						ps -eo ppid,pid,pcpu,comm,args | awk -v ts="$TIMESTAMP" -v ppid=$$ 'NR>1 && $1 == ppid {print ts "," $4 "," substr($0, index($0,$5)) "," $3}' >> "$LOG_FILE"
-					else
-						_continue=0
-					fi
-
-					sleep 1
-				done &
-			fi
-		fi
-	}
-
-	#log_cpu_usage &
-
 	checkout_to_latest_tested_version=0
 	debug=0
 	main_process_gb=
 	run_tests_that_fail_on_taurus=0
 	force_local_execution=0
 	dryrun=0
+	worker_generator_path=""
 	has_sbatch=0
 
 	function parse_toml {
@@ -721,6 +688,7 @@
 		continue_previous_job=""
 		calculate_pareto_front_of_job=""
 		mem_gb=
+		dependency=
 		gpus=
 		time=
 		if [[ -z $root_venv_dir ]]; then
@@ -862,6 +830,7 @@ EOF
 		"experiment_name"
 		"reservation"
 		"time"
+		"dependency"
 	)
 
 	complex_options_str=""
@@ -957,6 +926,36 @@ $complex_options_str
 
 			;;
 
+		--worker_generator_path=*)
+			old_worker_generator_path=\$worker_generator_path
+			worker_generator_path="\${i#*=}"
+
+			if [[ -n \$old_worker_generator_path ]]; then
+				red_text "--worker_generator_path was specified more than once. Using the last one.\\n"
+				args_string=\$(echo "\$args_string" | sed -e 's#[[:space:]]*--worker_generator_path=.*[[:space:]][[:space:]]*# #')
+			fi
+
+			args_string+=" --worker_generator_path \$worker_generator_path "
+
+			;;
+
+		--worker_generator_path)
+			old_worker_generator_path=\$worker_generator_path
+
+			shift
+			k=\$((k+1))
+
+			worker_generator_path="\${args[k]}"
+
+			if [[ -n \$old_worker_generator_path ]]; then
+				red_text "--worker_generator_path was specified more than once. Using the last one.\\n"
+				args_string=\$(echo "\$args_string" | sed -e 's#[[:space:]]*--worker_generator_path.*[[:space:]][[:space:]]*# #')
+			fi
+
+			args_string+=" --worker_generator_path \$worker_generator_path "
+
+			;;
+
 		--calculate_pareto_front_of_job=*)
 			old_continue=\$calculate_pareto_front_of_job
 			calculate_pareto_front_of_job="\${i#*=}"
@@ -1057,6 +1056,19 @@ EOF
 
 	eval "$PARAM_EVAL"
 
+	if [[ $worker_generator_path != "" ]]; then
+		old_uuid_dir="$worker_generator_path/state_files/run_uuid"
+		if [[ -e $old_uuid_dir ]]; then
+			RUN_UUID=$(cat $old_uuid_dir)
+			if [[ -z $SLURM_JOB_ID ]]; then
+				yellow_text "--worker_generator_path defined: $worker_generator_path. Using UUID from that run."
+				yellow_text "Set RUN_UUID to $RUN_UUID"
+			fi
+		else
+			red_text "Old UUID cannot be found"
+		fi
+	fi
+
 	if [[ $dryrun -eq 1 ]]; then
 		force_local_execution=1
 		live_share=0
@@ -1181,8 +1193,7 @@ EOF
 							myexit $exit_code
 						else
 							if [[ -z $OO_MAIN_TESTS ]]; then
-								yellow_text "The current version was not thoroughly tested. It may contain bugs. Checkout to $tag_commit_hash to get the latest tested version."
-								yellow_text "Use --checkout_to_latest_tested_version to automatically use the latest tested version."
+								yellow_text "The current version was not thoroughly tested. It may contain bugs. Checkout to $tag_commit_hash or use --checkout_to_latest_tested_version to get the latest tested version."
 							fi
 						fi
 					fi
@@ -1348,6 +1359,14 @@ EOF
 	trap - ERR
 	set +e
 
+	if [[ -z $dependency ]]; then
+		_dependency=$(get_config_from_config_file "dependency" 0 2>/dev/null)
+		retcode=$?
+		if [[ $retcode -ne 0 ]] && [[ -n $_dependency ]]; then
+			dependency=$_dependency
+		fi
+	fi
+
 	if [[ -z $reservation ]]; then
 		_reservation=$(get_config_from_config_file "reservation" 0 2>/dev/null)
 		retcode=$?
@@ -1656,12 +1675,43 @@ EOF
 			fi
 		fi
 
+		if [[ $dependency == "omniopt_singleton" ]]; then
+			k=0
+			deps=""
+			while read -r job_id; do
+				if scontrol show job "$job_id" 2>/dev/null | grep -qi omniopt; then
+					[[ -n $deps ]] && deps+=","
+					deps+="after:$job_id"
+					k=$((k+1))
+				fi
+			done < <(squeue -h -u "$USER" -o "%A")
+
+			if (( k > 0 )); then
+				dependency="$deps"
+				echo "Automatic dependencies: $dependency"
+			else
+				dependency=""
+			fi
+		fi
+
+		if [[ -n $dependency ]]; then
+			if [[ $dependency =~ ^(afterok:[0-9]+|afternotok:[0-9]+|after:[0-9]+|afterany:[0-9]+|singleton)(,(afterok:[0-9]+|afternotok:[0-9]+|after:[0-9]+|afterany:[0-9]+|singleton))*$ ]]; then
+				if [[ $debug -eq 1 ]]; then
+					yellow_text "Valid dependency found: $dependency. Adding it it to sbatch command."
+				fi
+				sbatch_command="$sbatch_command --dependency $dependency"
+			else
+				echo "Invalid dependency format: $dependency"
+			fi
+		fi
+
 		sbatch_command+=" $SCRIPT_DIR/omniopt $args_string"
 
 		if [[ "$debug" -eq "1" ]] || [[ -n $PRINT_SBATCH_COMMAND ]]; then
 			yellow_text "$sbatch_command"
 		fi
 
+
 		set +e
 
 		sbatch_result=$($sbatch_command)
@@ -1835,7 +1885,46 @@ EOF
 							exec 2> >(tee -ia $bash_logname >& 2)
 						fi
 
-						exit_code=$(cat "$LOG_PATH" | grep -i "exit-code:*" | sed -e 's#Exit-Code:*[[:space:]]*##i' -e 's#,.*##' | tail -n1)
+
+						exit_code_lines=$(grep -i "exit-code:*" "$LOG_PATH" 2>/dev/null)
+						if [ $? -ne 0 ] || [ -z "$exit_code_lines" ]; then
+							echo "WARN: grep failed or no exit-code line found."
+							exit_code_lines=""
+						fi
+
+						exit_code_sed=$(echo "$exit_code_lines" | sed -e 's#Exit-Code:*[[:space:]]*##i' -e 's#,.*##')
+						if [ $? -ne 0 ] || [ -z "$exit_code_sed" ]; then
+							echo "WARN: sed failed or no data after sed."
+							exit_code_sed=""
+						fi
+
+						exit_code_tail=$(echo "$exit_code_sed" | tail -n1)
+						if [ $? -ne 0 ] || [ -z "$exit_code_tail" ]; then
+							echo "WARN: tail failed or no data after tail."
+							exit_code_tail=""
+						fi
+
+						exit_code_only_digits=$(echo "$exit_code_tail" | grep -o '[0-9]\+')
+						if [ -z "$exit_code_only_digits" ]; then
+							echo "WARN: No valid exit code found, setting it to 3"
+							exit_code_only_digits=3
+						fi
+
+						exit_code="$exit_code_only_digits"
+
+						if ! [[ "$exit_code" =~ ^[0-9]+$ ]]; then
+							echo "WARN: exit_code invalid ('$exit_code'), setting to 3"
+							exit_code=3
+						fi
+
+						if (( exit_code != 0 )); then
+							if declare -f myexit > /dev/null; then
+								myexit "$exit_code"
+							else
+								echo "WARN: myexit function not found, exiting with code $exit_code"
+								exit "$exit_code"
+							fi
+						fi
 					else
 						red_text "$LOG_PATH could not be found\n"
 					fi
@@ -1865,7 +1954,7 @@ EOF
 			sbatch_command=$(echo "$sbatch_command" | sed -e 's#^[[:space:]]*##' -e 's#[[:space:]]*$##')
 			red_text "Failed to start sbatch job. Command:\n"
 			red_text "$sbatch_command\n"
-			red_text "Exit-Code: $exit_code\n"
+			red_text "Exit-Code (sbatch): $exit_code\n"
 
 			myexit $exit_code
 		fi