omniopt2 8471__py3-none-any.whl → 9171__py3-none-any.whl

.omniopt_plot_scatter.py

@@ -79,7 +79,7 @@ if ORIGINAL_PWD:
     os.chdir(ORIGINAL_PWD)
 
 @beartype
-def set_title(df_filtered: pd.DataFrame, result_column_values: pd.core.series.Series, num_entries: int, _min: Union[int, float, None], _max: Union[int, float, None]) -> None:
+def set_title(df_filtered: pd.DataFrame, result_column_values: pd.Series, num_entries: int, _min: Union[int, float, None], _max: Union[int, float, None]) -> None:
     title = helpers.get_title(args, result_column_values, df_filtered, num_entries, _min, _max)
 
     if fig:

.omniopt_plot_scatter_hex.py

@@ -93,7 +93,7 @@ if ORIGINAL_PWD:
     os.chdir(ORIGINAL_PWD)
 
 @beartype
-def set_title(df_filtered: pd.DataFrame, result_column_values: pd.core.series.Series, num_entries: int, _min: Union[float, int, None] = None, _max: Union[float, int, None] = None) -> None:
+def set_title(df_filtered: pd.DataFrame, result_column_values: pd.Series, num_entries: int, _min: Union[float, int, None] = None, _max: Union[float, int, None] = None) -> None:
     title = helpers.get_title(args, result_column_values, df_filtered, num_entries, _min, _max)
 
     if fig:

.pareto.py

@@ -0,0 +1,134 @@
+import os
+import importlib.util
+from typing import Tuple, List, Dict, Any
+from rich.table import Table
+from rich.text import Text
+import numpy as np
+
+
+# ----------------- Helpers import -----------------
+script_dir = os.path.dirname(os.path.realpath(__file__))
+helpers_file: str = f"{script_dir}/.helpers.py"
+spec = importlib.util.spec_from_file_location(
+    name="helpers",
+    location=helpers_file,
+)
+if spec is not None and spec.loader is not None:
+    helpers = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(helpers)
+else:
+    raise ImportError(f"Could not load module from {helpers_file}")
+
+# ----------------- Table utilities -----------------
+def pareto_front_table_filter_rows(rows: List[Dict[str, str]], idxs: List[int]) -> List[Dict[str, str]]:
+    result = []
+    for row in rows:
+        try:
+            trial_index = int(row["trial_index"])
+        except (KeyError, ValueError):
+            continue
+
+        if row.get("trial_status", "").strip().upper() == "COMPLETED" and trial_index in idxs:
+            result.append(row)
+    return result
+
+def pareto_front_table_add_headers(table: Table, param_cols: List[str], result_cols: List[str]) -> None:
+    for col in param_cols:
+        table.add_column(col, justify="center")
+    for col in result_cols:
+        table.add_column(Text(f"{col}", style="cyan"), justify="center")
+
+def pareto_front_table_add_rows(table: Table, rows: List[Dict[str, str]], param_cols: List[str], result_cols: List[str]) -> None:
+    for row in rows:
+        values = [str(helpers.to_int_when_possible(row[col])) for col in param_cols]
+        result_values = [Text(str(helpers.to_int_when_possible(row[col])), style="cyan") for col in result_cols]
+        table.add_row(*values, *result_values, style="bold green")
+
+# ----------------- Pareto internal utilities -----------------
+def _validate_xy(x: np.ndarray, y: np.ndarray) -> None:
+    if x.shape != y.shape:
+        raise ValueError("Input arrays x and y must have the same shape.")
+    if x.size == 0:
+        raise ValueError("Input arrays must not be empty.")
+    if not np.isfinite(x).all() or not np.isfinite(y).all():
+        raise ValueError("Input arrays must contain only finite numbers (no NaN/Inf).")
+
+def _point_dominates(xi: float, yi: float, xj: float, yj: float, x_minimize: bool, y_minimize: bool) -> bool:
+    """Return True if j dominates i (standard weak Pareto: >= in all, > in at least one)."""
+    if x_minimize:
+        x_better_eq = xj <= xi
+        x_strict = xj < xi
+    else:
+        x_better_eq = xj >= xi
+        x_strict = xj > xi
+
+    if y_minimize:
+        y_better_eq = yj <= yi
+        y_strict = yj < yi
+    else:
+        y_better_eq = yj >= yi
+        y_strict = yj > yi
+
+    return x_better_eq and y_better_eq and (x_strict or y_strict)
+
+def _find_dominated(x: np.ndarray, y: np.ndarray, x_minimize: bool, y_minimize: bool) -> np.ndarray:
+    n = len(x)
+    is_dominated = np.zeros(n, dtype=bool)
+    for i in range(n):
+        if is_dominated[i]:
+            continue
+        xi, yi = x[i], y[i]
+        for j in range(n):
+            if i == j:
+                continue
+            xj, yj = x[j], y[j]
+            if _point_dominates(xi, yi, xj, yj, x_minimize, y_minimize):
+                is_dominated[i] = True
+                break
+    return is_dominated
+
+# ----------------- Public Pareto functions -----------------
+def pareto_front_general(
+    x: np.ndarray,
+    y: np.ndarray,
+    x_minimize: bool = True,
+    y_minimize: bool = True
+) -> np.ndarray:
+    try:
+        xa = np.asarray(x, dtype=float).ravel()
+        ya = np.asarray(y, dtype=float).ravel()
+        _validate_xy(xa, ya)
+        is_dominated = _find_dominated(xa, ya, x_minimize, y_minimize)
+        return np.where(~is_dominated)[0]
+    except Exception as e:
+        print("Error in pareto_front_general:", str(e))
+        return np.array([], dtype=int)
+
+def pareto_front_filter_complete_points(
+    path_to_calculate: str,
+    records: Dict[Tuple[int, str], Dict[str, Dict[str, float]]],
+    primary_name: str,
+    secondary_name: str
+) -> List[Tuple[Tuple[int, str], float, float]]:
+    points = []
+    for key, metrics in records.items():
+        means = metrics['means']
+        if primary_name in means and secondary_name in means:
+            points.append((key, means[primary_name], means[secondary_name]))
+    if len(points) == 0:
+        raise ValueError(f"No full data points with both objectives found in {path_to_calculate}.")
+    return points
+
+def pareto_front_select_pareto_points(
+    x: np.ndarray,
+    y: np.ndarray,
+    x_minimize: bool,
+    y_minimize: bool,
+    points: List[Tuple[Any, float, float]],
+    num_points: int
+) -> List[Tuple[Any, float, float]]:
+    indices = pareto_front_general(x, y, x_minimize, y_minimize)
+    # Sort by x ascending
+    sorted_indices = indices[np.argsort(x[indices])]
+    selected_points = [points[i] for i in sorted_indices[:num_points]]
+    return selected_points

.shellscript_functions

@@ -207,13 +207,14 @@
 			bar="${bar} "
 		done
 
-		printf "[%s] %d%% \n" "$bar" "$percentage"
+		printf "[%s] \n" "$bar"
 	}
 
 	function ppip {
 		MODULE=$1
 		AS_REQUIREMENT_OF=$2
 		NUMBER_OF_MAIN_MODULES=$3
+		STR=$4
 
 		set +e
 
@@ -222,7 +223,7 @@
 		PROGRESSBAR=$(generate_progress_bar_setup "$NUMBER_OF_MAIN_MODULES")
 
 		if [[ -z $CI ]]; then
-			green_reset_line "${PROGRESSBAR}➤Installing $MODULE "
+			green_reset_line "${PROGRESSBAR}➤${STR}Installing $MODULE "
 		fi
 
 
@@ -235,10 +236,10 @@
 			if [[ "$MODULE" != "$AS_REQUIREMENT_OF" ]] && [[ "$AS_REQUIREMENT_OF" != "-" ]]; then
 				k=0
 
-				for i in $(pip3 install --disable-pip-version-check --dry-run "$MODULE" | grep -v "already satisfied" | grep "Collecting" | sed -e 's#Collecting ##' | grep -v "^$MODULE$"); do
+				for i in $(pip3 install --default-timeout=300 --default-timeout=300 --disable-pip-version-check --dry-run "$MODULE" | grep -v "already satisfied" | grep "Collecting" | sed -e 's#Collecting ##' | grep -v "^$MODULE$"); do
 					if [[ "$i" != "$MODULE" ]]; then
 						if [[ $k -eq 0 ]]; then
-							green_reset_line "${PROGRESSBAR}➤Installing requirements for $MODULE"
+							green_reset_line "${PROGRESSBAR}➤${STR}Installing requirements for $MODULE"
 						fi
 						ppip "$i" "$MODULE" "$NUMBER_OF_MAIN_MODULES" || {
 							red_reset_line "❌Failed to install $i."
@@ -251,11 +252,11 @@
 				done
 
 				if [[ $k -gt 0 ]]; then
-					green_reset_line "${PROGRESSBAR}➤Installed all requirements for $MODULE, now installing the package itself..."
+					green_reset_line "${PROGRESSBAR}➤${STR}Installed all requirements for $MODULE, now installing the package itself..."
 				fi
 			fi
 
-			green_reset_line "${PROGRESSBAR}➤Installing $MODULE..."
+			green_reset_line "${PROGRESSBAR}➤${STR}Installing $MODULE..."
 			mkdir -p logs
 			export PIP_DISABLE_PIP_VERSION_CHECK=1
 			INSTALL_ERRORS_FILE="logs/install_errors"
@@ -265,8 +266,8 @@
 			fi
 
 			if [[ -z $DEBUG ]]; then
-				pip3 --disable-pip-version-check install -q $MODULE >&2 2>> $INSTALL_ERRORS_FILE || pip3 --disable-pip-version-check install $MODULE >&2 2>> $INSTALL_ERRORS_FILE || {
-					red_reset_line "❌Failed to install $MODULE. Check $INSTALL_ERRORS_FILE"
+				pip3 --default-timeout=300 --disable-pip-version-check install -q $MODULE >&2 2>> $INSTALL_ERRORS_FILE || pip3 --default-timeout=300 --disable-pip-version-check install $MODULE >&2 2>> $INSTALL_ERRORS_FILE || {
+					red_reset_line "❌Failed to install $MODULE. Check $INSTALL_ERRORS_FILE\n"
 
 					if [[ -n $CI ]] || { [[ -f /proc/self/cgroup ]] && grep -qE '/docker|/lxc' /proc/self/cgroup; }; then
 						cat "$INSTALL_ERRORS_FILE" 
@@ -275,8 +276,8 @@
 					exit 20
 				}
 			else
-				pip3 --disable-pip-version-check install $MODULE >&2 2>> $INSTALL_ERRORS_FILE || pip3 --disable-pip-version-check install $MODULE >&2 2>> $INSTALL_ERRORS_FILE || {
-					red_reset_line "❌Failed to install $MODULE. Check $INSTALL_ERRORS_FILE"
+				pip3  --default-timeout=300 --disable-pip-version-check install $MODULE >&2 2>> $INSTALL_ERRORS_FILE || pip3 --default-timeout=300 --disable-pip-version-check install $MODULE >&2 2>> $INSTALL_ERRORS_FILE || {
+					red_reset_line "❌Failed to install $MODULE. Check $INSTALL_ERRORS_FILE\n"
 
 					if [[ -n $CI ]] || { [[ -f /proc/self/cgroup ]] && grep -qE '/docker|/lxc' /proc/self/cgroup; }; then
 						cat "$INSTALL_ERRORS_FILE" 
@@ -323,17 +324,25 @@
 		MAX_NR="${#install_those[@]}"
 		NUMBER_OF_INSTALLED_MODULES=$(get_nr_of_already_installed_modules)
 
-		PROGRESSBAR=$(generate_progress_bar_setup "$MAX_NR")
+		MAX_PERCENT=100
 
 		for key in "${!install_those[@]}"; do
+			idx=$((idx+1))
 			install_this=${install_those[$key]}
-			PROGRESSBAR=$(generate_progress_bar_setup "$MAX_NR")
+			percent=$((idx * 100 / MAX_NR))
+
+			PROGRESSBAR=$(generate_progress_bar "$idx" "$MAX_NR")
+
+			STR="Checking ${idx_padded}/${MAX_NR}, ${percent_padded}%: "
+
 			if [[ -z $CI ]]; then
-				green_reset_line "${PROGRESSBAR}➤Checking if $install_this is installed..."
+				printf -v idx_padded "%*d" "${#MAX_NR}" "$idx"
+				printf -v percent_padded "%*d" "${#MAX_PERCENT}" "$percent"
+				green_reset_line "${PROGRESSBAR}➤$STR$install_this..."
 			fi
 
 			if ! echo "$FROZEN" | grep -q "$install_this"; then
-				ppip "$install_this" "-" "$MAX_NR"
+				ppip "$install_this" "-" "$MAX_NR" "$STR"
 			fi
 		done
 
@@ -451,7 +460,7 @@
 				exit 20
 			}
 
-			downgrade_output=$(pip3 --disable-pip-version-check install -q pip==24.0) || {
+			downgrade_output=$(pip3 --default-timeout=300 --disable-pip-version-check install -q pip==24.0) || {
 				red_text "Failed to downgrade pip. Output:"
 				red_text "$downgrade_output"
 			}

.tests/pylint.rc

@@ -103,10 +103,6 @@ recursive=no
 # source root.
 source-roots=
 
-# When enabled, pylint would attempt to guess common misconfiguration and emit
-# user-friendly hints instead of false-positive error messages.
-suggestion-mode=yes
-
 # Allow loading of arbitrary C extensions. Extensions are imported into the
 # active Python interpreter and may run arbitrary code.
 unsafe-load-any-extension=no

README.md

@@ -57,7 +57,7 @@ for more details.
 
 ## Install from pypi
 
-This may not use the bleeding-edge version, but if you get the version from here it means, the test suite has completety tested it properly.
+This may not use the bleeding-edge version, but if you get the version from here it means, the test suite has completely tested it properly.
 
 ```command
 pip3 install omniopt2

omniopt

@@ -5,7 +5,7 @@
 # Step 1:
 # It checks whether the environment already exists, and
 # if so, loads it.
-# If it doesn't exist. it creates it in
+# If it doesn't exist, it creates it in
 # ~/.omniax_$(uname -m)_$(python3 --version | sed -e 's# #_#g')
 # i.e. e.g. ~/.omniax_x86_64_Python_3.11.2/ . This is done so that there
 # is no need for multiple installations, so that, once installed, it doesn't
@@ -211,7 +211,7 @@
 		fi
 
 		if [[ ! -e $file ]]; then
-			echo "Cannot parse non-existant toml." >&2
+			echo "Cannot parse non-existent toml." >&2
 			return 1
 		fi
 
@@ -238,7 +238,7 @@
 		fi
 
 		if [[ ! -e "$file" ]]; then
-			echo "Cannot parse non-existant yaml." >&2
+			echo "Cannot parse non-existent yaml." >&2
 			return 1
 		fi
 
@@ -270,7 +270,7 @@
 		fi
 
 		if [[ ! -e "$file" ]]; then
-			echo "Cannot parse non-existant json." >&2
+			echo "Cannot parse non-existent json." >&2
 			return 1
 		fi
 
@@ -337,7 +337,7 @@
 
 	int_re='^[+-]?[0-9]+$'
 
-	if [[ -n $PRINT_SEPERATOR ]]; then # for tests, so that things are properly visually seperated
+	if [[ -n $PRINT_SEPARATOR ]]; then # for tests, so that things are properly visually separated
 		echo ""
 		echo "========================================================================"
 		echo ""
@@ -364,17 +364,17 @@
 		fi
 	}
 
-	function start_periodidic_live_share {
+	function start_periodic_live_share {
 		while true; do
 			run_live_share 2>/dev/null >/dev/null
-			sleep 30
+			sleep 10
 		done
 	}
 
 	function myexit {
 		CODE=$1
 
-		end_all_bg_processes
+		end_all_bg_processes 2>/dev/null
 
 		if [[ $follow -eq 1 ]]; then
 			if [[ -z $CODE ]] || ! [[ $CODE =~ $int_re ]]; then
@@ -511,7 +511,11 @@
 			if [ $minutes -eq 1 ]; then
 				result="in about $minutes minute"
 			else
-				result="in about $minutes minutes"
+				if [ $minutes -eq 0 ]; then
+					result="soon"
+				else
+					result="in about $minutes minutes"
+				fi
 			fi
 			echo "$result"
 			return
@@ -1016,6 +1020,10 @@ $complex_options_str
 			args_string+=" --help "
 			;;
 
+		--memray)
+			export RUN_WITH_MEMRAY=1
+			;;
+
 		--flame_graph)
 			export RUN_WITH_PYSPY=1
 			;;
@@ -1634,19 +1642,27 @@ EOF
 		live_share_pid=""
 
 		if [[ $live_share -eq 1 ]]; then
-			start_periodidic_live_share &
+			start_periodic_live_share &
 			live_share_pid=$!
 		fi
 
 		if [[ -z $RUN_WITH_COVERAGE ]]; then
-			if [[ -z $RUN_WITH_PYSPY ]]; then
-				stdbuf -e 0 -o 0 python3 "$SCRIPT_DIR/.omniopt.py" $args_string
-				EXIT_CODE=$?
-			else
+			if [[ -n $RUN_WITH_PYSPY ]]; then
 				echo "Starting OmniOpt with Py-Spy"
 				pip install py-spy
 				stdbuf -e 0 -o 0 py-spy record --rate 10 --subprocesses --native --output "$RUN_UUID.svg" python3 -- "$SCRIPT_DIR/.omniopt.py" $args_string
 				EXIT_CODE=$?
+			elif [[ -n $RUN_WITH_MEMRAY ]]; then
+				echo "Starting OmniOpt with MemRay."
+				echo "Check later on with 'memray flamegraph $RUN_UUID.bin'"
+				export PYTORCH_NO_CUDA_MEMORY_CACHING=1
+				export PYTHONFAULTHANDLER=1
+				pip install memray
+				stdbuf -e 0 -o 0 memray run -o "$RUN_UUID.bin" -- "$SCRIPT_DIR/.omniopt.py" $args_string
+				EXIT_CODE=$?
+			else
+				stdbuf -e 0 -o 0 python3 "$SCRIPT_DIR/.omniopt.py" $args_string
+				EXIT_CODE=$?
 			fi
 		else
 			echo "Using coverage run -p because \$RUN_WITH_COVERAGE is set"
@@ -1655,7 +1671,7 @@ EOF
 		fi
 
 		if [[ $live_share -eq 1 ]] && [[ -n $live_share_pid ]]; then
-			kill -9 $live_share_pid
+			kill -9 $live_share_pid >/dev/null 2>/dev/null
 		fi
 
 		set -e
@@ -1865,7 +1881,7 @@ EOF
 						kill_tail_when_squeue_job_empty "$started_job_nr" &
 
 						tail_log_file() {
-							trap 'ask_cancel' SIGINT
+							#trap 'ask_cancel' SIGINT
 							# weird exec stuff for disabling the "Terminated" message coming from kill
 							exec 3>&2          # 3 is now a copy of 2
 							exec 2> /dev/null  # 2 now points to /dev/null
@@ -1912,34 +1928,29 @@ EOF
 						exit_code_lines=$(grep -i "exit-code:*" "$LOG_PATH" 2>/dev/null)
 						exit_code_lines_lines=$?
 						if [ $exit_code_lines_lines -ne 0 ] || [ -z "$exit_code_lines" ]; then
-							echo "WARN: grep failed or no exit-code line found."
 							exit_code_lines=""
 						fi
 
 						exit_code_sed=$(echo "$exit_code_lines" | sed -e 's#Exit-Code:*[[:space:]]*##i' -e 's#,.*##')
 						exit_code_sed_sed=$?
 						if [ $exit_code_sed_sed -ne 0 ] || [ -z "$exit_code_sed" ]; then
-							echo "WARN: sed failed or no data after sed."
 							exit_code_sed=""
 						fi
 
 						exit_code_tail=$(echo "$exit_code_sed" | tail -n1)
 						exit_code_tail_tail=$?
 						if [ $exit_code_tail_tail -ne 0 ] || [ -z "$exit_code_tail" ]; then
-							echo "WARN: tail failed or no data after tail."
 							exit_code_tail=""
 						fi
 
-						exit_code_only_digits=$(echo "$exit_code_tail" | grep -o '[0-9]\+')
+						exit_code_only_digits=$(echo "$exit_code_tail" | grep -o '[0-9]\+' | tail -n1)
 						if [ -z "$exit_code_only_digits" ]; then
-							echo "WARN: No valid exit code found, setting it to 3"
 							exit_code_only_digits=3
 						fi
 
 						exit_code="$exit_code_only_digits"
 
 						if ! [[ "$exit_code" =~ ^[0-9]+$ ]]; then
-							echo "WARN: exit_code invalid ('$exit_code'), setting to 3"
 							exit_code=3
 						fi
 

{omniopt2-8471.data → omniopt2-9171.data}/data/bin/.helpers.py

@@ -10,7 +10,6 @@ try:
     import difflib
     import logging
     import os
-    import platform
     import re
     import traceback
     import numpy as np
@@ -227,12 +226,6 @@ def print_color(color: str, text: str) -> None:
         print(f"Error: {e}")
         print(text)
 
-def check_python_version() -> None:
-    python_version = platform.python_version()
-    supported_versions = ["3.8.10", "3.8.20", "3.9.21", "3.10.4", "3.10.12", "3.11.2", "3.11.9", "3.9.2", "3.11.3", "3.12.3", "3.12.4", "3.12.5", "3.12.6", "3.12.7", "3.12.7+", "3.12.8", "3.13.1", "3.13.2", "3.13.3", "3.13.5"]
-    if python_version not in supported_versions:
-        print_color("yellow", f"Warning: Supported python versions are {', '.join(supported_versions)}, but you are running {python_version}. This may or may not cause problems. Just is just a warning.")
-
 def create_widgets(_data: Any) -> Any:
     _plt, button, MAXIMUM_TEXTBOX, MINIMUM_TEXTBOX, _args, TEXTBOX_MINIMUM, TEXTBOX_MAXIMUM, update_graph = _data
 
@@ -882,5 +875,3 @@ def die_if_cannot_be_plotted(run_dir: Optional[str]) -> None:
     if not can_be_plotted(run_dir):
         log_error(f"{run_dir} contains multiple RESULTS and thus can only be plotted by parallel plot")
         sys.exit(2)
-
-check_python_version()