stouputils 1.14.3__py3-none-any.whl → 1.15.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- stouputils/data_science/config/get.py +51 -51
- stouputils/data_science/data_processing/image/__init__.py +66 -66
- stouputils/data_science/data_processing/image/auto_contrast.py +79 -79
- stouputils/data_science/data_processing/image/axis_flip.py +58 -58
- stouputils/data_science/data_processing/image/bias_field_correction.py +74 -74
- stouputils/data_science/data_processing/image/binary_threshold.py +73 -73
- stouputils/data_science/data_processing/image/blur.py +59 -59
- stouputils/data_science/data_processing/image/brightness.py +54 -54
- stouputils/data_science/data_processing/image/canny.py +110 -110
- stouputils/data_science/data_processing/image/clahe.py +92 -92
- stouputils/data_science/data_processing/image/common.py +30 -30
- stouputils/data_science/data_processing/image/contrast.py +53 -53
- stouputils/data_science/data_processing/image/curvature_flow_filter.py +74 -74
- stouputils/data_science/data_processing/image/denoise.py +378 -378
- stouputils/data_science/data_processing/image/histogram_equalization.py +123 -123
- stouputils/data_science/data_processing/image/invert.py +64 -64
- stouputils/data_science/data_processing/image/laplacian.py +60 -60
- stouputils/data_science/data_processing/image/median_blur.py +52 -52
- stouputils/data_science/data_processing/image/noise.py +59 -59
- stouputils/data_science/data_processing/image/normalize.py +65 -65
- stouputils/data_science/data_processing/image/random_erase.py +66 -66
- stouputils/data_science/data_processing/image/resize.py +69 -69
- stouputils/data_science/data_processing/image/rotation.py +80 -80
- stouputils/data_science/data_processing/image/salt_pepper.py +68 -68
- stouputils/data_science/data_processing/image/sharpening.py +55 -55
- stouputils/data_science/data_processing/image/shearing.py +64 -64
- stouputils/data_science/data_processing/image/threshold.py +64 -64
- stouputils/data_science/data_processing/image/translation.py +71 -71
- stouputils/data_science/data_processing/image/zoom.py +83 -83
- stouputils/data_science/data_processing/image_augmentation.py +118 -118
- stouputils/data_science/data_processing/image_preprocess.py +183 -183
- stouputils/data_science/data_processing/prosthesis_detection.py +359 -359
- stouputils/data_science/data_processing/technique.py +481 -481
- stouputils/data_science/dataset/__init__.py +45 -45
- stouputils/data_science/dataset/dataset.py +292 -292
- stouputils/data_science/dataset/dataset_loader.py +135 -135
- stouputils/data_science/dataset/grouping_strategy.py +296 -296
- stouputils/data_science/dataset/image_loader.py +100 -100
- stouputils/data_science/dataset/xy_tuple.py +696 -696
- stouputils/data_science/metric_dictionnary.py +106 -106
- stouputils/data_science/mlflow_utils.py +206 -206
- stouputils/data_science/models/abstract_model.py +149 -149
- stouputils/data_science/models/all.py +85 -85
- stouputils/data_science/models/keras/all.py +38 -38
- stouputils/data_science/models/keras/convnext.py +62 -62
- stouputils/data_science/models/keras/densenet.py +50 -50
- stouputils/data_science/models/keras/efficientnet.py +60 -60
- stouputils/data_science/models/keras/mobilenet.py +56 -56
- stouputils/data_science/models/keras/resnet.py +52 -52
- stouputils/data_science/models/keras/squeezenet.py +233 -233
- stouputils/data_science/models/keras/vgg.py +42 -42
- stouputils/data_science/models/keras/xception.py +38 -38
- stouputils/data_science/models/keras_utils/callbacks/__init__.py +20 -20
- stouputils/data_science/models/keras_utils/callbacks/colored_progress_bar.py +219 -219
- stouputils/data_science/models/keras_utils/callbacks/learning_rate_finder.py +148 -148
- stouputils/data_science/models/keras_utils/callbacks/model_checkpoint_v2.py +31 -31
- stouputils/data_science/models/keras_utils/callbacks/progressive_unfreezing.py +249 -249
- stouputils/data_science/models/keras_utils/callbacks/warmup_scheduler.py +66 -66
- stouputils/data_science/models/keras_utils/losses/__init__.py +12 -12
- stouputils/data_science/models/keras_utils/losses/next_generation_loss.py +56 -56
- stouputils/data_science/models/keras_utils/visualizations.py +416 -416
- stouputils/data_science/models/sandbox.py +116 -116
- stouputils/data_science/range_tuple.py +234 -234
- stouputils/data_science/utils.py +285 -285
- stouputils/decorators.py +53 -39
- stouputils/decorators.pyi +2 -2
- stouputils/installer/__init__.py +18 -18
- stouputils/installer/linux.py +144 -144
- stouputils/installer/main.py +223 -223
- stouputils/installer/windows.py +136 -136
- stouputils/io.py +16 -9
- stouputils/parallel.py +88 -2
- stouputils/parallel.pyi +21 -1
- stouputils/print.py +229 -2
- stouputils/print.pyi +90 -1
- stouputils/py.typed +1 -1
- {stouputils-1.14.3.dist-info → stouputils-1.15.1.dist-info}/METADATA +1 -1
- {stouputils-1.14.3.dist-info → stouputils-1.15.1.dist-info}/RECORD +80 -80
- {stouputils-1.14.3.dist-info → stouputils-1.15.1.dist-info}/WHEEL +1 -1
- {stouputils-1.14.3.dist-info → stouputils-1.15.1.dist-info}/entry_points.txt +0 -0
stouputils/installer/windows.py
CHANGED
|
@@ -1,136 +1,136 @@
|
|
|
1
|
-
""" Installer module for Windows specific functions.
|
|
2
|
-
|
|
3
|
-
Provides Windows specific implementations for checking administrator privileges,
|
|
4
|
-
determining appropriate installation paths (global/local), and modifying
|
|
5
|
-
the user's PATH environment variable.
|
|
6
|
-
"""
|
|
7
|
-
# Imports
|
|
8
|
-
import os
|
|
9
|
-
|
|
10
|
-
from ..decorators import LogLevels, handle_error
|
|
11
|
-
from ..io import clean_path
|
|
12
|
-
from ..print import debug, info, warning
|
|
13
|
-
from .common import ask_install_type, prompt_for_path
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
# Functions
|
|
17
|
-
@handle_error(message="Failed to add to PATH (Windows)", error_log=LogLevels.WARNING_TRACEBACK)
|
|
18
|
-
def add_to_path_windows(install_path: str) -> bool | None:
|
|
19
|
-
""" Add install_path to the User PATH environment variable on Windows.
|
|
20
|
-
|
|
21
|
-
Args:
|
|
22
|
-
install_path (str): The path to add to the User PATH environment variable.
|
|
23
|
-
|
|
24
|
-
Returns:
|
|
25
|
-
bool | None: True if the path was added to the User PATH environment variable, None otherwise.
|
|
26
|
-
"""
|
|
27
|
-
# Convert install_path to a Windows path if it's not already
|
|
28
|
-
install_path = install_path.replace("/", "\\")
|
|
29
|
-
os.makedirs(install_path, exist_ok=True)
|
|
30
|
-
|
|
31
|
-
# Get current user PATH
|
|
32
|
-
import winreg
|
|
33
|
-
with winreg.OpenKey(winreg.HKEY_CURRENT_USER, "Environment", 0, winreg.KEY_READ | winreg.KEY_WRITE) as key:
|
|
34
|
-
|
|
35
|
-
# Get the number of values in the registry key
|
|
36
|
-
num_values = winreg.QueryInfoKey(key)[1]
|
|
37
|
-
|
|
38
|
-
# Find the index of the 'Path' value
|
|
39
|
-
path_index = -1
|
|
40
|
-
for i in range(num_values):
|
|
41
|
-
if winreg.EnumValue(key, i)[0] == 'Path':
|
|
42
|
-
path_index = i
|
|
43
|
-
break
|
|
44
|
-
|
|
45
|
-
# Get the current path value
|
|
46
|
-
current_path: str = winreg.EnumValue(key, path_index)[1]
|
|
47
|
-
|
|
48
|
-
# Check if path is already present
|
|
49
|
-
if install_path not in current_path.split(';'):
|
|
50
|
-
new_path: str = f"{current_path};{install_path}"
|
|
51
|
-
winreg.SetValueEx(key, "Path", 0, winreg.REG_EXPAND_SZ, new_path)
|
|
52
|
-
debug(f"Added '{install_path}' to user PATH. Please restart your terminal for changes to take effect.")
|
|
53
|
-
else:
|
|
54
|
-
debug(f"'{install_path}' is already in user PATH.")
|
|
55
|
-
return True
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
def check_admin_windows() -> bool:
|
|
59
|
-
""" Check if the script is running with administrator privileges on Windows. """
|
|
60
|
-
try:
|
|
61
|
-
import ctypes
|
|
62
|
-
return ctypes.windll.shell32.IsUserAnAdmin() != 0
|
|
63
|
-
except Exception:
|
|
64
|
-
return False
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
@handle_error(message="Failed to get installation path (Windows)", error_log=LogLevels.ERROR_TRACEBACK)
|
|
68
|
-
def get_install_path_windows(
|
|
69
|
-
program_name: str,
|
|
70
|
-
ask_global: int = 0,
|
|
71
|
-
add_path: bool = True,
|
|
72
|
-
append_to_path: str = "",
|
|
73
|
-
default_global: str = os.environ.get("ProgramFiles", "C:\\Program Files")
|
|
74
|
-
) -> str:
|
|
75
|
-
""" Get the installation path for the program
|
|
76
|
-
|
|
77
|
-
Args:
|
|
78
|
-
program_name (str): The name of the program to install.
|
|
79
|
-
ask_global (int): 0 = ask for anything, 1 = install globally, 2 = install locally
|
|
80
|
-
add_path (bool): Whether to add the program to the PATH environment variable. (Only if installed globally)
|
|
81
|
-
append_to_path (str): String to append to the installation path when adding to PATH.
|
|
82
|
-
(ex: "bin" if executables are in the bin folder)
|
|
83
|
-
default_global (str): The default global installation path.
|
|
84
|
-
(Default is "C:\\Program Files" which is the most common location for executables on Windows)
|
|
85
|
-
|
|
86
|
-
Returns:
|
|
87
|
-
str: The installation path.
|
|
88
|
-
"""
|
|
89
|
-
# Default path is located in the current working directory
|
|
90
|
-
default_local_path: str = clean_path(os.path.join(os.getcwd(), program_name))
|
|
91
|
-
|
|
92
|
-
# Define default global path (used in prompt even if not chosen initially)
|
|
93
|
-
default_global_path: str = clean_path(os.path.join(default_global, program_name))
|
|
94
|
-
|
|
95
|
-
# Ask user for installation type (global/local)
|
|
96
|
-
install_type: str = ask_install_type(ask_global, default_local_path, default_global_path)
|
|
97
|
-
|
|
98
|
-
# If the user wants to install globally,
|
|
99
|
-
if install_type == 'g':
|
|
100
|
-
|
|
101
|
-
# Check if the user has admin privileges,
|
|
102
|
-
if not check_admin_windows():
|
|
103
|
-
|
|
104
|
-
# If the user doesn't have admin privileges, fallback to local
|
|
105
|
-
warning(
|
|
106
|
-
f"Global installation requires administrator privileges. Please re-run as administrator.\n"
|
|
107
|
-
f"Install locally instead to '{default_local_path}'? (Y/n): "
|
|
108
|
-
)
|
|
109
|
-
if input().lower() == 'n':
|
|
110
|
-
info("Installation cancelled.")
|
|
111
|
-
return ""
|
|
112
|
-
else:
|
|
113
|
-
# Fallback to local path if user agrees
|
|
114
|
-
return prompt_for_path(
|
|
115
|
-
f"Falling back to local installation path: {default_local_path}.",
|
|
116
|
-
default_local_path
|
|
117
|
-
)
|
|
118
|
-
|
|
119
|
-
# If the user has admin privileges,
|
|
120
|
-
else:
|
|
121
|
-
# Ask it user wants to override the default global install path
|
|
122
|
-
install_path: str = prompt_for_path(
|
|
123
|
-
f"Default global installation path is {default_global_path}.",
|
|
124
|
-
default_global_path
|
|
125
|
-
)
|
|
126
|
-
if add_path:
|
|
127
|
-
add_to_path_windows(os.path.join(install_path, append_to_path))
|
|
128
|
-
return install_path
|
|
129
|
-
|
|
130
|
-
# Local install
|
|
131
|
-
else: # install_type == 'l'
|
|
132
|
-
return prompt_for_path(
|
|
133
|
-
f"Default local installation path is {default_local_path}.",
|
|
134
|
-
default_local_path
|
|
135
|
-
)
|
|
136
|
-
|
|
1
|
+
""" Installer module for Windows specific functions.
|
|
2
|
+
|
|
3
|
+
Provides Windows specific implementations for checking administrator privileges,
|
|
4
|
+
determining appropriate installation paths (global/local), and modifying
|
|
5
|
+
the user's PATH environment variable.
|
|
6
|
+
"""
|
|
7
|
+
# Imports
|
|
8
|
+
import os
|
|
9
|
+
|
|
10
|
+
from ..decorators import LogLevels, handle_error
|
|
11
|
+
from ..io import clean_path
|
|
12
|
+
from ..print import debug, info, warning
|
|
13
|
+
from .common import ask_install_type, prompt_for_path
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
# Functions
|
|
17
|
+
@handle_error(message="Failed to add to PATH (Windows)", error_log=LogLevels.WARNING_TRACEBACK)
|
|
18
|
+
def add_to_path_windows(install_path: str) -> bool | None:
|
|
19
|
+
""" Add install_path to the User PATH environment variable on Windows.
|
|
20
|
+
|
|
21
|
+
Args:
|
|
22
|
+
install_path (str): The path to add to the User PATH environment variable.
|
|
23
|
+
|
|
24
|
+
Returns:
|
|
25
|
+
bool | None: True if the path was added to the User PATH environment variable, None otherwise.
|
|
26
|
+
"""
|
|
27
|
+
# Convert install_path to a Windows path if it's not already
|
|
28
|
+
install_path = install_path.replace("/", "\\")
|
|
29
|
+
os.makedirs(install_path, exist_ok=True)
|
|
30
|
+
|
|
31
|
+
# Get current user PATH
|
|
32
|
+
import winreg
|
|
33
|
+
with winreg.OpenKey(winreg.HKEY_CURRENT_USER, "Environment", 0, winreg.KEY_READ | winreg.KEY_WRITE) as key:
|
|
34
|
+
|
|
35
|
+
# Get the number of values in the registry key
|
|
36
|
+
num_values = winreg.QueryInfoKey(key)[1]
|
|
37
|
+
|
|
38
|
+
# Find the index of the 'Path' value
|
|
39
|
+
path_index = -1
|
|
40
|
+
for i in range(num_values):
|
|
41
|
+
if winreg.EnumValue(key, i)[0] == 'Path':
|
|
42
|
+
path_index = i
|
|
43
|
+
break
|
|
44
|
+
|
|
45
|
+
# Get the current path value
|
|
46
|
+
current_path: str = winreg.EnumValue(key, path_index)[1]
|
|
47
|
+
|
|
48
|
+
# Check if path is already present
|
|
49
|
+
if install_path not in current_path.split(';'):
|
|
50
|
+
new_path: str = f"{current_path};{install_path}"
|
|
51
|
+
winreg.SetValueEx(key, "Path", 0, winreg.REG_EXPAND_SZ, new_path)
|
|
52
|
+
debug(f"Added '{install_path}' to user PATH. Please restart your terminal for changes to take effect.")
|
|
53
|
+
else:
|
|
54
|
+
debug(f"'{install_path}' is already in user PATH.")
|
|
55
|
+
return True
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def check_admin_windows() -> bool:
|
|
59
|
+
""" Check if the script is running with administrator privileges on Windows. """
|
|
60
|
+
try:
|
|
61
|
+
import ctypes
|
|
62
|
+
return ctypes.windll.shell32.IsUserAnAdmin() != 0
|
|
63
|
+
except Exception:
|
|
64
|
+
return False
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
@handle_error(message="Failed to get installation path (Windows)", error_log=LogLevels.ERROR_TRACEBACK)
|
|
68
|
+
def get_install_path_windows(
|
|
69
|
+
program_name: str,
|
|
70
|
+
ask_global: int = 0,
|
|
71
|
+
add_path: bool = True,
|
|
72
|
+
append_to_path: str = "",
|
|
73
|
+
default_global: str = os.environ.get("ProgramFiles", "C:\\Program Files")
|
|
74
|
+
) -> str:
|
|
75
|
+
""" Get the installation path for the program
|
|
76
|
+
|
|
77
|
+
Args:
|
|
78
|
+
program_name (str): The name of the program to install.
|
|
79
|
+
ask_global (int): 0 = ask for anything, 1 = install globally, 2 = install locally
|
|
80
|
+
add_path (bool): Whether to add the program to the PATH environment variable. (Only if installed globally)
|
|
81
|
+
append_to_path (str): String to append to the installation path when adding to PATH.
|
|
82
|
+
(ex: "bin" if executables are in the bin folder)
|
|
83
|
+
default_global (str): The default global installation path.
|
|
84
|
+
(Default is "C:\\Program Files" which is the most common location for executables on Windows)
|
|
85
|
+
|
|
86
|
+
Returns:
|
|
87
|
+
str: The installation path.
|
|
88
|
+
"""
|
|
89
|
+
# Default path is located in the current working directory
|
|
90
|
+
default_local_path: str = clean_path(os.path.join(os.getcwd(), program_name))
|
|
91
|
+
|
|
92
|
+
# Define default global path (used in prompt even if not chosen initially)
|
|
93
|
+
default_global_path: str = clean_path(os.path.join(default_global, program_name))
|
|
94
|
+
|
|
95
|
+
# Ask user for installation type (global/local)
|
|
96
|
+
install_type: str = ask_install_type(ask_global, default_local_path, default_global_path)
|
|
97
|
+
|
|
98
|
+
# If the user wants to install globally,
|
|
99
|
+
if install_type == 'g':
|
|
100
|
+
|
|
101
|
+
# Check if the user has admin privileges,
|
|
102
|
+
if not check_admin_windows():
|
|
103
|
+
|
|
104
|
+
# If the user doesn't have admin privileges, fallback to local
|
|
105
|
+
warning(
|
|
106
|
+
f"Global installation requires administrator privileges. Please re-run as administrator.\n"
|
|
107
|
+
f"Install locally instead to '{default_local_path}'? (Y/n): "
|
|
108
|
+
)
|
|
109
|
+
if input().lower() == 'n':
|
|
110
|
+
info("Installation cancelled.")
|
|
111
|
+
return ""
|
|
112
|
+
else:
|
|
113
|
+
# Fallback to local path if user agrees
|
|
114
|
+
return prompt_for_path(
|
|
115
|
+
f"Falling back to local installation path: {default_local_path}.",
|
|
116
|
+
default_local_path
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
# If the user has admin privileges,
|
|
120
|
+
else:
|
|
121
|
+
# Ask it user wants to override the default global install path
|
|
122
|
+
install_path: str = prompt_for_path(
|
|
123
|
+
f"Default global installation path is {default_global_path}.",
|
|
124
|
+
default_global_path
|
|
125
|
+
)
|
|
126
|
+
if add_path:
|
|
127
|
+
add_to_path_windows(os.path.join(install_path, append_to_path))
|
|
128
|
+
return install_path
|
|
129
|
+
|
|
130
|
+
# Local install
|
|
131
|
+
else: # install_type == 'l'
|
|
132
|
+
return prompt_for_path(
|
|
133
|
+
f"Default local installation path is {default_local_path}.",
|
|
134
|
+
default_local_path
|
|
135
|
+
)
|
|
136
|
+
|
stouputils/io.py
CHANGED
|
@@ -188,16 +188,23 @@ def csv_dump(
|
|
|
188
188
|
done: bool = False
|
|
189
189
|
|
|
190
190
|
# Handle Polars DataFrame
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
copy_kwargs.setdefault("include_header", has_header)
|
|
197
|
-
data.write_csv(output, *args, **copy_kwargs)
|
|
198
|
-
done = True
|
|
199
|
-
except Exception:
|
|
191
|
+
import sys
|
|
192
|
+
if sys.version_info >= (3, 14) and not sys._is_gil_enabled(): # pyright: ignore[reportPrivateUsage]
|
|
193
|
+
# Skip Polars on free-threaded Python 3.14 due to segfault
|
|
194
|
+
# TODO: Remove this check when Polars is fixed
|
|
195
|
+
# See https://github.com/pola-rs/polars/issues/21889 and https://github.com/durandtibo/coola/issues/1066
|
|
200
196
|
pass
|
|
197
|
+
else:
|
|
198
|
+
try:
|
|
199
|
+
import polars as pl # type: ignore
|
|
200
|
+
if isinstance(data, pl.DataFrame):
|
|
201
|
+
copy_kwargs = kwargs.copy()
|
|
202
|
+
copy_kwargs.setdefault("separator", delimiter)
|
|
203
|
+
copy_kwargs.setdefault("include_header", has_header)
|
|
204
|
+
data.write_csv(output, *args, **copy_kwargs)
|
|
205
|
+
done = True
|
|
206
|
+
except Exception:
|
|
207
|
+
pass
|
|
201
208
|
|
|
202
209
|
# Handle pandas DataFrame
|
|
203
210
|
if not done:
|
stouputils/parallel.py
CHANGED
|
@@ -7,6 +7,16 @@ This module provides utility functions for parallel processing, such as:
|
|
|
7
7
|
|
|
8
8
|
I highly encourage you to read the function docstrings to understand when to use each method.
|
|
9
9
|
|
|
10
|
+
Priority (nice) mapping for multiprocessing():
|
|
11
|
+
|
|
12
|
+
- Unix-style values from -20 (highest priority) to 19 (lowest priority)
|
|
13
|
+
- Windows automatic mapping:
|
|
14
|
+
* -20 to -10: HIGH_PRIORITY_CLASS
|
|
15
|
+
* -9 to -1: ABOVE_NORMAL_PRIORITY_CLASS
|
|
16
|
+
* 0: NORMAL_PRIORITY_CLASS
|
|
17
|
+
* 1 to 9: BELOW_NORMAL_PRIORITY_CLASS
|
|
18
|
+
* 10 to 19: IDLE_PRIORITY_CLASS
|
|
19
|
+
|
|
10
20
|
.. image:: https://raw.githubusercontent.com/Stoupy51/stouputils/refs/heads/main/assets/parallel_module.gif
|
|
11
21
|
:alt: stouputils parallel examples
|
|
12
22
|
"""
|
|
@@ -42,6 +52,7 @@ def multiprocessing[T, R](
|
|
|
42
52
|
desc: str = "",
|
|
43
53
|
max_workers: int | float = CPU_COUNT,
|
|
44
54
|
delay_first_calls: float = 0,
|
|
55
|
+
nice: int | None = None,
|
|
45
56
|
color: str = MAGENTA,
|
|
46
57
|
bar_format: str = BAR_FORMAT,
|
|
47
58
|
ascii: bool = False,
|
|
@@ -69,6 +80,11 @@ def multiprocessing[T, R](
|
|
|
69
80
|
delay_first_calls (float): Apply i*delay_first_calls seconds delay to the first "max_workers" calls.
|
|
70
81
|
For instance, the first process will be delayed by 0 seconds, the second by 1 second, etc.
|
|
71
82
|
(Defaults to 0): This can be useful to avoid functions being called in the same second.
|
|
83
|
+
nice (int | None): Adjust the priority of worker processes (Defaults to None).
|
|
84
|
+
Use Unix-style values: -20 (highest priority) to 19 (lowest priority).
|
|
85
|
+
Positive values reduce priority, negative values increase it.
|
|
86
|
+
Automatically converted to appropriate priority class on Windows.
|
|
87
|
+
If None, no priority adjustment is made.
|
|
72
88
|
color (str): Color of the progress bar (Defaults to MAGENTA)
|
|
73
89
|
bar_format (str): Format of the progress bar (Defaults to BAR_FORMAT)
|
|
74
90
|
ascii (bool): Whether to use ASCII or Unicode characters for the progress bar
|
|
@@ -140,13 +156,21 @@ def multiprocessing[T, R](
|
|
|
140
156
|
# Do multiprocessing only if there is more than 1 argument and more than 1 CPU
|
|
141
157
|
if max_workers > 1 and len(args) > 1:
|
|
142
158
|
def process() -> list[Any]:
|
|
159
|
+
# Wrap function with nice if specified
|
|
160
|
+
if nice is not None:
|
|
161
|
+
wrapped_args = [(nice, func, arg) for arg in args]
|
|
162
|
+
wrapped_func = _nice_wrapper
|
|
163
|
+
else:
|
|
164
|
+
wrapped_args = args
|
|
165
|
+
wrapped_func = func
|
|
166
|
+
|
|
143
167
|
if verbose:
|
|
144
168
|
return list(process_map(
|
|
145
|
-
|
|
169
|
+
wrapped_func, wrapped_args, max_workers=max_workers, chunksize=chunksize, desc=desc, bar_format=bar_format, ascii=ascii, **tqdm_kwargs
|
|
146
170
|
)) # type: ignore
|
|
147
171
|
else:
|
|
148
172
|
with Pool(max_workers) as pool:
|
|
149
|
-
return list(pool.map(
|
|
173
|
+
return list(pool.map(wrapped_func, wrapped_args, chunksize=chunksize)) # type: ignore
|
|
150
174
|
try:
|
|
151
175
|
return process()
|
|
152
176
|
except RuntimeError as e:
|
|
@@ -382,6 +406,68 @@ def run_in_subprocess[R](
|
|
|
382
406
|
raise RuntimeError("Subprocess did not return any result") from e
|
|
383
407
|
|
|
384
408
|
|
|
409
|
+
# "Private" function to wrap function execution with nice priority (must be at module level for pickling)
|
|
410
|
+
def _nice_wrapper[T, R](args: tuple[int, Callable[[T], R], T]) -> R:
|
|
411
|
+
""" Wrapper that applies nice priority then executes the function.
|
|
412
|
+
|
|
413
|
+
Args:
|
|
414
|
+
args (tuple): Tuple containing (nice_value, func, arg)
|
|
415
|
+
|
|
416
|
+
Returns:
|
|
417
|
+
R: Result of the function execution
|
|
418
|
+
"""
|
|
419
|
+
nice_value, func, arg = args
|
|
420
|
+
_set_process_priority(nice_value)
|
|
421
|
+
return func(arg)
|
|
422
|
+
|
|
423
|
+
# "Private" function to set process priority (must be at module level for pickling on Windows)
|
|
424
|
+
def _set_process_priority(nice_value: int) -> None:
|
|
425
|
+
""" Set the priority of the current process.
|
|
426
|
+
|
|
427
|
+
Args:
|
|
428
|
+
nice_value (int): Unix-style priority value (-20 to 19)
|
|
429
|
+
"""
|
|
430
|
+
try:
|
|
431
|
+
import sys
|
|
432
|
+
if sys.platform == "win32":
|
|
433
|
+
# Map Unix nice values to Windows priority classes
|
|
434
|
+
# -20 to -10: HIGH, -9 to -1: ABOVE_NORMAL, 0: NORMAL, 1-9: BELOW_NORMAL, 10-19: IDLE
|
|
435
|
+
try:
|
|
436
|
+
import psutil
|
|
437
|
+
if nice_value <= -10:
|
|
438
|
+
priority = psutil.HIGH_PRIORITY_CLASS
|
|
439
|
+
elif nice_value < 0:
|
|
440
|
+
priority = psutil.ABOVE_NORMAL_PRIORITY_CLASS
|
|
441
|
+
elif nice_value == 0:
|
|
442
|
+
priority = psutil.NORMAL_PRIORITY_CLASS
|
|
443
|
+
elif nice_value < 10:
|
|
444
|
+
priority = psutil.BELOW_NORMAL_PRIORITY_CLASS
|
|
445
|
+
else:
|
|
446
|
+
priority = psutil.IDLE_PRIORITY_CLASS
|
|
447
|
+
psutil.Process().nice(priority)
|
|
448
|
+
except ImportError:
|
|
449
|
+
# Fallback to ctypes if psutil is not available
|
|
450
|
+
import ctypes
|
|
451
|
+
# Windows priority class constants
|
|
452
|
+
if nice_value <= -10:
|
|
453
|
+
priority = 0x00000080 # HIGH_PRIORITY_CLASS
|
|
454
|
+
elif nice_value < 0:
|
|
455
|
+
priority = 0x00008000 # ABOVE_NORMAL_PRIORITY_CLASS
|
|
456
|
+
elif nice_value == 0:
|
|
457
|
+
priority = 0x00000020 # NORMAL_PRIORITY_CLASS
|
|
458
|
+
elif nice_value < 10:
|
|
459
|
+
priority = 0x00004000 # BELOW_NORMAL_PRIORITY_CLASS
|
|
460
|
+
else:
|
|
461
|
+
priority = 0x00000040 # IDLE_PRIORITY_CLASS
|
|
462
|
+
kernel32 = ctypes.windll.kernel32
|
|
463
|
+
handle = kernel32.GetCurrentProcess()
|
|
464
|
+
kernel32.SetPriorityClass(handle, priority)
|
|
465
|
+
else:
|
|
466
|
+
# Unix-like systems
|
|
467
|
+
os.nice(nice_value)
|
|
468
|
+
except Exception:
|
|
469
|
+
pass # Silently ignore if we can't set priority
|
|
470
|
+
|
|
385
471
|
# "Private" function for subprocess wrapper (must be at module level for pickling on Windows)
|
|
386
472
|
def _subprocess_wrapper[R](
|
|
387
473
|
result_queue: Any,
|
stouputils/parallel.pyi
CHANGED
|
@@ -10,7 +10,7 @@ CPU_COUNT: int
|
|
|
10
10
|
T = TypeVar('T')
|
|
11
11
|
R = TypeVar('R')
|
|
12
12
|
|
|
13
|
-
def multiprocessing[T, R](func: Callable[..., R] | list[Callable[..., R]], args: Iterable[T], use_starmap: bool = False, chunksize: int = 1, desc: str = '', max_workers: int | float = ..., delay_first_calls: float = 0, color: str = ..., bar_format: str = ..., ascii: bool = False, smooth_tqdm: bool = True, **tqdm_kwargs: Any) -> list[R]:
|
|
13
|
+
def multiprocessing[T, R](func: Callable[..., R] | list[Callable[..., R]], args: Iterable[T], use_starmap: bool = False, chunksize: int = 1, desc: str = '', max_workers: int | float = ..., delay_first_calls: float = 0, nice: int | None = None, color: str = ..., bar_format: str = ..., ascii: bool = False, smooth_tqdm: bool = True, **tqdm_kwargs: Any) -> list[R]:
|
|
14
14
|
''' Method to execute a function in parallel using multiprocessing
|
|
15
15
|
|
|
16
16
|
\t- For CPU-bound operations where the GIL (Global Interpreter Lock) is a bottleneck.
|
|
@@ -32,6 +32,11 @@ def multiprocessing[T, R](func: Callable[..., R] | list[Callable[..., R]], args:
|
|
|
32
32
|
\t\tdelay_first_calls\t(float):\t\t\tApply i*delay_first_calls seconds delay to the first "max_workers" calls.
|
|
33
33
|
\t\t\tFor instance, the first process will be delayed by 0 seconds, the second by 1 second, etc.
|
|
34
34
|
\t\t\t(Defaults to 0): This can be useful to avoid functions being called in the same second.
|
|
35
|
+
\t\tnice\t\t\t\t(int | None):\t\tAdjust the priority of worker processes (Defaults to None).
|
|
36
|
+
\t\t\tUse Unix-style values: -20 (highest priority) to 19 (lowest priority).
|
|
37
|
+
\t\t\tPositive values reduce priority, negative values increase it.
|
|
38
|
+
\t\t\tAutomatically converted to appropriate priority class on Windows.
|
|
39
|
+
\t\t\tIf None, no priority adjustment is made.
|
|
35
40
|
\t\tcolor\t\t\t\t(str):\t\t\t\tColor of the progress bar (Defaults to MAGENTA)
|
|
36
41
|
\t\tbar_format\t\t\t(str):\t\t\t\tFormat of the progress bar (Defaults to BAR_FORMAT)
|
|
37
42
|
\t\tascii\t\t\t\t(bool):\t\t\t\tWhether to use ASCII or Unicode characters for the progress bar
|
|
@@ -169,6 +174,21 @@ def run_in_subprocess[R](func: Callable[..., R], *args: Any, timeout: float | No
|
|
|
169
174
|
\t\t\t> # With timeout to prevent hanging
|
|
170
175
|
\t\t\t> run_in_subprocess(some_gpu_func, data, timeout=300.0)
|
|
171
176
|
\t'''
|
|
177
|
+
def _nice_wrapper[T, R](args: tuple[int, Callable[[T], R], T]) -> R:
|
|
178
|
+
""" Wrapper that applies nice priority then executes the function.
|
|
179
|
+
|
|
180
|
+
\tArgs:
|
|
181
|
+
\t\targs (tuple): Tuple containing (nice_value, func, arg)
|
|
182
|
+
|
|
183
|
+
\tReturns:
|
|
184
|
+
\t\tR: Result of the function execution
|
|
185
|
+
\t"""
|
|
186
|
+
def _set_process_priority(nice_value: int) -> None:
|
|
187
|
+
""" Set the priority of the current process.
|
|
188
|
+
|
|
189
|
+
\tArgs:
|
|
190
|
+
\t\tnice_value (int): Unix-style priority value (-20 to 19)
|
|
191
|
+
\t"""
|
|
172
192
|
def _subprocess_wrapper[R](result_queue: Any, func: Callable[..., R], args: tuple[Any, ...], kwargs: dict[str, Any]) -> None:
|
|
173
193
|
""" Wrapper function to execute the target function and store the result in the queue.
|
|
174
194
|
|