stouputils 1.17.0__py3-none-any.whl → 1.18.1__py3-none-any.whl

stouputils/parallel/common.pyi

@@ -0,0 +1,53 @@
+from collections.abc import Callable
+
+CPU_COUNT: int
+
+def nice_wrapper[T, R](args: tuple[int, Callable[[T], R], T]) -> R:
+    """ Wrapper that applies nice priority then executes the function.
+
+\tArgs:
+\t\targs (tuple): Tuple containing (nice_value, func, arg)
+
+\tReturns:
+\t\tR: Result of the function execution
+\t"""
+def set_process_priority(nice_value: int) -> None:
+    """ Set the priority of the current process.
+
+\tArgs:
+\t\tnice_value (int): Unix-style priority value (-20 to 19)
+\t"""
+def starmap[T, R](args: tuple[Callable[[T], R], list[T]]) -> R:
+    """ Private function to use starmap using args[0](\\*args[1])
+
+\tArgs:
+\t\targs (tuple): Tuple containing the function and the arguments list to pass to the function
+\tReturns:
+\t\tobject: Result of the function execution
+\t"""
+def delayed_call[T, R](args: tuple[Callable[[T], R], float, T]) -> R:
+    """ Private function to apply delay before calling the target function
+
+\tArgs:
+\t\targs (tuple): Tuple containing the function, delay in seconds, and the argument to pass to the function
+\tReturns:
+\t\tobject: Result of the function execution
+\t"""
+def handle_parameters[T, R](func: Callable[[T], R] | list[Callable[[T], R]], args: list[T], use_starmap: bool, delay_first_calls: float, max_workers: int, desc: str, color: str) -> tuple[str, Callable[[T], R], list[T]]:
+    ''' Private function to handle the parameters for multiprocessing or multithreading functions
+
+\tArgs:
+\t\tfunc\t\t\t\t(Callable | list[Callable]):\tFunction to execute, or list of functions (one per argument)
+\t\targs\t\t\t\t(list):\t\t\t\tList of arguments to pass to the function(s)
+\t\tuse_starmap\t\t\t(bool):\t\t\t\tWhether to use starmap or not (Defaults to False):
+\t\t\tTrue means the function will be called like func(\\*args[i]) instead of func(args[i])
+\t\tdelay_first_calls\t(int):\t\t\t\tApply i*delay_first_calls seconds delay to the first "max_workers" calls.
+\t\t\tFor instance, the first process will be delayed by 0 seconds, the second by 1 second, etc. (Defaults to 0):
+\t\t\tThis can be useful to avoid functions being called in the same second.
+\t\tmax_workers\t\t\t(int):\t\t\t\tNumber of workers to use
+\t\tdesc\t\t\t\t(str):\t\t\t\tDescription of the function execution displayed in the progress bar
+\t\tcolor\t\t\t\t(str):\t\t\t\tColor of the progress bar
+
+\tReturns:
+\t\ttuple[str, Callable[[T], R], list[T]]:\tTuple containing the description, function, and arguments
+\t'''

stouputils/parallel/multi.py

@@ -0,0 +1,309 @@
+
+# Imports
+import time
+from collections.abc import Callable, Iterable
+from typing import Any
+
+from ..ctx import SetMPStartMethod
+from ..print import BAR_FORMAT, MAGENTA
+from .capturer import CaptureOutput
+from .common import CPU_COUNT, handle_parameters, nice_wrapper
+
+
+# Small test functions for doctests
+def doctest_square(x: int) -> int:
+	return x * x
+def doctest_slow(x: int) -> int:
+	time.sleep(0.1)
+	return x
+
+# Functions
+def multiprocessing[T, R](
+	func: Callable[..., R] | list[Callable[..., R]],
+	args: Iterable[T],
+	use_starmap: bool = False,
+	chunksize: int = 1,
+	desc: str = "",
+	max_workers: int | float = CPU_COUNT,
+	capture_output: bool = False,
+	delay_first_calls: float = 0,
+	nice: int | None = None,
+	color: str = MAGENTA,
+	bar_format: str = BAR_FORMAT,
+	ascii: bool = False,
+	smooth_tqdm: bool = True,
+	**tqdm_kwargs: Any
+) -> list[R]:
+	r""" Method to execute a function in parallel using multiprocessing
+
+	- For CPU-bound operations where the GIL (Global Interpreter Lock) is a bottleneck.
+	- When the task can be divided into smaller, independent sub-tasks that can be executed concurrently.
+	- For computationally intensive tasks like scientific simulations, data analysis, or machine learning workloads.
+
+	Args:
+		func				(Callable | list[Callable]):	Function to execute, or list of functions (one per argument)
+		args				(Iterable):			Iterable of arguments to pass to the function(s)
+		use_starmap			(bool):				Whether to use starmap or not (Defaults to False):
+			True means the function will be called like func(\*args[i]) instead of func(args[i])
+		chunksize			(int):				Number of arguments to process at a time
+			(Defaults to 1 for proper progress bar display)
+		desc				(str):				Description displayed in the progress bar
+			(if not provided no progress bar will be displayed)
+		max_workers			(int | float):		Number of workers to use (Defaults to CPU_COUNT), -1 means CPU_COUNT.
+			If float between 0 and 1, it's treated as a percentage of CPU_COUNT.
+			If negative float between -1 and 0, it's treated as a percentage of len(args).
+		capture_output		(bool):				Whether to capture stdout/stderr from the worker processes (Defaults to True)
+		delay_first_calls	(float):			Apply i*delay_first_calls seconds delay to the first "max_workers" calls.
+			For instance, the first process will be delayed by 0 seconds, the second by 1 second, etc.
+			(Defaults to 0): This can be useful to avoid functions being called in the same second.
+		nice				(int | None):		Adjust the priority of worker processes (Defaults to None).
+			Use Unix-style values: -20 (highest priority) to 19 (lowest priority).
+			Positive values reduce priority, negative values increase it.
+			Automatically converted to appropriate priority class on Windows.
+			If None, no priority adjustment is made.
+		color				(str):				Color of the progress bar (Defaults to MAGENTA)
+		bar_format			(str):				Format of the progress bar (Defaults to BAR_FORMAT)
+		ascii				(bool):				Whether to use ASCII or Unicode characters for the progress bar
+		smooth_tqdm			(bool):				Whether to enable smooth progress bar updates by setting miniters and mininterval (Defaults to True)
+		**tqdm_kwargs		(Any):				Additional keyword arguments to pass to tqdm
+
+	Returns:
+		list[object]:	Results of the function execution
+
+	Examples:
+		.. code-block:: python
+
+			> multiprocessing(doctest_square, args=[1, 2, 3])
+			[1, 4, 9]
+
+			> multiprocessing(int.__mul__, [(1,2), (3,4), (5,6)], use_starmap=True)
+			[2, 12, 30]
+
+			> # Using a list of functions (one per argument)
+			> multiprocessing([doctest_square, doctest_square, doctest_square], [1, 2, 3])
+			[1, 4, 9]
+
+			> # Will process in parallel with progress bar
+			> multiprocessing(doctest_slow, range(10), desc="Processing")
+			[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+
+			> # Will process in parallel with progress bar and delay the first threads
+			> multiprocessing(
+			.     doctest_slow,
+			.     range(10),
+			.     desc="Processing with delay",
+			.     max_workers=2,
+			.     delay_first_calls=0.6
+			. )
+			[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+	"""
+	# Imports
+	import multiprocessing as mp
+	from multiprocessing import Pool
+
+	from tqdm.auto import tqdm
+	from tqdm.contrib.concurrent import process_map  # pyright: ignore[reportUnknownVariableType]
+
+	# Handle parameters
+	args = list(args)  # Ensure we have a list (not other iterable)
+	if max_workers == -1:
+		max_workers = CPU_COUNT
+	if isinstance(max_workers, float):
+		if max_workers > 0:
+			assert max_workers <= 1, "max_workers as positive float must be between 0 and 1 (percentage of CPU_COUNT)"
+			max_workers = int(max_workers * CPU_COUNT)
+		else:
+			assert -1 <= max_workers < 0, "max_workers as negative float must be between -1 and 0 (percentage of len(args))"
+			max_workers = int(-max_workers * len(args))
+	verbose: bool = desc != ""
+	desc, func, args = handle_parameters(func, args, use_starmap, delay_first_calls, max_workers, desc, color)
+	if bar_format == BAR_FORMAT:
+		bar_format = bar_format.replace(MAGENTA, color)
+	if smooth_tqdm:
+		tqdm_kwargs.setdefault("mininterval", 0.0)
+		try:
+			total = len(args) # type: ignore
+			import shutil
+			width = shutil.get_terminal_size().columns
+			tqdm_kwargs.setdefault("miniters", max(1, total // width))
+		except (TypeError, OSError):
+			tqdm_kwargs.setdefault("miniters", 1)
+
+	# Do multiprocessing only if there is more than 1 argument and more than 1 CPU
+	if max_workers > 1 and len(args) > 1:
+		# Wrap function with nice if specified
+		if nice is not None:
+			wrapped_args = [(nice, func, arg) for arg in args]
+			wrapped_func = nice_wrapper
+		else:
+			wrapped_args = args
+			wrapped_func = func
+
+		# Capture output if specified
+		capturer: CaptureOutput | None = None
+		if capture_output:
+			capturer = CaptureOutput()
+			capturer.start_listener()
+			wrapped_args = [(capturer, wrapped_func, arg) for arg in wrapped_args]
+			wrapped_func = capture_subprocess_output
+
+		def process() -> list[Any]:
+			if verbose:
+				return list(process_map(
+					wrapped_func, wrapped_args, max_workers=max_workers, chunksize=chunksize, desc=desc, bar_format=bar_format, ascii=ascii, **tqdm_kwargs
+				)) # type: ignore
+			else:
+				with Pool(max_workers) as pool:
+					return list(pool.map(wrapped_func, wrapped_args, chunksize=chunksize))	# type: ignore
+		try:
+			return process()
+		except RuntimeError as e:
+			if "SemLock created in a fork context is being shared with a process in a spawn context" in str(e):
+
+				# Try with alternate start method
+				with SetMPStartMethod("spawn" if mp.get_start_method() != "spawn" else "fork"):
+					return process()
+			else: # Re-raise if it's not the SemLock error
+				raise
+		finally:
+			if capturer is not None:
+				capturer.parent_close_write()
+				capturer.join_listener(timeout=5.0)
+
+	# Single process execution
+	else:
+		if verbose:
+			return [func(arg) for arg in tqdm(args, total=len(args), desc=desc, bar_format=bar_format, ascii=ascii, **tqdm_kwargs)]
+		else:
+			return [func(arg) for arg in args]
+
+
+def multithreading[T, R](
+	func: Callable[..., R] | list[Callable[..., R]],
+	args: Iterable[T],
+	use_starmap: bool = False,
+	desc: str = "",
+	max_workers: int | float = CPU_COUNT,
+	delay_first_calls: float = 0,
+	color: str = MAGENTA,
+	bar_format: str = BAR_FORMAT,
+	ascii: bool = False,
+	smooth_tqdm: bool = True,
+	**tqdm_kwargs: Any
+	) -> list[R]:
+	r""" Method to execute a function in parallel using multithreading, you should use it:
+
+	- For I/O-bound operations where the GIL is not a bottleneck, such as network requests or disk operations.
+	- When the task involves waiting for external resources, such as network responses or user input.
+	- For operations that involve a lot of waiting, such as GUI event handling or handling user input.
+
+	Args:
+		func				(Callable | list[Callable]):	Function to execute, or list of functions (one per argument)
+		args				(Iterable):			Iterable of arguments to pass to the function(s)
+		use_starmap			(bool):				Whether to use starmap or not (Defaults to False):
+			True means the function will be called like func(\*args[i]) instead of func(args[i])
+		desc				(str):				Description displayed in the progress bar
+			(if not provided no progress bar will be displayed)
+		max_workers			(int | float):		Number of workers to use (Defaults to CPU_COUNT), -1 means CPU_COUNT.
+			If float between 0 and 1, it's treated as a percentage of CPU_COUNT.
+			If negative float between -1 and 0, it's treated as a percentage of len(args).
+		delay_first_calls	(float):			Apply i*delay_first_calls seconds delay to the first "max_workers" calls.
+			For instance with value to 1, the first thread will be delayed by 0 seconds, the second by 1 second, etc.
+			(Defaults to 0): This can be useful to avoid functions being called in the same second.
+		color				(str):				Color of the progress bar (Defaults to MAGENTA)
+		bar_format			(str):				Format of the progress bar (Defaults to BAR_FORMAT)
+		ascii				(bool):				Whether to use ASCII or Unicode characters for the progress bar
+		smooth_tqdm			(bool):				Whether to enable smooth progress bar updates by setting miniters and mininterval (Defaults to True)
+		**tqdm_kwargs		(Any):				Additional keyword arguments to pass to tqdm
+
+	Returns:
+		list[object]:	Results of the function execution
+
+	Examples:
+		.. code-block:: python
+
+			> multithreading(doctest_square, args=[1, 2, 3])
+			[1, 4, 9]
+
+			> multithreading(int.__mul__, [(1,2), (3,4), (5,6)], use_starmap=True)
+			[2, 12, 30]
+
+			> # Using a list of functions (one per argument)
+			> multithreading([doctest_square, doctest_square, doctest_square], [1, 2, 3])
+			[1, 4, 9]
+
+			> # Will process in parallel with progress bar
+			> multithreading(doctest_slow, range(10), desc="Threading")
+			[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+
+			> # Will process in parallel with progress bar and delay the first threads
+			> multithreading(
+			.     doctest_slow,
+			.     range(10),
+			.     desc="Threading with delay",
+			.     max_workers=2,
+			.     delay_first_calls=0.6
+			. )
+			[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+	"""
+	# Imports
+	from concurrent.futures import ThreadPoolExecutor
+
+	from tqdm.auto import tqdm
+
+	# Handle parameters
+	args = list(args)  # Ensure we have a list (not other iterable)
+	if max_workers == -1:
+		max_workers = CPU_COUNT
+	if isinstance(max_workers, float):
+		if max_workers > 0:
+			assert max_workers <= 1, "max_workers as positive float must be between 0 and 1 (percentage of CPU_COUNT)"
+			max_workers = int(max_workers * CPU_COUNT)
+		else:
+			assert -1 <= max_workers < 0, "max_workers as negative float must be between -1 and 0 (percentage of len(args))"
+			max_workers = int(-max_workers * len(args))
+	verbose: bool = desc != ""
+	desc, func, args = handle_parameters(func, args, use_starmap, delay_first_calls, max_workers, desc, color)
+	if bar_format == BAR_FORMAT:
+		bar_format = bar_format.replace(MAGENTA, color)
+	if smooth_tqdm:
+		tqdm_kwargs.setdefault("mininterval", 0.0)
+		try:
+			total = len(args) # type: ignore
+			import shutil
+			width = shutil.get_terminal_size().columns
+			tqdm_kwargs.setdefault("miniters", max(1, total // width))
+		except (TypeError, OSError):
+			tqdm_kwargs.setdefault("miniters", 1)
+
+	# Do multithreading only if there is more than 1 argument and more than 1 CPU
+	if max_workers > 1 and len(args) > 1:
+		if verbose:
+			with ThreadPoolExecutor(max_workers) as executor:
+				return list(tqdm(executor.map(func, args), total=len(args), desc=desc, bar_format=bar_format, ascii=ascii, **tqdm_kwargs))
+		else:
+			with ThreadPoolExecutor(max_workers) as executor:
+				return list(executor.map(func, args))
+
+	# Single process execution
+	else:
+		if verbose:
+			return [func(arg) for arg in tqdm(args, total=len(args), desc=desc, bar_format=bar_format, ascii=ascii, **tqdm_kwargs)]
+		else:
+			return [func(arg) for arg in args]
+
+
+# "Private" function for capturing multiprocessing subprocess
+def capture_subprocess_output[T, R](args: tuple[CaptureOutput, Callable[[T], R], T]) -> R:
+	""" Wrapper function to execute the target function in a subprocess with optional output capture.
+
+	Args:
+		tuple[CaptureOutput,Callable,T]: Tuple containing:
+			CaptureOutput: Capturer object to redirect stdout/stderr
+			Callable: Target function to execute
+			T: Argument to pass to the target function
+	"""
+	capturer, func, arg = args
+	capturer.redirect()
+	return func(arg)
+

stouputils/{parallel.pyi → parallel/multi.pyi}

@@ -1,16 +1,13 @@
-from .ctx import SetMPStartMethod as SetMPStartMethod
-from .print import BAR_FORMAT as BAR_FORMAT, MAGENTA as MAGENTA
-from collections.abc import Callable, Iterable
-from typing import Any, TypeVar
+from ..ctx import SetMPStartMethod as SetMPStartMethod
+from ..print import BAR_FORMAT as BAR_FORMAT, MAGENTA as MAGENTA
+from .capturer import CaptureOutput as CaptureOutput
+from .common import CPU_COUNT as CPU_COUNT, handle_parameters as handle_parameters, nice_wrapper as nice_wrapper
+from collections.abc import Callable as Callable, Iterable
+from typing import Any
 
 def doctest_square(x: int) -> int: ...
 def doctest_slow(x: int) -> int: ...
-
-CPU_COUNT: int
-T = TypeVar('T')
-R = TypeVar('R')
-
-def multiprocessing[T, R](func: Callable[..., R] | list[Callable[..., R]], args: Iterable[T], use_starmap: bool = False, chunksize: int = 1, desc: str = '', max_workers: int | float = ..., delay_first_calls: float = 0, nice: int | None = None, color: str = ..., bar_format: str = ..., ascii: bool = False, smooth_tqdm: bool = True, **tqdm_kwargs: Any) -> list[R]:
+def multiprocessing[T, R](func: Callable[..., R] | list[Callable[..., R]], args: Iterable[T], use_starmap: bool = False, chunksize: int = 1, desc: str = '', max_workers: int | float = ..., capture_output: bool = False, delay_first_calls: float = 0, nice: int | None = None, color: str = ..., bar_format: str = ..., ascii: bool = False, smooth_tqdm: bool = True, **tqdm_kwargs: Any) -> list[R]:
     ''' Method to execute a function in parallel using multiprocessing
 
 \t- For CPU-bound operations where the GIL (Global Interpreter Lock) is a bottleneck.
@@ -29,6 +26,7 @@ def multiprocessing[T, R](func: Callable[..., R] | list[Callable[..., R]], args:
 \t\tmax_workers\t\t\t(int | float):\t\tNumber of workers to use (Defaults to CPU_COUNT), -1 means CPU_COUNT.
 \t\t\tIf float between 0 and 1, it\'s treated as a percentage of CPU_COUNT.
 \t\t\tIf negative float between -1 and 0, it\'s treated as a percentage of len(args).
+\t\tcapture_output\t\t(bool):\t\t\t\tWhether to capture stdout/stderr from the worker processes (Defaults to True)
 \t\tdelay_first_calls\t(float):\t\t\tApply i*delay_first_calls seconds delay to the first "max_workers" calls.
 \t\t\tFor instance, the first process will be delayed by 0 seconds, the second by 1 second, etc.
 \t\t\t(Defaults to 0): This can be useful to avoid functions being called in the same second.
@@ -129,108 +127,12 @@ def multithreading[T, R](func: Callable[..., R] | list[Callable[..., R]], args:
 \t\t\t. )
 \t\t\t[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
 \t'''
-def run_in_subprocess[R](func: Callable[..., R], *args: Any, timeout: float | None = None, no_join: bool = False, **kwargs: Any) -> R:
-    ''' Execute a function in a subprocess with positional and keyword arguments.
-
-\tThis is useful when you need to run a function in isolation to avoid memory leaks,
-\tresource conflicts, or to ensure a clean execution environment. The subprocess will
-\tbe created, run the function with the provided arguments, and return the result.
-
-\tArgs:
-\t\tfunc         (Callable):     The function to execute in a subprocess.
-\t\t\t(SHOULD BE A TOP-LEVEL FUNCTION TO BE PICKLABLE)
-\t\t*args        (Any):          Positional arguments to pass to the function.
-\t\ttimeout      (float | None): Maximum time in seconds to wait for the subprocess.
-\t\t\tIf None, wait indefinitely. If the subprocess exceeds this time, it will be terminated.
-\t\tno_join      (bool):         If True, do not wait for the subprocess to finish (fire-and-forget).
-\t\t**kwargs     (Any):          Keyword arguments to pass to the function.
-
-\tReturns:
-\t\tR: The return value of the function.
-
-\tRaises:
-\t\tRuntimeError: If the subprocess exits with a non-zero exit code or times out.
-\t\tTimeoutError: If the subprocess exceeds the specified timeout.
-
-\tExamples:
-\t\t.. code-block:: python
-
-\t\t\t> # Simple function execution
-\t\t\t> run_in_subprocess(doctest_square, 5)
-\t\t\t25
-
-\t\t\t> # Function with multiple arguments
-\t\t\t> def add(a: int, b: int) -> int:
-\t\t\t.     return a + b
-\t\t\t> run_in_subprocess(add, 10, 20)
-\t\t\t30
-
-\t\t\t> # Function with keyword arguments
-\t\t\t> def greet(name: str, greeting: str = "Hello") -> str:
-\t\t\t.     return f"{greeting}, {name}!"
-\t\t\t> run_in_subprocess(greet, "World", greeting="Hi")
-\t\t\t\'Hi, World!\'
-
-\t\t\t> # With timeout to prevent hanging
-\t\t\t> run_in_subprocess(some_gpu_func, data, timeout=300.0)
-\t'''
-def _nice_wrapper[T, R](args: tuple[int, Callable[[T], R], T]) -> R:
-    """ Wrapper that applies nice priority then executes the function.
-
-\tArgs:
-\t\targs (tuple): Tuple containing (nice_value, func, arg)
-
-\tReturns:
-\t\tR: Result of the function execution
-\t"""
-def _set_process_priority(nice_value: int) -> None:
-    """ Set the priority of the current process.
+def capture_subprocess_output[T, R](args: tuple[CaptureOutput, Callable[[T], R], T]) -> R:
+    """ Wrapper function to execute the target function in a subprocess with optional output capture.
 
 \tArgs:
-\t\tnice_value (int): Unix-style priority value (-20 to 19)
+\t\ttuple[CaptureOutput,Callable,T]: Tuple containing:
+\t\t\tCaptureOutput: Capturer object to redirect stdout/stderr
+\t\t\tCallable: Target function to execute
+\t\t\tT: Argument to pass to the target function
 \t"""
-def _subprocess_wrapper[R](result_queue: Any, func: Callable[..., R], args: tuple[Any, ...], kwargs: dict[str, Any]) -> None:
-    """ Wrapper function to execute the target function and store the result in the queue.
-
-\tMust be at module level to be pickable on Windows (spawn context).
-
-\tArgs:
-\t\tresult_queue (multiprocessing.Queue | None):  Queue to store the result or exception (None if detached).
-\t\tfunc         (Callable):                            The target function to execute.
-\t\targs         (tuple):                               Positional arguments for the function.
-\t\tkwargs       (dict):                                Keyword arguments for the function.
-\t"""
-def _starmap[T, R](args: tuple[Callable[[T], R], list[T]]) -> R:
-    """ Private function to use starmap using args[0](\\*args[1])
-
-\tArgs:
-\t\targs (tuple): Tuple containing the function and the arguments list to pass to the function
-\tReturns:
-\t\tobject: Result of the function execution
-\t"""
-def _delayed_call[T, R](args: tuple[Callable[[T], R], float, T]) -> R:
-    """ Private function to apply delay before calling the target function
-
-\tArgs:
-\t\targs (tuple): Tuple containing the function, delay in seconds, and the argument to pass to the function
-\tReturns:
-\t\tobject: Result of the function execution
-\t"""
-def _handle_parameters[T, R](func: Callable[[T], R] | list[Callable[[T], R]], args: list[T], use_starmap: bool, delay_first_calls: float, max_workers: int, desc: str, color: str) -> tuple[str, Callable[[T], R], list[T]]:
-    ''' Private function to handle the parameters for multiprocessing or multithreading functions
-
-\tArgs:
-\t\tfunc\t\t\t\t(Callable | list[Callable]):\tFunction to execute, or list of functions (one per argument)
-\t\targs\t\t\t\t(list):\t\t\t\tList of arguments to pass to the function(s)
-\t\tuse_starmap\t\t\t(bool):\t\t\t\tWhether to use starmap or not (Defaults to False):
-\t\t\tTrue means the function will be called like func(\\*args[i]) instead of func(args[i])
-\t\tdelay_first_calls\t(int):\t\t\t\tApply i*delay_first_calls seconds delay to the first "max_workers" calls.
-\t\t\tFor instance, the first process will be delayed by 0 seconds, the second by 1 second, etc. (Defaults to 0):
-\t\t\tThis can be useful to avoid functions being called in the same second.
-\t\tmax_workers\t\t\t(int):\t\t\t\tNumber of workers to use (Defaults to CPU_COUNT)
-\t\tdesc\t\t\t\t(str):\t\t\t\tDescription of the function execution displayed in the progress bar
-\t\tcolor\t\t\t\t(str):\t\t\t\tColor of the progress bar
-
-\tReturns:
-\t\ttuple[str, Callable[[T], R], list[T]]:\tTuple containing the description, function, and arguments
-\t'''

stouputils/parallel/subprocess.py

@@ -0,0 +1,163 @@
+
+# Imports
+import time
+from collections.abc import Callable
+from typing import Any
+
+from .capturer import CaptureOutput
+
+
+def run_in_subprocess[R](
+	func: Callable[..., R],
+	*args: Any,
+	timeout: float | None = None,
+	no_join: bool = False,
+	capture_output: bool = False,
+	**kwargs: Any
+) -> R:
+	""" Execute a function in a subprocess with positional and keyword arguments.
+
+	This is useful when you need to run a function in isolation to avoid memory leaks,
+	resource conflicts, or to ensure a clean execution environment. The subprocess will
+	be created, run the function with the provided arguments, and return the result.
+
+	Args:
+		func           (Callable):     The function to execute in a subprocess.
+			(SHOULD BE A TOP-LEVEL FUNCTION TO BE PICKLABLE)
+		*args          (Any):          Positional arguments to pass to the function.
+		timeout        (float | None): Maximum time in seconds to wait for the subprocess.
+			If None, wait indefinitely. If the subprocess exceeds this time, it will be terminated.
+		no_join        (bool):         If True, do not wait for the subprocess to finish (fire-and-forget).
+		capture_output (bool):         If True, capture the subprocess' stdout/stderr and relay it
+			in real time to the parent's stdout. This enables seeing print() output
+			from the subprocess in the main process.
+		**kwargs       (Any):          Keyword arguments to pass to the function.
+
+	Returns:
+		R: The return value of the function.
+
+	Raises:
+		RuntimeError: If the subprocess exits with a non-zero exit code or times out.
+		TimeoutError: If the subprocess exceeds the specified timeout.
+
+	Examples:
+		.. code-block:: python
+
+			> # Simple function execution
+			> run_in_subprocess(doctest_square, 5)
+			25
+
+			> # Function with multiple arguments
+			> def add(a: int, b: int) -> int:
+			.     return a + b
+			> run_in_subprocess(add, 10, 20)
+			30
+
+			> # Function with keyword arguments
+			> def greet(name: str, greeting: str = "Hello") -> str:
+			.     return f"{greeting}, {name}!"
+			> run_in_subprocess(greet, "World", greeting="Hi")
+			'Hi, World!'
+
+			> # With timeout to prevent hanging
+			> run_in_subprocess(some_gpu_func, data, timeout=300.0)
+	"""
+	import multiprocessing as mp
+	from multiprocessing import Queue
+
+	# Create a queue to get the result from the subprocess (only if we need to wait)
+	result_queue: Queue[R | Exception] | None = None if no_join else Queue()
+
+	# Optionally setup output capture pipe and listener
+	capturer: CaptureOutput | None = None
+	if capture_output:
+		capturer = CaptureOutput()
+
+	# Create and start the subprocess using the module-level wrapper
+	process: mp.Process = mp.Process(
+		target=_subprocess_wrapper,
+		args=(result_queue, func, args, kwargs),
+		kwargs={"_capturer": capturer}
+	)
+	process.start()
+
+	# For capture_output we must close the parent's copy of the write fd and start listener
+	if capturer is not None:
+		capturer.parent_close_write()
+		capturer.start_listener()
+
+	# Detach process if no_join (fire-and-forget)
+	if result_queue is None:
+		# If capturing, leave listener running in background (daemon)
+		return None  # type: ignore
+
+	# Use a single try/finally to ensure we always drain the listener once
+	# and avoid repeating join calls in multiple branches.
+	try:
+		process.join(timeout=timeout)
+
+		# Check if process is still alive (timed out)
+		if process.is_alive():
+			process.terminate()
+			time.sleep(0.5)  # Give it a moment to terminate gracefully
+			if process.is_alive():
+				process.kill()
+			process.join()
+			raise TimeoutError(f"Subprocess exceeded timeout of {timeout} seconds and was terminated")
+
+		# Check exit code
+		if process.exitcode != 0:
+			# Try to get any exception from the queue (non-blocking)
+			if not result_queue.empty():
+				result_or_exception = result_queue.get_nowait()
+				if isinstance(result_or_exception, Exception):
+					raise result_or_exception
+			raise RuntimeError(f"Subprocess failed with exit code {process.exitcode}")
+
+		# Retrieve the result
+		try:
+			result_or_exception = result_queue.get_nowait()
+			if isinstance(result_or_exception, Exception):
+				raise result_or_exception
+			return result_or_exception
+		except Exception as e:
+			raise RuntimeError("Subprocess did not return any result") from e
+	finally:
+		if capturer is not None:
+			capturer.join_listener(timeout=5.0)
+
+
+# "Private" function for subprocess wrapper (must be at module level for pickling on Windows)
+def _subprocess_wrapper[R](
+	result_queue: Any,
+	func: Callable[..., R],
+	args: tuple[Any, ...],
+	kwargs: dict[str, Any],
+	_capturer: CaptureOutput | None = None
+) -> None:
+	""" Wrapper function to execute the target function and store the result in the queue.
+
+	Must be at module level to be pickable on Windows (spawn context).
+
+	Args:
+		result_queue (multiprocessing.Queue | None):  Queue to store the result or exception (None if detached).
+		func         (Callable):                      The target function to execute.
+		args         (tuple):                         Positional arguments for the function.
+		kwargs       (dict):                          Keyword arguments for the function.
+		_capturer    (CaptureOutput | None):          Optional CaptureOutput instance for stdout capture.
+	"""
+	try:
+		# If a CaptureOutput instance was passed, redirect stdout/stderr to the pipe.
+		if _capturer is not None:
+			_capturer.redirect()
+
+		# Execute the target function and put the result in the queue
+		result: R = func(*args, **kwargs)
+		if result_queue is not None:
+			result_queue.put(result)
+
+	# Handle cleanup and exceptions
+	except Exception as e:
+		if result_queue is not None:
+			result_queue.put(e)
+