stouputils 1.9.0__tar.gz → 1.9.1__tar.gz

{stouputils-1.9.0 → stouputils-1.9.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: stouputils
-Version: 1.9.0
+Version: 1.9.1
 Summary: Stouputils is a collection of utility modules designed to simplify and enhance the development process. It includes a range of tools for tasks such as execution of doctests, display utilities, decorators, as well as context managers, and many more.
 Project-URL: Homepage, https://github.com/Stoupy51/stouputils
 Project-URL: Issues, https://github.com/Stoupy51/stouputils/issues

{stouputils-1.9.0 → stouputils-1.9.1}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "stouputils"
-version = "1.9.0"
+version = "1.9.1"
 description = "Stouputils is a collection of utility modules designed to simplify and enhance the development process. It includes a range of tools for tasks such as execution of doctests, display utilities, decorators, as well as context managers, and many more."
 readme = "README.md"
 requires-python = ">=3.10"

{stouputils-1.9.0 → stouputils-1.9.1}/stouputils/parallel.py

@@ -3,6 +3,7 @@ This module provides utility functions for parallel processing, such as:
 
 - multiprocessing(): Execute a function in parallel using multiprocessing
 - multithreading(): Execute a function in parallel using multithreading
+- run_in_subprocess(): Execute a function in a subprocess with args and kwargs
 
 I highly encourage you to read the function docstrings to understand when to use each method.
 
@@ -35,7 +36,7 @@ R = TypeVar("R")
 # Functions
 @handle_error(error_log=LogLevels.ERROR_TRACEBACK)
 def multiprocessing(
-	func: Callable[[T], R],
+	func: Callable[[T], R] | list[Callable[[T], R]],
 	args: list[T],
 	use_starmap: bool = False,
 	chunksize: int = 1,
@@ -53,15 +54,15 @@ def multiprocessing(
 	- For computationally intensive tasks like scientific simulations, data analysis, or machine learning workloads.
 
 	Args:
-		func				(Callable):			Function to execute
-		args				(list):				List of arguments to pass to the function
+		func				(Callable | list[Callable]):	Function to execute, or list of functions (one per argument)
+		args				(list):				List of arguments to pass to the function(s)
 		use_starmap			(bool):				Whether to use starmap or not (Defaults to False):
 			True means the function will be called like func(\*args[i]) instead of func(args[i])
 		chunksize			(int):				Number of arguments to process at a time
 			(Defaults to 1 for proper progress bar display)
 		desc				(str):				Description displayed in the progress bar
 			(if not provided no progress bar will be displayed)
-		max_workers			(int):				Number of workers to use (Defaults to CPU_COUNT)
+		max_workers			(int):				Number of workers to use (Defaults to CPU_COUNT), -1 means CPU_COUNT
 		delay_first_calls	(float):			Apply i*delay_first_calls seconds delay to the first "max_workers" calls.
 			For instance, the first process will be delayed by 0 seconds, the second by 1 second, etc.
 			(Defaults to 0): This can be useful to avoid functions being called in the same second.
@@ -81,6 +82,10 @@ def multiprocessing(
 			> multiprocessing(int.__mul__, [(1,2), (3,4), (5,6)], use_starmap=True)
 			[2, 12, 30]
 
+			> # Using a list of functions (one per argument)
+			> multiprocessing([doctest_square, doctest_square, doctest_square], [1, 2, 3])
+			[1, 4, 9]
+
 			> # Will process in parallel with progress bar
 			> multiprocessing(doctest_slow, range(10), desc="Processing")
 			[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
@@ -103,6 +108,8 @@ def multiprocessing(
 	from tqdm.contrib.concurrent import process_map  # pyright: ignore[reportUnknownVariableType]
 
 	# Handle parameters
+	if max_workers == -1:
+		max_workers = CPU_COUNT
 	verbose: bool = desc != ""
 	desc, func, args = _handle_parameters(func, args, use_starmap, delay_first_calls, max_workers, desc, color)
 	if bar_format == BAR_FORMAT:
@@ -145,7 +152,7 @@ def multiprocessing(
 
 @handle_error(error_log=LogLevels.ERROR_TRACEBACK)
 def multithreading(
-	func: Callable[[T], R],
+	func: Callable[[T], R] | list[Callable[[T], R]],
 	args: list[T],
 	use_starmap: bool = False,
 	desc: str = "",
@@ -162,13 +169,13 @@ def multithreading(
 	- For operations that involve a lot of waiting, such as GUI event handling or handling user input.
 
 	Args:
-		func				(Callable):			Function to execute
-		args				(list):				List of arguments to pass to the function
+		func				(Callable | list[Callable]):	Function to execute, or list of functions (one per argument)
+		args				(list):				List of arguments to pass to the function(s)
 		use_starmap			(bool):				Whether to use starmap or not (Defaults to False):
 			True means the function will be called like func(\*args[i]) instead of func(args[i])
 		desc				(str):				Description displayed in the progress bar
 			(if not provided no progress bar will be displayed)
-		max_workers			(int):				Number of workers to use (Defaults to CPU_COUNT)
+		max_workers			(int):				Number of workers to use (Defaults to CPU_COUNT), -1 means CPU_COUNT
 		delay_first_calls	(float):			Apply i*delay_first_calls seconds delay to the first "max_workers" calls.
 			For instance with value to 1, the first thread will be delayed by 0 seconds, the second by 1 second, etc.
 			(Defaults to 0): This can be useful to avoid functions being called in the same second.
@@ -188,6 +195,10 @@ def multithreading(
 			> multithreading(int.__mul__, [(1,2), (3,4), (5,6)], use_starmap=True)
 			[2, 12, 30]
 
+			> # Using a list of functions (one per argument)
+			> multithreading([doctest_square, doctest_square, doctest_square], [1, 2, 3])
+			[1, 4, 9]
+
 			> # Will process in parallel with progress bar
 			> multithreading(doctest_slow, range(10), desc="Threading")
 			[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
@@ -208,6 +219,8 @@ def multithreading(
 	from tqdm.auto import tqdm
 
 	# Handle parameters
+	if max_workers == -1:
+		max_workers = CPU_COUNT
 	verbose: bool = desc != ""
 	desc, func, args = _handle_parameters(func, args, use_starmap, delay_first_calls, max_workers, desc, color)
 	if bar_format == BAR_FORMAT:
@@ -230,6 +243,99 @@ def multithreading(
 			return [func(arg) for arg in args]
 
 
+@handle_error(error_log=LogLevels.ERROR_TRACEBACK)
+def run_in_subprocess(
+	func: Callable[..., R],
+	*args: Any,
+	**kwargs: Any
+) -> R:
+	""" Execute a function in a subprocess with positional and keyword arguments.
+
+	This is useful when you need to run a function in isolation to avoid memory leaks,
+	resource conflicts, or to ensure a clean execution environment. The subprocess will
+	be created, run the function with the provided arguments, and return the result.
+
+	Args:
+		func     (Callable): The function to execute in a subprocess.
+			(SHOULD BE A TOP-LEVEL FUNCTION TO BE PICKLABLE)
+		*args    (Any):      Positional arguments to pass to the function.
+		**kwargs (Any):      Keyword arguments to pass to the function.
+
+	Returns:
+		R: The return value of the function.
+
+	Raises:
+		RuntimeError: If the subprocess exits with a non-zero exit code.
+
+	Examples:
+		.. code-block:: python
+
+			> # Simple function execution
+			> run_in_subprocess(doctest_square, 5)
+			25
+
+			> # Function with multiple arguments
+			> def add(a: int, b: int) -> int:
+			.     return a + b
+			> run_in_subprocess(add, 10, 20)
+			30
+
+			> # Function with keyword arguments
+			> def greet(name: str, greeting: str = "Hello") -> str:
+			.     return f"{greeting}, {name}!"
+			> run_in_subprocess(greet, "World", greeting="Hi")
+			'Hi, World!'
+	"""
+	import multiprocessing as mp
+	from multiprocessing import Queue
+
+	# Create a queue to get the result from the subprocess
+	result_queue: Queue[R | Exception] = Queue()
+
+	# Create and start the subprocess using the module-level wrapper
+	process: mp.Process = mp.Process(
+		target=_subprocess_wrapper,
+		args=(result_queue, func, args, kwargs)
+	)
+	process.start()
+	process.join()
+
+	# Check exit code
+	if process.exitcode != 0:
+		raise RuntimeError(f"Subprocess failed with exit code {process.exitcode}")
+
+	# Retrieve the result
+	if not result_queue.empty():
+		result_or_exception = result_queue.get()
+		if isinstance(result_or_exception, Exception):
+			raise result_or_exception
+		return result_or_exception
+	else:
+		raise RuntimeError("Subprocess did not return any result")
+
+
+# "Private" function for subprocess wrapper (must be at module level for pickling on Windows)
+def _subprocess_wrapper(
+	result_queue: Any,
+	func: Callable[..., R],
+	args: tuple[Any, ...],
+	kwargs: dict[str, Any]
+) -> None:
+	""" Wrapper function to execute the target function and store the result in the queue.
+
+	Must be at module level to be pickable on Windows (spawn context).
+
+	Args:
+		result_queue (multiprocessing.Queue):  Queue to store the result or exception.
+		func         (Callable):               The target function to execute.
+		args         (tuple):                  Positional arguments for the function.
+		kwargs       (dict):                   Keyword arguments for the function.
+	"""
+	try:
+		result: R = func(*args, **kwargs)
+		result_queue.put(result)
+	except Exception as e:
+		result_queue.put(e)
 
 # "Private" function to use starmap
 def _starmap(args: tuple[Callable[[T], R], list[T]]) -> R:
@@ -258,7 +364,7 @@ def _delayed_call(args: tuple[Callable[[T], R], float, T]) -> R:
 
 # "Private" function to handle parameters for multiprocessing or multithreading functions
 def _handle_parameters(
-	func: Callable[[T], R],
+	func: Callable[[T], R] | list[Callable[[T], R]],
 	args: list[T],
 	use_starmap: bool,
 	delay_first_calls: float,
@@ -269,8 +375,8 @@ def _handle_parameters(
 	r""" Private function to handle the parameters for multiprocessing or multithreading functions
 
 	Args:
-		func				(Callable):			Function to execute
-		args				(list):				List of arguments to pass to the function
+		func				(Callable | list[Callable]):	Function to execute, or list of functions (one per argument)
+		args				(list):				List of arguments to pass to the function(s)
 		use_starmap			(bool):				Whether to use starmap or not (Defaults to False):
 			True means the function will be called like func(\*args[i]) instead of func(args[i])
 		delay_first_calls	(int):				Apply i*delay_first_calls seconds delay to the first "max_workers" calls.
@@ -285,8 +391,15 @@ def _handle_parameters(
 	"""
 	desc = color + desc
 
-	# If use_starmap is True, we use the __starmap function
-	if use_starmap:
+	# Handle list of functions: validate and convert to starmap format
+	if isinstance(func, list):
+		func = cast(list[Callable[[T], R]], func)
+		assert len(func) == len(args), f"Length mismatch: {len(func)} functions but {len(args)} arguments"
+		args = [(f, arg) for f, arg in zip(func, args, strict=False)] # type: ignore
+		func = _starmap # type: ignore
+
+	# If use_starmap is True, we use the _starmap function
+	elif use_starmap:
 		args = [(func, arg) for arg in args] # type: ignore
 		func = _starmap # type: ignore
 
@@ -298,5 +411,5 @@ def _handle_parameters(
 		]
 		func = _delayed_call  # type: ignore
 
-	return desc, func, args
+	return desc, func, args # type: ignore
 

{stouputils-1.9.0 → stouputils-1.9.1}/stouputils/parallel.pyi

@@ -1,7 +1,7 @@
 from .decorators import LogLevels as LogLevels, handle_error as handle_error
 from .print import BAR_FORMAT as BAR_FORMAT, MAGENTA as MAGENTA
-from collections.abc import Callable as Callable
-from typing import TypeVar
+from collections.abc import Callable
+from typing import Any, TypeVar
 
 def doctest_square(x: int) -> int: ...
 def doctest_slow(x: int) -> int: ...
@@ -10,7 +10,7 @@ CPU_COUNT: int
 T = TypeVar('T')
 R = TypeVar('R')
 
-def multiprocessing(func: Callable[[T], R], args: list[T], use_starmap: bool = False, chunksize: int = 1, desc: str = '', max_workers: int = ..., delay_first_calls: float = 0, color: str = ..., bar_format: str = ..., ascii: bool = False) -> list[R]:
+def multiprocessing(func: Callable[[T], R] | list[Callable[[T], R]], args: list[T], use_starmap: bool = False, chunksize: int = 1, desc: str = '', max_workers: int = ..., delay_first_calls: float = 0, color: str = ..., bar_format: str = ..., ascii: bool = False) -> list[R]:
     ''' Method to execute a function in parallel using multiprocessing
 
 \t- For CPU-bound operations where the GIL (Global Interpreter Lock) is a bottleneck.
@@ -18,15 +18,15 @@ def multiprocessing(func: Callable[[T], R], args: list[T], use_starmap: bool = F
 \t- For computationally intensive tasks like scientific simulations, data analysis, or machine learning workloads.
 
 \tArgs:
-\t\tfunc\t\t\t\t(Callable):\t\t\tFunction to execute
-\t\targs\t\t\t\t(list):\t\t\t\tList of arguments to pass to the function
+\t\tfunc\t\t\t\t(Callable | list[Callable]):\tFunction to execute, or list of functions (one per argument)
+\t\targs\t\t\t\t(list):\t\t\t\tList of arguments to pass to the function(s)
 \t\tuse_starmap\t\t\t(bool):\t\t\t\tWhether to use starmap or not (Defaults to False):
 \t\t\tTrue means the function will be called like func(\\*args[i]) instead of func(args[i])
 \t\tchunksize\t\t\t(int):\t\t\t\tNumber of arguments to process at a time
 \t\t\t(Defaults to 1 for proper progress bar display)
 \t\tdesc\t\t\t\t(str):\t\t\t\tDescription displayed in the progress bar
 \t\t\t(if not provided no progress bar will be displayed)
-\t\tmax_workers\t\t\t(int):\t\t\t\tNumber of workers to use (Defaults to CPU_COUNT)
+\t\tmax_workers\t\t\t(int):\t\t\t\tNumber of workers to use (Defaults to CPU_COUNT), -1 means CPU_COUNT
 \t\tdelay_first_calls\t(float):\t\t\tApply i*delay_first_calls seconds delay to the first "max_workers" calls.
 \t\t\tFor instance, the first process will be delayed by 0 seconds, the second by 1 second, etc.
 \t\t\t(Defaults to 0): This can be useful to avoid functions being called in the same second.
@@ -46,6 +46,10 @@ def multiprocessing(func: Callable[[T], R], args: list[T], use_starmap: bool = F
 \t\t\t> multiprocessing(int.__mul__, [(1,2), (3,4), (5,6)], use_starmap=True)
 \t\t\t[2, 12, 30]
 
+\t\t\t> # Using a list of functions (one per argument)
+\t\t\t> multiprocessing([doctest_square, doctest_square, doctest_square], [1, 2, 3])
+\t\t\t[1, 4, 9]
+
 \t\t\t> # Will process in parallel with progress bar
 \t\t\t> multiprocessing(doctest_slow, range(10), desc="Processing")
 \t\t\t[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
@@ -60,7 +64,7 @@ def multiprocessing(func: Callable[[T], R], args: list[T], use_starmap: bool = F
 \t\t\t. )
 \t\t\t[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
 \t'''
-def multithreading(func: Callable[[T], R], args: list[T], use_starmap: bool = False, desc: str = '', max_workers: int = ..., delay_first_calls: float = 0, color: str = ..., bar_format: str = ..., ascii: bool = False) -> list[R]:
+def multithreading(func: Callable[[T], R] | list[Callable[[T], R]], args: list[T], use_starmap: bool = False, desc: str = '', max_workers: int = ..., delay_first_calls: float = 0, color: str = ..., bar_format: str = ..., ascii: bool = False) -> list[R]:
     ''' Method to execute a function in parallel using multithreading, you should use it:
 
 \t- For I/O-bound operations where the GIL is not a bottleneck, such as network requests or disk operations.
@@ -68,13 +72,13 @@ def multithreading(func: Callable[[T], R], args: list[T], use_starmap: bool = Fa
 \t- For operations that involve a lot of waiting, such as GUI event handling or handling user input.
 
 \tArgs:
-\t\tfunc\t\t\t\t(Callable):\t\t\tFunction to execute
-\t\targs\t\t\t\t(list):\t\t\t\tList of arguments to pass to the function
+\t\tfunc\t\t\t\t(Callable | list[Callable]):\tFunction to execute, or list of functions (one per argument)
+\t\targs\t\t\t\t(list):\t\t\t\tList of arguments to pass to the function(s)
 \t\tuse_starmap\t\t\t(bool):\t\t\t\tWhether to use starmap or not (Defaults to False):
 \t\t\tTrue means the function will be called like func(\\*args[i]) instead of func(args[i])
 \t\tdesc\t\t\t\t(str):\t\t\t\tDescription displayed in the progress bar
 \t\t\t(if not provided no progress bar will be displayed)
-\t\tmax_workers\t\t\t(int):\t\t\t\tNumber of workers to use (Defaults to CPU_COUNT)
+\t\tmax_workers\t\t\t(int):\t\t\t\tNumber of workers to use (Defaults to CPU_COUNT), -1 means CPU_COUNT
 \t\tdelay_first_calls\t(float):\t\t\tApply i*delay_first_calls seconds delay to the first "max_workers" calls.
 \t\t\tFor instance with value to 1, the first thread will be delayed by 0 seconds, the second by 1 second, etc.
 \t\t\t(Defaults to 0): This can be useful to avoid functions being called in the same second.
@@ -94,6 +98,10 @@ def multithreading(func: Callable[[T], R], args: list[T], use_starmap: bool = Fa
 \t\t\t> multithreading(int.__mul__, [(1,2), (3,4), (5,6)], use_starmap=True)
 \t\t\t[2, 12, 30]
 
+\t\t\t> # Using a list of functions (one per argument)
+\t\t\t> multithreading([doctest_square, doctest_square, doctest_square], [1, 2, 3])
+\t\t\t[1, 4, 9]
+
 \t\t\t> # Will process in parallel with progress bar
 \t\t\t> multithreading(doctest_slow, range(10), desc="Threading")
 \t\t\t[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
@@ -108,6 +116,55 @@ def multithreading(func: Callable[[T], R], args: list[T], use_starmap: bool = Fa
 \t\t\t. )
 \t\t\t[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
 \t'''
+def run_in_subprocess(func: Callable[..., R], *args: Any, **kwargs: Any) -> R:
+    ''' Execute a function in a subprocess with positional and keyword arguments.
+
+\tThis is useful when you need to run a function in isolation to avoid memory leaks,
+\tresource conflicts, or to ensure a clean execution environment. The subprocess will
+\tbe created, run the function with the provided arguments, and return the result.
+
+\tArgs:
+\t\tfunc     (Callable): The function to execute in a subprocess.
+\t\t\t(SHOULD BE A TOP-LEVEL FUNCTION TO BE PICKLABLE)
+\t\t*args    (Any):      Positional arguments to pass to the function.
+\t\t**kwargs (Any):      Keyword arguments to pass to the function.
+
+\tReturns:
+\t\tR: The return value of the function.
+
+\tRaises:
+\t\tRuntimeError: If the subprocess exits with a non-zero exit code.
+
+\tExamples:
+\t\t.. code-block:: python
+
+\t\t\t> # Simple function execution
+\t\t\t> run_in_subprocess(doctest_square, 5)
+\t\t\t25
+
+\t\t\t> # Function with multiple arguments
+\t\t\t> def add(a: int, b: int) -> int:
+\t\t\t.     return a + b
+\t\t\t> run_in_subprocess(add, 10, 20)
+\t\t\t30
+
+\t\t\t> # Function with keyword arguments
+\t\t\t> def greet(name: str, greeting: str = "Hello") -> str:
+\t\t\t.     return f"{greeting}, {name}!"
+\t\t\t> run_in_subprocess(greet, "World", greeting="Hi")
+\t\t\t\'Hi, World!\'
+\t'''
+def _subprocess_wrapper(result_queue: Any, func: Callable[..., R], args: tuple[Any, ...], kwargs: dict[str, Any]) -> None:
+    """ Wrapper function to execute the target function and store the result in the queue.
+
+\tMust be at module level to be pickable on Windows (spawn context).
+
+\tArgs:
+\t\tresult_queue (multiprocessing.Queue):  Queue to store the result or exception.
+\t\tfunc         (Callable):               The target function to execute.
+\t\targs         (tuple):                  Positional arguments for the function.
+\t\tkwargs       (dict):                   Keyword arguments for the function.
+\t"""
 def _starmap(args: tuple[Callable[[T], R], list[T]]) -> R:
     """ Private function to use starmap using args[0](\\*args[1])
 
@@ -124,12 +181,12 @@ def _delayed_call(args: tuple[Callable[[T], R], float, T]) -> R:
 \tReturns:
 \t\tobject: Result of the function execution
 \t"""
-def _handle_parameters(func: Callable[[T], R], args: list[T], use_starmap: bool, delay_first_calls: float, max_workers: int, desc: str, color: str) -> tuple[str, Callable[[T], R], list[T]]:
+def _handle_parameters(func: Callable[[T], R] | list[Callable[[T], R]], args: list[T], use_starmap: bool, delay_first_calls: float, max_workers: int, desc: str, color: str) -> tuple[str, Callable[[T], R], list[T]]:
     ''' Private function to handle the parameters for multiprocessing or multithreading functions
 
 \tArgs:
-\t\tfunc\t\t\t\t(Callable):\t\t\tFunction to execute
-\t\targs\t\t\t\t(list):\t\t\t\tList of arguments to pass to the function
+\t\tfunc\t\t\t\t(Callable | list[Callable]):\tFunction to execute, or list of functions (one per argument)
+\t\targs\t\t\t\t(list):\t\t\t\tList of arguments to pass to the function(s)
 \t\tuse_starmap\t\t\t(bool):\t\t\t\tWhether to use starmap or not (Defaults to False):
 \t\t\tTrue means the function will be called like func(\\*args[i]) instead of func(args[i])
 \t\tdelay_first_calls\t(int):\t\t\t\tApply i*delay_first_calls seconds delay to the first "max_workers" calls.