FlowerPower 0.31.0__py3-none-any.whl → 0.31.1__py3-none-any.whl

flowerpower/utils/misc.py

@@ -9,218 +9,219 @@ from typing import Any
 import msgspec
 from fsspec_utils import AbstractFileSystem, filesystem
 from .security import validate_file_path
+from fsspec_utils.utils import run_parallel
 
-if importlib.util.find_spec("joblib"):
-    from joblib import Parallel, delayed
-    from rich.progress import (BarColumn, Progress, TextColumn,
-                               TimeElapsedColumn)
+# if importlib.util.find_spec("joblib"):
+#     from joblib import Parallel, delayed
+#     from rich.progress import (BarColumn, Progress, TextColumn,
+#                                TimeElapsedColumn)
 
-    def _prepare_parallel_args(
-        args: tuple, kwargs: dict
-    ) -> tuple[list, list, dict, dict, int]:
-        """Prepare and validate arguments for parallel execution.
+#     def _prepare_parallel_args(
+#         args: tuple, kwargs: dict
+#     ) -> tuple[list, list, dict, dict, int]:
+#         """Prepare and validate arguments for parallel execution.
         
-        Args:
-            args: Positional arguments
-            kwargs: Keyword arguments
+#         Args:
+#             args: Positional arguments
+#             kwargs: Keyword arguments
             
-        Returns:
-            tuple: (iterables, fixed_args, iterable_kwargs, fixed_kwargs, first_iterable_len)
+#         Returns:
+#             tuple: (iterables, fixed_args, iterable_kwargs, fixed_kwargs, first_iterable_len)
             
-        Raises:
-            ValueError: If no iterable arguments or length mismatch
-        """
-        iterables = []
-        fixed_args = []
-        iterable_kwargs = {}
-        fixed_kwargs = {}
-        first_iterable_len = None
-
-        # Process positional arguments
-        for arg in args:
-            if isinstance(arg, (list, tuple)) and not isinstance(arg[0], (list, tuple)):
-                iterables.append(arg)
-                if first_iterable_len is None:
-                    first_iterable_len = len(arg)
-                elif len(arg) != first_iterable_len:
-                    raise ValueError(
-                        f"Iterable length mismatch: argument has length {len(arg)}, expected {first_iterable_len}"
-                    )
-            else:
-                fixed_args.append(arg)
-
-        # Process keyword arguments
-        for key, value in kwargs.items():
-            if isinstance(value, (list, tuple)) and not isinstance(
-                value[0], (list, tuple)
-            ):
-                if first_iterable_len is None:
-                    first_iterable_len = len(value)
-                elif len(value) != first_iterable_len:
-                    raise ValueError(
-                        f"Iterable length mismatch: {key} has length {len(value)}, expected {first_iterable_len}"
-                    )
-                iterable_kwargs[key] = value
-            else:
-                fixed_kwargs[key] = value
-
-        if first_iterable_len is None:
-            raise ValueError("At least one iterable argument is required")
-
-        return iterables, fixed_args, iterable_kwargs, fixed_kwargs, first_iterable_len
-
-    def _execute_parallel_with_progress(
-        func: callable,
-        iterables: list,
-        fixed_args: list,
-        iterable_kwargs: dict,
-        fixed_kwargs: dict,
-        param_combinations: list,
-        parallel_kwargs: dict,
-    ) -> list:
-        """Execute parallel tasks with progress tracking.
+#         Raises:
+#             ValueError: If no iterable arguments or length mismatch
+#         """
+#         iterables = []
+#         fixed_args = []
+#         iterable_kwargs = {}
+#         fixed_kwargs = {}
+#         first_iterable_len = None
+
+#         # Process positional arguments
+#         for arg in args:
+#             if isinstance(arg, (list, tuple)) and not isinstance(arg[0], (list, tuple)):
+#                 iterables.append(arg)
+#                 if first_iterable_len is None:
+#                     first_iterable_len = len(arg)
+#                 elif len(arg) != first_iterable_len:
+#                     raise ValueError(
+#                         f"Iterable length mismatch: argument has length {len(arg)}, expected {first_iterable_len}"
+#                     )
+#             else:
+#                 fixed_args.append(arg)
+
+#         # Process keyword arguments
+#         for key, value in kwargs.items():
+#             if isinstance(value, (list, tuple)) and not isinstance(
+#                 value[0], (list, tuple)
+#             ):
+#                 if first_iterable_len is None:
+#                     first_iterable_len = len(value)
+#                 elif len(value) != first_iterable_len:
+#                     raise ValueError(
+#                         f"Iterable length mismatch: {key} has length {len(value)}, expected {first_iterable_len}"
+#                     )
+#                 iterable_kwargs[key] = value
+#             else:
+#                 fixed_kwargs[key] = value
+
+#         if first_iterable_len is None:
+#             raise ValueError("At least one iterable argument is required")
+
+#         return iterables, fixed_args, iterable_kwargs, fixed_kwargs, first_iterable_len
+
+#     def _execute_parallel_with_progress(
+#         func: callable,
+#         iterables: list,
+#         fixed_args: list,
+#         iterable_kwargs: dict,
+#         fixed_kwargs: dict,
+#         param_combinations: list,
+#         parallel_kwargs: dict,
+#     ) -> list:
+#         """Execute parallel tasks with progress tracking.
         
-        Args:
-            func: Function to execute
-            iterables: List of iterable arguments
-            fixed_args: List of fixed arguments
-            iterable_kwargs: Dictionary of iterable keyword arguments
-            fixed_kwargs: Dictionary of fixed keyword arguments
-            param_combinations: List of parameter combinations
-            parallel_kwargs: Parallel execution configuration
+#         Args:
+#             func: Function to execute
+#             iterables: List of iterable arguments
+#             fixed_args: List of fixed arguments
+#             iterable_kwargs: Dictionary of iterable keyword arguments
+#             fixed_kwargs: Dictionary of fixed keyword arguments
+#             param_combinations: List of parameter combinations
+#             parallel_kwargs: Parallel execution configuration
             
-        Returns:
-            list: Results from parallel execution
-        """
-        results = [None] * len(param_combinations)
-        with Progress(
-            TextColumn("[progress.description]{task.description}"),
-            BarColumn(),
-            "[progress.percentage]{task.percentage:>3.0f}%",
-            TimeElapsedColumn(),
-            transient=True,
-        ) as progress:
-            task = progress.add_task(
-                "Running in parallel...", total=len(param_combinations)
-            )
-
-            def wrapper(idx, param_tuple):
-                res = func(
-                    *(list(param_tuple[: len(iterables)]) + fixed_args),
-                    **{
-                        k: v
-                        for k, v in zip(
-                            iterable_kwargs.keys(), param_tuple[len(iterables) :]
-                        )
-                    },
-                    **fixed_kwargs,
-                )
-                progress.update(task, advance=1)
-                return idx, res
-
-            for idx, result in Parallel(**parallel_kwargs)(
-                delayed(wrapper)(i, param_tuple)
-                for i, param_tuple in enumerate(param_combinations)
-            ):
-                results[idx] = result
-        return results
-
-    def _execute_parallel_without_progress(
-        func: callable,
-        iterables: list,
-        fixed_args: list,
-        iterable_kwargs: dict,
-        fixed_kwargs: dict,
-        param_combinations: list,
-        parallel_kwargs: dict,
-    ) -> list:
-        """Execute parallel tasks without progress tracking.
+#         Returns:
+#             list: Results from parallel execution
+#         """
+#         results = [None] * len(param_combinations)
+#         with Progress(
+#             TextColumn("[progress.description]{task.description}"),
+#             BarColumn(),
+#             "[progress.percentage]{task.percentage:>3.0f}%",
+#             TimeElapsedColumn(),
+#             transient=True,
+#         ) as progress:
+#             task = progress.add_task(
+#                 "Running in parallel...", total=len(param_combinations)
+#             )
+
+#             def wrapper(idx, param_tuple):
+#                 res = func(
+#                     *(list(param_tuple[: len(iterables)]) + fixed_args),
+#                     **{
+#                         k: v
+#                         for k, v in zip(
+#                             iterable_kwargs.keys(), param_tuple[len(iterables) :]
+#                         )
+#                     },
+#                     **fixed_kwargs,
+#                 )
+#                 progress.update(task, advance=1)
+#                 return idx, res
+#
+#             for idx, result in Parallel(**parallel_kwargs)(
+#                 delayed(wrapper)(i, param_tuple)
+#                 for i, param_tuple in enumerate(param_combinations)
+#             ):
+#                 results[idx] = result
+#         return results
+
+#     def _execute_parallel_without_progress(
+#         func: callable,
+#         iterables: list,
+#         fixed_args: list,
+#         iterable_kwargs: dict,
+#         fixed_kwargs: dict,
+#         param_combinations: list,
+#         parallel_kwargs: dict,
+#     ) -> list:
+#         """Execute parallel tasks without progress tracking.
         
-        Args:
-            func: Function to execute
-            iterables: List of iterable arguments
-            fixed_args: List of fixed arguments
-            iterable_kwargs: Dictionary of iterable keyword arguments
-            fixed_kwargs: Dictionary of fixed keyword arguments
-            param_combinations: List of parameter combinations
-            parallel_kwargs: Parallel execution configuration
+#         Args:
+#             func: Function to execute
+#             iterables: List of iterable arguments
+#             fixed_args: List of fixed arguments
+#             iterable_kwargs: Dictionary of iterable keyword arguments
+#             fixed_kwargs: Dictionary of fixed keyword arguments
+#             param_combinations: List of parameter combinations
+#             parallel_kwargs: Parallel execution configuration
             
-        Returns:
-            list: Results from parallel execution
-        """
-        return Parallel(**parallel_kwargs)(
-            delayed(func)(
-                *(list(param_tuple[: len(iterables)]) + fixed_args),
-                **{
-                    k: v
-                    for k, v in zip(
-                        iterable_kwargs.keys(), param_tuple[len(iterables) :]
-                    )
-                },
-                **fixed_kwargs,
-            )
-            for param_tuple in param_combinations
-        )
-
-    def run_parallel(
-        func: callable,
-        *args,
-        n_jobs: int = -1,
-        backend: str = "threading",
-        verbose: bool = True,
-        **kwargs,
-    ) -> list[any]:
-        """Runs a function for a list of parameters in parallel.
-
-        Args:
-            func (Callable): function to run in parallel
-            *args: Positional arguments. Can be single values or iterables
-            n_jobs (int, optional): Number of joblib workers. Defaults to -1
-            backend (str, optional): joblib backend. Valid options are
-                `loky`,`threading`, `mutliprocessing` or `sequential`. Defaults to "threading"
-            verbose (bool, optional): Show progress bar. Defaults to True
-            **kwargs: Keyword arguments. Can be single values or iterables
-
-        Returns:
-            list[any]: Function output
-
-        Examples:
-            >>> # Single iterable argument
-            >>> run_parallel(func, [1,2,3], fixed_arg=42)
-
-            >>> # Multiple iterables in args and kwargs
-            >>> run_parallel(func, [1,2,3], val=[7,8,9], fixed=42)
-
-            >>> # Only kwargs iterables
-            >>> run_parallel(func, x=[1,2,3], y=[4,5,6], fixed=42)
-        """
-        parallel_kwargs = {"n_jobs": n_jobs, "backend": backend, "verbose": 0}
-
-        # Prepare and validate arguments
-        iterables, fixed_args, iterable_kwargs, fixed_kwargs, first_iterable_len = _prepare_parallel_args(
-            args, kwargs
-        )
-
-        # Create parameter combinations
-        all_iterables = iterables + list(iterable_kwargs.values())
-        param_combinations = list(zip(*all_iterables))
-
-        # Execute with or without progress tracking
-        if not verbose:
-            return _execute_parallel_without_progress(
-                func, iterables, fixed_args, iterable_kwargs, fixed_kwargs,
-                param_combinations, parallel_kwargs
-            )
-        else:
-            return _execute_parallel_with_progress(
-                func, iterables, fixed_args, iterable_kwargs, fixed_kwargs,
-                param_combinations, parallel_kwargs
-            )
-
-else:
-
-    def run_parallel(*args, **kwargs):
-        raise ImportError("joblib not installed")
+#         Returns:
+#             list: Results from parallel execution
+#         """
+#         return Parallel(**parallel_kwargs)(
+#             delayed(func)(
+#                 *(list(param_tuple[: len(iterables)]) + fixed_args),
+#                 **{
+#                     k: v
+#                     for k, v in zip(
+#                         iterable_kwargs.keys(), param_tuple[len(iterables) :]
+#                     )
+#                 },
+#                 **fixed_kwargs,
+#             )
+#             for param_tuple in param_combinations
+#         )
+
+#     def run_parallel(
+#         func: callable,
+#         *args,
+#         n_jobs: int = -1,
+#         backend: str = "threading",
+#         verbose: bool = True,
+#         **kwargs,
+#     ) -> list[any]:
+#         """Runs a function for a list of parameters in parallel.
+
+#         Args:
+#             func (Callable): function to run in parallel
+#             *args: Positional arguments. Can be single values or iterables
+#             n_jobs (int, optional): Number of joblib workers. Defaults to -1
+#             backend (str, optional): joblib backend. Valid options are
+#                 `loky`,`threading`, `mutliprocessing` or `sequential`. Defaults to "threading"
+#             verbose (bool, optional): Show progress bar. Defaults to True
+#             **kwargs: Keyword arguments. Can be single values or iterables
+
+#         Returns:
+#             list[any]: Function output
+
+#         Examples:
+#             >>> # Single iterable argument
+#             >>> run_parallel(func, [1,2,3], fixed_arg=42)
+
+#             >>> # Multiple iterables in args and kwargs
+#             >>> run_parallel(func, [1,2,3], val=[7,8,9], fixed=42)
+
+#             >>> # Only kwargs iterables
+#             >>> run_parallel(func, x=[1,2,3], y=[4,5,6], fixed=42)
+#         """
+#         parallel_kwargs = {"n_jobs": n_jobs, "backend": backend, "verbose": 0}
+
+#         # Prepare and validate arguments
+#         iterables, fixed_args, iterable_kwargs, fixed_kwargs, first_iterable_len = _prepare_parallel_args(
+#             args, kwargs
+#         )
+
+#         # Create parameter combinations
+#         all_iterables = iterables + list(iterable_kwargs.values())
+#         param_combinations = list(zip(*all_iterables))
+
+#         # Execute with or without progress tracking
+#         if not verbose:
+#             return _execute_parallel_without_progress(
+#                 func, iterables, fixed_args, iterable_kwargs, fixed_kwargs,
+#                 param_combinations, parallel_kwargs
+#             )
+#         else:
+#             return _execute_parallel_with_progress(
+#                 func, iterables, fixed_args, iterable_kwargs, fixed_kwargs,
+#                 param_combinations, parallel_kwargs
+#             )
+
+# else:
+
+#     def run_parallel(*args, **kwargs):
+#         raise ImportError("joblib not installed")
 
 
 def get_partitions_from_path(

{flowerpower-0.31.0.dist-info → flowerpower-0.31.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: FlowerPower
-Version: 0.31.0
+Version: 0.31.1
 Summary: A simple workflow framework for building and managing data processing pipelines
 Author-email: "Volker L." <ligno.blades@gmail.com>
 Project-URL: Homepage, https://github.com/legout/flowerpower

{flowerpower-0.31.0.dist-info → flowerpower-0.31.1.dist-info}/RECORD

@@ -40,14 +40,14 @@ flowerpower/utils/config.py,sha256=JC_kKq9-VJrW7ndWiehm5s-C0JG4_4oBffuwhVbaibQ,1
 flowerpower/utils/executor.py,sha256=AqO1RLxI9uKWX9OPu9uK6yOX2CNQMptRKpvcG2o-tak,6258
 flowerpower/utils/filesystem.py,sha256=W5kpJ0IERSsI3hQd7XFtbuk9bcIXidl7Qgw2bQVOkd0,6346
 flowerpower/utils/logging.py,sha256=WUnpoEmr9H5MGFF68jlshhEJcuMtW49M7VVRstmqIAg,1192
-flowerpower/utils/misc.py,sha256=iRfGHxVYCBqk5SsydR_jKBvsGXmI1mOqc_1UyNJKjr4,14236
+flowerpower/utils/misc.py,sha256=_BbAXmcbhDm5Y-RpW-HhJ5txu8KoaDQjdmRJwzNs7aM,14649
 flowerpower/utils/monkey.py,sha256=vJMYANjZI13PNbEQThdX0EFP1_6bGNHgnpF7HwReNyM,58
 flowerpower/utils/open_telemetry.py,sha256=fQWJWbIQFtKIxMBjAWeF12NGnqT0isO3A3j-DSOv_vE,949
 flowerpower/utils/security.py,sha256=7TPwD4nqjVm2rFBjJ0qM86wEQql-9j3Jo8udV3mSBoA,7113
 flowerpower/utils/templates.py,sha256=ouyEeSDqa9PjW8c32fGpcINlpC0WToawRFZkMPtwsLE,1591
-flowerpower-0.31.0.dist-info/licenses/LICENSE,sha256=9AkLexxrmr0aBgSHiqxpJk9wgazpP1CTJyiDyr56J9k,1063
-flowerpower-0.31.0.dist-info/METADATA,sha256=z11b6gBDVqqNGH3y_SF0n_xgJbRV0qd69TmtTo2pJqg,17202
-flowerpower-0.31.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-flowerpower-0.31.0.dist-info/entry_points.txt,sha256=61X11i5a2IwC9LBiP20XCDl5zMOigGCjMCx17B7bDbQ,52
-flowerpower-0.31.0.dist-info/top_level.txt,sha256=VraH4WtEUfSxs5L-rXwDQhzQb9eLHTUtgvmFZ2dAYnA,12
-flowerpower-0.31.0.dist-info/RECORD,,
+flowerpower-0.31.1.dist-info/licenses/LICENSE,sha256=9AkLexxrmr0aBgSHiqxpJk9wgazpP1CTJyiDyr56J9k,1063
+flowerpower-0.31.1.dist-info/METADATA,sha256=TgvnINYIvNKmijCZg0U11cHEFzC2eEA3LzO_8YuwdFc,17202
+flowerpower-0.31.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+flowerpower-0.31.1.dist-info/entry_points.txt,sha256=61X11i5a2IwC9LBiP20XCDl5zMOigGCjMCx17B7bDbQ,52
+flowerpower-0.31.1.dist-info/top_level.txt,sha256=VraH4WtEUfSxs5L-rXwDQhzQb9eLHTUtgvmFZ2dAYnA,12
+flowerpower-0.31.1.dist-info/RECORD,,