FlowerPower 0.11.6.7__tar.gz → 0.11.6.9__tar.gz

{flowerpower-0.11.6.7/src/FlowerPower.egg-info → flowerpower-0.11.6.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: FlowerPower
-Version: 0.11.6.7
+Version: 0.11.6.9
 Summary: A simple workflow framework. Hamilton + APScheduler = FlowerPower
 Author-email: "Volker L." <ligno.blades@gmail.com>
 Project-URL: Homepage, https://github.com/legout/flowerpower

{flowerpower-0.11.6.7 → flowerpower-0.11.6.9}/pyproject.toml

@@ -4,7 +4,7 @@ description = "A simple workflow framework. Hamilton + APScheduler = FlowerPower
 authors = [{ name = "Volker L.", email = "ligno.blades@gmail.com" }]
 readme = "README.md"
 requires-python = ">= 3.11"
-version = "0.11.6.7"
+version = "0.11.6.9"
 keywords = [
   "hamilton",
   "workflow",

{flowerpower-0.11.6.7 → flowerpower-0.11.6.9/src/FlowerPower.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: FlowerPower
-Version: 0.11.6.7
+Version: 0.11.6.9
 Summary: A simple workflow framework. Hamilton + APScheduler = FlowerPower
 Author-email: "Volker L." <ligno.blades@gmail.com>
 Project-URL: Homepage, https://github.com/legout/flowerpower

{flowerpower-0.11.6.7 → flowerpower-0.11.6.9}/src/flowerpower/fs/ext.py

@@ -949,10 +949,10 @@ def _read_parquet(
     if concat:
         # Unify schemas before concatenation if opt_dtypes or multiple tables
         if isinstance(tables, list):
-            if len(tables) > 1:
-                schemas = [t.schema for t in tables]
-                unified_schema = unify_schemas_pa(schemas)
-                tables = [cast_schema(t, unified_schema) for t in tables]
+            # if len(tables) > 1:
+            #    schemas = [t.schema for t in tables]
+            #    unified_schema = unify_schemas_pa(schemas)
+            #    tables = [cast_schema(t, unified_schema) for t in tables]
             result = pa.concat_tables(
                 [table for table in tables if table.num_rows > 0],
                 promote_options="permissive",

{flowerpower-0.11.6.7 → flowerpower-0.11.6.9}/src/flowerpower/plugins/io/base.py

@@ -185,7 +185,7 @@ class BaseFileReader(BaseFileIO, gc=False):
     include_file_path: bool = field(default=False)
     concat: bool = field(default=True)
     batch_size: int | None = field(default=None)
-    opt_dtypes: bool = field(default=True)
+    opt_dtypes: bool = field(default=False)
     use_threads: bool = field(default=True)
     conn: duckdb.DuckDBPyConnection | None = field(default=None)
     ctx: datafusion.SessionContext | None = field(default=None)

{flowerpower-0.11.6.7 → flowerpower-0.11.6.9}/src/flowerpower/utils/misc.py

@@ -266,6 +266,8 @@ else:
 
 if importlib.util.find_spec("joblib"):
     from joblib import Parallel, delayed
+    from rich.progress import (BarColumn, Progress, TextColumn,
+                               TimeElapsedColumn)
 
     def run_parallel(
         func: callable,
@@ -299,19 +301,15 @@ if importlib.util.find_spec("joblib"):
             >>> # Only kwargs iterables
             >>> run_parallel(func, x=[1,2,3], y=[4,5,6], fixed=42)
         """
-        # Special kwargs for run_parallel itself
         parallel_kwargs = {"n_jobs": n_jobs, "backend": backend, "verbose": 0}
 
-        # Process args and kwargs to separate iterables and fixed values
         iterables = []
         fixed_args = []
         iterable_kwargs = {}
         fixed_kwargs = {}
 
-        # Get the length of the first iterable to determine number of iterations
         first_iterable_len = None
 
-        # Process args
         for arg in args:
             if isinstance(arg, (list, tuple)) and not isinstance(arg[0], (list, tuple)):
                 iterables.append(arg)
@@ -324,7 +322,6 @@ if importlib.util.find_spec("joblib"):
             else:
                 fixed_args.append(arg)
 
-        # Process kwargs
         for key, value in kwargs.items():
             if isinstance(value, (list, tuple)) and not isinstance(
                 value[0], (list, tuple)
@@ -342,26 +339,56 @@ if importlib.util.find_spec("joblib"):
         if first_iterable_len is None:
             raise ValueError("At least one iterable argument is required")
 
-        # Create parameter combinations
         all_iterables = iterables + list(iterable_kwargs.values())
-        param_combinations = list(zip(*all_iterables))  # Convert to list for tqdm
-
-        # if verbose:
-        #    param_combinations = tqdm.tqdm(param_combinations)
-
-        return Parallel(**parallel_kwargs)(
-            delayed(func)(
-                *(list(param_tuple[: len(iterables)]) + fixed_args),
-                **{
-                    k: v
-                    for k, v in zip(
-                        iterable_kwargs.keys(), param_tuple[len(iterables) :]
-                    )
-                },
-                **fixed_kwargs,
+        param_combinations = list(zip(*all_iterables))
+
+        if not verbose:
+            return Parallel(**parallel_kwargs)(
+                delayed(func)(
+                    *(list(param_tuple[: len(iterables)]) + fixed_args),
+                    **{
+                        k: v
+                        for k, v in zip(
+                            iterable_kwargs.keys(), param_tuple[len(iterables) :]
+                        )
+                    },
+                    **fixed_kwargs,
+                )
+                for param_tuple in param_combinations
             )
-            for param_tuple in param_combinations
-        )
+        else:
+            results = [None] * len(param_combinations)
+            with Progress(
+                TextColumn("[progress.description]{task.description}"),
+                BarColumn(),
+                "[progress.percentage]{task.percentage:>3.0f}%",
+                TimeElapsedColumn(),
+                transient=True,
+            ) as progress:
+                task = progress.add_task(
+                    "Running in parallel...", total=len(param_combinations)
+                )
+
+                def wrapper(idx, param_tuple):
+                    res = func(
+                        *(list(param_tuple[: len(iterables)]) + fixed_args),
+                        **{
+                            k: v
+                            for k, v in zip(
+                                iterable_kwargs.keys(), param_tuple[len(iterables) :]
+                            )
+                        },
+                        **fixed_kwargs,
+                    )
+                    progress.update(task, advance=1)
+                    return idx, res
+
+                for idx, result in Parallel(**parallel_kwargs)(
+                    delayed(wrapper)(i, param_tuple)
+                    for i, param_tuple in enumerate(param_combinations)
+                ):
+                    results[idx] = result
+            return results
 
 else: