FlowerPower 0.11.6.7__py3-none-any.whl → 0.11.6.9__py3-none-any.whl

flowerpower/fs/ext.py

@@ -949,10 +949,10 @@ def _read_parquet(
     if concat:
         # Unify schemas before concatenation if opt_dtypes or multiple tables
         if isinstance(tables, list):
-            if len(tables) > 1:
-                schemas = [t.schema for t in tables]
-                unified_schema = unify_schemas_pa(schemas)
-                tables = [cast_schema(t, unified_schema) for t in tables]
+            # if len(tables) > 1:
+            #    schemas = [t.schema for t in tables]
+            #    unified_schema = unify_schemas_pa(schemas)
+            #    tables = [cast_schema(t, unified_schema) for t in tables]
             result = pa.concat_tables(
                 [table for table in tables if table.num_rows > 0],
                 promote_options="permissive",

flowerpower/plugins/io/base.py

@@ -185,7 +185,7 @@ class BaseFileReader(BaseFileIO, gc=False):
     include_file_path: bool = field(default=False)
     concat: bool = field(default=True)
     batch_size: int | None = field(default=None)
-    opt_dtypes: bool = field(default=True)
+    opt_dtypes: bool = field(default=False)
     use_threads: bool = field(default=True)
     conn: duckdb.DuckDBPyConnection | None = field(default=None)
     ctx: datafusion.SessionContext | None = field(default=None)

flowerpower/utils/misc.py

@@ -266,6 +266,8 @@ else:
 
 if importlib.util.find_spec("joblib"):
     from joblib import Parallel, delayed
+    from rich.progress import (BarColumn, Progress, TextColumn,
+                               TimeElapsedColumn)
 
     def run_parallel(
         func: callable,
@@ -299,19 +301,15 @@ if importlib.util.find_spec("joblib"):
             >>> # Only kwargs iterables
             >>> run_parallel(func, x=[1,2,3], y=[4,5,6], fixed=42)
         """
-        # Special kwargs for run_parallel itself
         parallel_kwargs = {"n_jobs": n_jobs, "backend": backend, "verbose": 0}
 
-        # Process args and kwargs to separate iterables and fixed values
         iterables = []
         fixed_args = []
         iterable_kwargs = {}
         fixed_kwargs = {}
 
-        # Get the length of the first iterable to determine number of iterations
         first_iterable_len = None
 
-        # Process args
         for arg in args:
             if isinstance(arg, (list, tuple)) and not isinstance(arg[0], (list, tuple)):
                 iterables.append(arg)
@@ -324,7 +322,6 @@ if importlib.util.find_spec("joblib"):
             else:
                 fixed_args.append(arg)
 
-        # Process kwargs
         for key, value in kwargs.items():
             if isinstance(value, (list, tuple)) and not isinstance(
                 value[0], (list, tuple)
@@ -342,26 +339,56 @@ if importlib.util.find_spec("joblib"):
         if first_iterable_len is None:
             raise ValueError("At least one iterable argument is required")
 
-        # Create parameter combinations
         all_iterables = iterables + list(iterable_kwargs.values())
-        param_combinations = list(zip(*all_iterables))  # Convert to list for tqdm
-
-        # if verbose:
-        #    param_combinations = tqdm.tqdm(param_combinations)
-
-        return Parallel(**parallel_kwargs)(
-            delayed(func)(
-                *(list(param_tuple[: len(iterables)]) + fixed_args),
-                **{
-                    k: v
-                    for k, v in zip(
-                        iterable_kwargs.keys(), param_tuple[len(iterables) :]
-                    )
-                },
-                **fixed_kwargs,
+        param_combinations = list(zip(*all_iterables))
+
+        if not verbose:
+            return Parallel(**parallel_kwargs)(
+                delayed(func)(
+                    *(list(param_tuple[: len(iterables)]) + fixed_args),
+                    **{
+                        k: v
+                        for k, v in zip(
+                            iterable_kwargs.keys(), param_tuple[len(iterables) :]
+                        )
+                    },
+                    **fixed_kwargs,
+                )
+                for param_tuple in param_combinations
             )
-            for param_tuple in param_combinations
-        )
+        else:
+            results = [None] * len(param_combinations)
+            with Progress(
+                TextColumn("[progress.description]{task.description}"),
+                BarColumn(),
+                "[progress.percentage]{task.percentage:>3.0f}%",
+                TimeElapsedColumn(),
+                transient=True,
+            ) as progress:
+                task = progress.add_task(
+                    "Running in parallel...", total=len(param_combinations)
+                )
+
+                def wrapper(idx, param_tuple):
+                    res = func(
+                        *(list(param_tuple[: len(iterables)]) + fixed_args),
+                        **{
+                            k: v
+                            for k, v in zip(
+                                iterable_kwargs.keys(), param_tuple[len(iterables) :]
+                            )
+                        },
+                        **fixed_kwargs,
+                    )
+                    progress.update(task, advance=1)
+                    return idx, res
+
+                for idx, result in Parallel(**parallel_kwargs)(
+                    delayed(wrapper)(i, param_tuple)
+                    for i, param_tuple in enumerate(param_combinations)
+                ):
+                    results[idx] = result
+            return results
 
 else:
 

{flowerpower-0.11.6.7.dist-info → flowerpower-0.11.6.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: FlowerPower
-Version: 0.11.6.7
+Version: 0.11.6.9
 Summary: A simple workflow framework. Hamilton + APScheduler = FlowerPower
 Author-email: "Volker L." <ligno.blades@gmail.com>
 Project-URL: Homepage, https://github.com/legout/flowerpower

{flowerpower-0.11.6.7.dist-info → flowerpower-0.11.6.9.dist-info}/RECORD

@@ -18,7 +18,7 @@ flowerpower/cli/pipeline.py,sha256=60P6u_QOSgp0jJXEMxazEEo5Sh7-SWFo-Kkuaz21YuI,3
 flowerpower/cli/utils.py,sha256=nDSSj_1nlYlMmj252kRZeohhFqHv9yvdgDEduQCyWOc,5152
 flowerpower/fs/__init__.py,sha256=uZaPXErEfQqQRbKRIjkB9yiygd45X5_psYn9-VVrBTQ,910
 flowerpower/fs/base.py,sha256=TqgqBsaFj13O1NpAr8kHuGJ9CTlaSWViMB8Ai_iuCjs,22761
-flowerpower/fs/ext.py,sha256=jshry-nBIjasijqrZZsqVn6QyB8Zh3amt8v1lWk0hEo,70205
+flowerpower/fs/ext.py,sha256=RXEu2VYl1I1V2kb4kwny2nEzUByBzWFH81mMH8HNoTk,70210
 flowerpower/fs/storage_options.py,sha256=msq5TpxAU8tcE_Bxjw6SyxaFa75UjdYnR4-O9U2wmbk,48034
 flowerpower/job_queue/__init__.py,sha256=a25hIqv2xoFKb4JZlyUukS0ppZ9-2sJKH3XAvbk3rlk,10788
 flowerpower/job_queue/base.py,sha256=YwLunDQSyqkSU_vJ69C5SSybJeJP1bAiZ3teUtOchxA,13640
@@ -44,7 +44,7 @@ flowerpower/pipeline/manager.py,sha256=KVpOclUEUAETUNJamJJGuKt3oxCaLitQgxWxkE1q0
 flowerpower/pipeline/registry.py,sha256=6ngmHyKyQsxvIO4qRYxljedY0BE1wE3lpfksEGOzjNs,18963
 flowerpower/pipeline/runner.py,sha256=dsSVYixFXqlxFk8EJfT4wV_7IwgkXq0ErwH_yf_NGS8,25654
 flowerpower/pipeline/visualizer.py,sha256=amjMrl5NetErE198HzZBPWVZBi_t5jj9ydxWpuNLoTI,5013
-flowerpower/plugins/io/base.py,sha256=_MruHw-eC67AHm6pieClLQ1X3uFvHg1uqIWsBuZOZos,97097
+flowerpower/plugins/io/base.py,sha256=oGxTKobs0M19hPV842EelAeJ01EBz6kDdGv_4GTyFzk,97098
 flowerpower/plugins/io/metadata.py,sha256=PCrepLilXRWKDsB5BKFF_-OFs712s1zBeitW-84lDLQ,7005
 flowerpower/plugins/io/helpers/datetime.py,sha256=1WBUg2ywcsodJQwoF6JiIGc9yhVobvE2IErWp4i95m4,10649
 flowerpower/plugins/io/helpers/polars.py,sha256=cuzMby0a90AMFXhNEycf53UOwdHw4uxnx322l3m7jB0,27443
@@ -89,14 +89,14 @@ flowerpower/settings/logging.py,sha256=xzoiM-87WGduC-RJu8RfyeweHGTM7SnHXHQPbxKwe
 flowerpower/settings/retry.py,sha256=W3AAVSxmTUAeeSbzRGA37RxqtKKyUdi2MkwDzCsXP94,181
 flowerpower/utils/callback.py,sha256=sGYSrEbnl0xfRa1X-mA-om3erpH7pmpsWdyPQ9HpU7E,6736
 flowerpower/utils/logging.py,sha256=QO1Q3pv-m3mK2xT_VbkjTsomFnDeuqBmRuOtoxaowgg,864
-flowerpower/utils/misc.py,sha256=7VKlmso0jyJwOZ2VR1mPIiN_3LHpbJ0nJ_gkgVzJGWg,15986
+flowerpower/utils/misc.py,sha256=SK913FEYmOjS7GfSQ8oXNFiQ7owrH70fxp952vN5aWg,17099
 flowerpower/utils/monkey.py,sha256=VPl3yimoWhwD9kI05BFsjNvtyQiDyLfY4Q85Bb6Ma0w,2903
 flowerpower/utils/open_telemetry.py,sha256=fQWJWbIQFtKIxMBjAWeF12NGnqT0isO3A3j-DSOv_vE,949
 flowerpower/utils/scheduler.py,sha256=2zJ_xmLXpvXUQNF1XS2Gqm3Ogo907ctZ50GtvQB_rhE,9354
 flowerpower/utils/templates.py,sha256=ouyEeSDqa9PjW8c32fGpcINlpC0WToawRFZkMPtwsLE,1591
-flowerpower-0.11.6.7.dist-info/licenses/LICENSE,sha256=9AkLexxrmr0aBgSHiqxpJk9wgazpP1CTJyiDyr56J9k,1063
-flowerpower-0.11.6.7.dist-info/METADATA,sha256=GVpO7ezPrWQ-YOnadysNNaOKVUOvzJR3GwseHdlALQY,21612
-flowerpower-0.11.6.7.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-flowerpower-0.11.6.7.dist-info/entry_points.txt,sha256=61X11i5a2IwC9LBiP20XCDl5zMOigGCjMCx17B7bDbQ,52
-flowerpower-0.11.6.7.dist-info/top_level.txt,sha256=VraH4WtEUfSxs5L-rXwDQhzQb9eLHTUtgvmFZ2dAYnA,12
-flowerpower-0.11.6.7.dist-info/RECORD,,
+flowerpower-0.11.6.9.dist-info/licenses/LICENSE,sha256=9AkLexxrmr0aBgSHiqxpJk9wgazpP1CTJyiDyr56J9k,1063
+flowerpower-0.11.6.9.dist-info/METADATA,sha256=7NW4WfKaqeQN2ahn6Y9u_cJmRr5xMWfNcJzHjlJeqjk,21612
+flowerpower-0.11.6.9.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+flowerpower-0.11.6.9.dist-info/entry_points.txt,sha256=61X11i5a2IwC9LBiP20XCDl5zMOigGCjMCx17B7bDbQ,52
+flowerpower-0.11.6.9.dist-info/top_level.txt,sha256=VraH4WtEUfSxs5L-rXwDQhzQb9eLHTUtgvmFZ2dAYnA,12
+flowerpower-0.11.6.9.dist-info/RECORD,,