FlowerPower 0.11.6.4__py3-none-any.whl → 0.11.6.6__py3-none-any.whl

flowerpower/fs/ext.py

@@ -923,7 +923,10 @@ def _read_parquet(
                 schemas = [t.schema for t in tables]
                 unified_schema = unify_schemas_pa(schemas)
                 tables = [cast_schema(t, unified_schema) for t in tables]
-            result = pa.concat_tables(tables, promote_options="permissive")
+            result = pa.concat_tables(
+                [table for table in tables if table.num_rows > 0],
+                promote_options="permissive",
+            )
             # if opt_dtypes:
             #    result = opt_dtype_pa(result, strict=False)
             return result
@@ -932,7 +935,10 @@ def _read_parquet(
             #    tables = opt_dtype_pa(tables, strict=False)
             return tables
         else:
-            return pa.concat_tables(tables, promote_options="permissive")
+            return pa.concat_tables(
+                [table for table in tables if table.num_rows > 0],
+                promote_options="permissive",
+            )
     return tables
 
 
@@ -1052,7 +1058,10 @@ def _read_parquet_batches(
                 schemas = [t.schema for t in batch_tables]
                 unified_schema = unify_schemas_pa(schemas)
                 batch_tables = [cast_schema(t, unified_schema) for t in batch_tables]
-            result = pa.concat_tables(batch_tables, promote_options="permissive")
+            result = pa.concat_tables(
+                [table for table in batch_tables if table.num_rows > 0],
+                promote_options="permissive",
+            )
             # if opt_dtypes:
             #    result = opt_dtype_pa(result, strict=False)
             yield result

flowerpower/plugins/io/helpers/polars.py

@@ -68,20 +68,20 @@ def _optimize_string_column(
     cleaned_expr = _clean_string_expr(col_name)
     non_null = series.drop_nulls().replace({"-": None, "": None, "None": None})
     if len(non_null) == 0:
-        return pl.col(col_name).cast(pl.Int8)
+        return pl.col(col_name).cast(series.dtype)
 
     stripped = non_null.str.strip_chars()
     lowercase = stripped.str.to_lowercase()
 
     # Check for boolean values
-    if lowercase.str.contains(BOOLEAN_REGEX).all():
+    if lowercase.str.contains(BOOLEAN_REGEX).all(ignore_nulls=False):
         return (
             cleaned_expr.str.to_lowercase()
             .str.contains(BOOLEAN_TRUE_REGEX)
             .alias(col_name)
         )
 
-    elif stripped.str.contains(INTEGER_REGEX).all():
+    elif stripped.str.contains(INTEGER_REGEX).all(ignore_nulls=False):
         int_expr = cleaned_expr.cast(pl.Int64)
         return (
             int_expr.shrink_dtype().alias(col_name)
@@ -90,7 +90,7 @@ def _optimize_string_column(
         )
 
     # Check for numeric values
-    elif stripped.str.contains(FLOAT_REGEX).all():
+    elif stripped.str.contains(FLOAT_REGEX).all(ignore_nulls=False):
         float_expr = cleaned_expr.str.replace_all(",", ".").cast(pl.Float64)
 
         if shrink_numerics:
@@ -104,7 +104,7 @@ def _optimize_string_column(
         return float_expr.alias(col_name)
 
     try:
-        if stripped.str.contains(DATETIME_REGEX).all():
+        if stripped.str.contains(DATETIME_REGEX).all(ignore_nulls=False):
             return cleaned_expr.str.to_datetime(
                 strict=False, time_unit="us", time_zone=time_zone
             ).alias(col_name)
@@ -123,7 +123,7 @@ def _get_column_expr(
 
     # Handle all-null columns
     if series.is_null().all():
-        return pl.col(col_name).cast(pl.Int8)
+        return pl.col(col_name).cast(series.dtype)
 
     # Process based on current type
     if series.dtype.is_numeric():

flowerpower/plugins/io/helpers/pyarrow.py

@@ -255,7 +255,7 @@ def _all_match_regex(array: pa.Array, pattern: str) -> bool:
     Uses pyarrow.compute.match_substring_regex for vectorized evaluation.
     """
     if len(array) == 0 or array.null_count == len(array):
-        return True
+        return False
 
     # Check if al values match the pattern
     return pc.all(pc.match_substring_regex(array, pattern, ignore_case=True)).as_py()
@@ -272,7 +272,7 @@ def _optimize_string_array(
     if len(array) == 0:
         return pa.array([], type=pa.int8())
     if array.null_count == len(array):
-        return pa.array([None] * len(array), type=pa.int8())
+        return pa.array([None] * len(array), type=array.type)
 
     # Clean string values
     cleaned_array = _clean_string_array(array)
@@ -342,7 +342,7 @@ def _process_column(
 
     # Handle all-null columns
     if array.null_count == len(array):
-        return pa.array([None] * len(array), type=pa.int8())
+        return pa.array([None] * len(array), type=array.type)
 
     # Process based on current type
     if pa.types.is_floating(array.type) or pa.types.is_integer(array.type):

{flowerpower-0.11.6.4.dist-info → flowerpower-0.11.6.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: FlowerPower
-Version: 0.11.6.4
+Version: 0.11.6.6
 Summary: A simple workflow framework. Hamilton + APScheduler = FlowerPower
 Author-email: "Volker L." <ligno.blades@gmail.com>
 Project-URL: Homepage, https://github.com/legout/flowerpower

{flowerpower-0.11.6.4.dist-info → flowerpower-0.11.6.6.dist-info}/RECORD

@@ -18,7 +18,7 @@ flowerpower/cli/pipeline.py,sha256=60P6u_QOSgp0jJXEMxazEEo5Sh7-SWFo-Kkuaz21YuI,3
 flowerpower/cli/utils.py,sha256=nDSSj_1nlYlMmj252kRZeohhFqHv9yvdgDEduQCyWOc,5152
 flowerpower/fs/__init__.py,sha256=uZaPXErEfQqQRbKRIjkB9yiygd45X5_psYn9-VVrBTQ,910
 flowerpower/fs/base.py,sha256=TqgqBsaFj13O1NpAr8kHuGJ9CTlaSWViMB8Ai_iuCjs,22761
-flowerpower/fs/ext.py,sha256=2NmhSbCIL0qnONMRNPHcPUuR39bGjWpxJE4hNHU5Rvw,69044
+flowerpower/fs/ext.py,sha256=U-MX-OYsqhNzapcGVaGTifeMTZ5hY7VlAtQ8c8j5zi0,69314
 flowerpower/fs/storage_options.py,sha256=msq5TpxAU8tcE_Bxjw6SyxaFa75UjdYnR4-O9U2wmbk,48034
 flowerpower/job_queue/__init__.py,sha256=a25hIqv2xoFKb4JZlyUukS0ppZ9-2sJKH3XAvbk3rlk,10788
 flowerpower/job_queue/base.py,sha256=YwLunDQSyqkSU_vJ69C5SSybJeJP1bAiZ3teUtOchxA,13640
@@ -47,8 +47,8 @@ flowerpower/pipeline/visualizer.py,sha256=amjMrl5NetErE198HzZBPWVZBi_t5jj9ydxWpu
 flowerpower/plugins/io/base.py,sha256=_MruHw-eC67AHm6pieClLQ1X3uFvHg1uqIWsBuZOZos,97097
 flowerpower/plugins/io/metadata.py,sha256=PCrepLilXRWKDsB5BKFF_-OFs712s1zBeitW-84lDLQ,7005
 flowerpower/plugins/io/helpers/datetime.py,sha256=1WBUg2ywcsodJQwoF6JiIGc9yhVobvE2IErWp4i95m4,10649
-flowerpower/plugins/io/helpers/polars.py,sha256=GkiGVHDRduStAnaQSxqXmoT55D9nfxVqCHUSgf5hS0M,27361
-flowerpower/plugins/io/helpers/pyarrow.py,sha256=xsHDvEqUgb__ZFNXyfXHRccCxoW-62v4hmEj4NAA0XM,13900
+flowerpower/plugins/io/helpers/polars.py,sha256=cuzMby0a90AMFXhNEycf53UOwdHw4uxnx322l3m7jB0,27443
+flowerpower/plugins/io/helpers/pyarrow.py,sha256=NwA2NAPMIcGmaFE3gx1jKYW_-6gAxQ8Oczdgk4Av-s8,13903
 flowerpower/plugins/io/helpers/sql.py,sha256=BPIxjarKF3p93EdtUu-md8KislE9q8IWNSeZ5toFU6U,7298
 flowerpower/plugins/io/loader/__init__.py,sha256=MKH42nvVokaWas0wFgX1yrpU5iLpvHjRqqF-KzwLHCg,780
 flowerpower/plugins/io/loader/csv.py,sha256=Q5bmcbbr530sT1kQ2YiJwvsMUPqi0VcZWsLOygmzRyI,827
@@ -94,9 +94,9 @@ flowerpower/utils/monkey.py,sha256=VPl3yimoWhwD9kI05BFsjNvtyQiDyLfY4Q85Bb6Ma0w,2
 flowerpower/utils/open_telemetry.py,sha256=fQWJWbIQFtKIxMBjAWeF12NGnqT0isO3A3j-DSOv_vE,949
 flowerpower/utils/scheduler.py,sha256=2zJ_xmLXpvXUQNF1XS2Gqm3Ogo907ctZ50GtvQB_rhE,9354
 flowerpower/utils/templates.py,sha256=ouyEeSDqa9PjW8c32fGpcINlpC0WToawRFZkMPtwsLE,1591
-flowerpower-0.11.6.4.dist-info/licenses/LICENSE,sha256=9AkLexxrmr0aBgSHiqxpJk9wgazpP1CTJyiDyr56J9k,1063
-flowerpower-0.11.6.4.dist-info/METADATA,sha256=y90zVAntdH6BJpp1vM2magFQYMI3Tj_UsntYFI6dcj0,21612
-flowerpower-0.11.6.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-flowerpower-0.11.6.4.dist-info/entry_points.txt,sha256=61X11i5a2IwC9LBiP20XCDl5zMOigGCjMCx17B7bDbQ,52
-flowerpower-0.11.6.4.dist-info/top_level.txt,sha256=VraH4WtEUfSxs5L-rXwDQhzQb9eLHTUtgvmFZ2dAYnA,12
-flowerpower-0.11.6.4.dist-info/RECORD,,
+flowerpower-0.11.6.6.dist-info/licenses/LICENSE,sha256=9AkLexxrmr0aBgSHiqxpJk9wgazpP1CTJyiDyr56J9k,1063
+flowerpower-0.11.6.6.dist-info/METADATA,sha256=LaDZ3aj-zU2iPSpQxEJE3A9QrjCYLIvHxk_tCmP2ams,21612
+flowerpower-0.11.6.6.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+flowerpower-0.11.6.6.dist-info/entry_points.txt,sha256=61X11i5a2IwC9LBiP20XCDl5zMOigGCjMCx17B7bDbQ,52
+flowerpower-0.11.6.6.dist-info/top_level.txt,sha256=VraH4WtEUfSxs5L-rXwDQhzQb9eLHTUtgvmFZ2dAYnA,12
+flowerpower-0.11.6.6.dist-info/RECORD,,