FlowerPower 0.11.5.6__py3-none-any.whl → 0.11.5.8__py3-none-any.whl

flowerpower/plugins/io/base.py

@@ -365,7 +365,7 @@ class BaseFileReader(BaseFileIO, gc=False):
             df = [df.lazy() for df in self._to_polars_dataframe()]
 
         else:
-            df = self._to_polars_dataframe.lazy()
+            df = self._to_polars_dataframe().lazy()
         if metadata:
             metadata = get_dataframe_metadata(df, path=self.path, format=self.format)
             return df, metadata
@@ -1391,7 +1391,6 @@ class BaseDatasetWriter(BaseFileWriter, gc=False):
                 mode=mode or self.mode,
                 basename=basename or self.basename,
                 schema=schema or self.schema_,
-                partition_flavor=partitioning_flavor or self.partitioning_flavor,
                 partition_by=partition_by or self.partition_by,
                 compression=compression or self.compression,
                 row_group_size=row_group_size or self.row_group_size,

flowerpower/plugins/io/metadata.py

@@ -1,5 +1,6 @@
 import datetime as dt
 import importlib
+import os
 
 import duckdb
 import pandas as pd
@@ -112,10 +113,8 @@ def get_dataframe_metadata(
         "num_columns": len(schema),
         "num_rows": num_rows,
         "num_files": num_files,
-        "name": kwargs.get("name", None),
-        "description": kwargs.get("description", None),
-        "id": kwargs.get("id", None),
     }
+    metadata.update(kwargs)
     return {k: v for k, v in metadata.items() if v is not None}
 
 
@@ -161,10 +160,8 @@ def get_duckdb_metadata(
         "num_columns": shape[1] if shape else None,
         "num_rows": shape[0] if shape else None,
         "num_files": len(fs.glob(path)) if include_num_files else None,
-        "name": kwargs.get("name", None),
-        "description": kwargs.get("description", None),
-        "id": kwargs.get("id", None),
     }
+    metadata.update(kwargs)
     return {k: v for k, v in metadata.items() if v is not None}
 
 
@@ -175,19 +172,19 @@ def get_pyarrow_dataset_metadata(
     **kwargs,
 ) -> dict:
     schema = get_serializable_schema(ds.schema)
+    files = ds.files
+
     metadata = {
-        "path": path,
+        "path": path or os.path.dirname(files[0]),
         "format": format,
         "timestamp": dt.datetime.now().timestamp(),
         "schema": schema,
         "partition_columns": ds.partitioning.schema.names if ds.partitioning else None,
         "num_columns": len(ds.schema),
         "num_rows": None,
-        "num_files": len(ds.files),
-        "name": kwargs.get("name", None),
-        "description": kwargs.get("description", None),
-        "id": kwargs.get("id", None),
+        "num_files": len(files),
     }
+    metadata.update(kwargs)
     return metadata
 
 

flowerpower/plugins/io/saver/deltatable.py

@@ -60,14 +60,14 @@ class DeltaTableWriter(BaseDatasetWriter, gc=False):
         )
         | None = None,
         mode: str = "append",  # "overwrite" | "append" | "error | "ignore"
-        schema: pa.Schema | None = None,
+        # schema: pa.Schema | None = None,
         schema_mode: str | None = None,  # "merge" | "overwrite"
         partition_by: list[str] | None = None,
-        partition_filters: list[tuple[str, str, Any]] | None = None,
+        # partition_filters: list[tuple[str, str, Any]] | None = None,
         predicate: str | None = None,
         target_file_size: int | None = None,
-        large_dtypes: bool = False,
-        custom_metadata: dict[str, Any] | None = None,
+        # large_dtypes: bool = False,
+        # custom_metadata: dict[str, Any] | None = None,
         post_commithook_properties: PostCommitHookProperties | None = None,
         commit_properties: CommitProperties | None = None,
         # writerproperties
@@ -156,16 +156,16 @@ class DeltaTableWriter(BaseDatasetWriter, gc=False):
                 self._base_path,
                 data,
                 mode=mode,
-                schema=schema or self.schema_,
+                # schema=schema or self.schema_,
                 partition_by=partition_by or self.partition_by,
                 storage_options=self.storage_options.to_object_store_kwargs(),
                 description=self.description,
                 schema_mode=schema_mode,
-                partition_filters=partition_filters,
+                # partition_filters=partition_filters,
                 predicate=predicate,
                 target_file_size=target_file_size,
-                large_dtypes=large_dtypes,
-                custom_metadata=custom_metadata,
+                # large_dtypes=large_dtypes,
+                # custom_metadata=custom_metadata,
                 post_commithook_properties=post_commithook_properties,
                 commit_properties=commit_properties,
                 writer_properties=writer_properties,

{flowerpower-0.11.5.6.dist-info → flowerpower-0.11.5.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: FlowerPower
-Version: 0.11.5.6
+Version: 0.11.5.8
 Summary: A simple workflow framework. Hamilton + APScheduler = FlowerPower
 Author-email: "Volker L." <ligno.blades@gmail.com>
 Project-URL: Homepage, https://github.com/legout/flowerpower

{flowerpower-0.11.5.6.dist-info → flowerpower-0.11.5.8.dist-info}/RECORD

@@ -44,8 +44,8 @@ flowerpower/pipeline/manager.py,sha256=KVpOclUEUAETUNJamJJGuKt3oxCaLitQgxWxkE1q0
 flowerpower/pipeline/registry.py,sha256=WWQoaxtgnlntFEIPQzYM1gk0zUXwrH2PmDLGbTzhrZs,18991
 flowerpower/pipeline/runner.py,sha256=dsSVYixFXqlxFk8EJfT4wV_7IwgkXq0ErwH_yf_NGS8,25654
 flowerpower/pipeline/visualizer.py,sha256=amjMrl5NetErE198HzZBPWVZBi_t5jj9ydxWpuNLoTI,5013
-flowerpower/plugins/io/base.py,sha256=rRQin2cdQaLlGpeqfZ2CTDT-vzVXn5lTDZ62hFiyRRE,79151
-flowerpower/plugins/io/metadata.py,sha256=31FoVyRz6zdWMWda0f1GHq0xMokVB3SVYrLT2TejOH8,7271
+flowerpower/plugins/io/base.py,sha256=pWQt7vS_n3jR3J9YaVgMCXQ6utWuo3uKYZdTJ3pLiuY,79071
+flowerpower/plugins/io/metadata.py,sha256=PCrepLilXRWKDsB5BKFF_-OFs712s1zBeitW-84lDLQ,7005
 flowerpower/plugins/io/helpers/datetime.py,sha256=1WBUg2ywcsodJQwoF6JiIGc9yhVobvE2IErWp4i95m4,10649
 flowerpower/plugins/io/helpers/polars.py,sha256=xWLjmZBUhbJPd0m5pkHreMSMeNLyJThJt0R7V7yzWEs,18068
 flowerpower/plugins/io/helpers/sql.py,sha256=BPIxjarKF3p93EdtUu-md8KislE9q8IWNSeZ5toFU6U,7298
@@ -64,7 +64,7 @@ flowerpower/plugins/io/loader/pydala.py,sha256=iCvHQCYAisrxZIaQzZrzTmWaND_LzamuM
 flowerpower/plugins/io/loader/sqlite.py,sha256=CCI2bPLMVYUOg7mxy54sbR15qwvKRGL8r_982jARUig,609
 flowerpower/plugins/io/saver/__init__.py,sha256=RWcUdrcmi2JR4mb77jPZd-lSG5rQBIqS7Pl4vbPfUes,780
 flowerpower/plugins/io/saver/csv.py,sha256=i_kZvuco5joBaBfidplWglQsNqO7v873j_Xo5f78S5M,736
-flowerpower/plugins/io/saver/deltatable.py,sha256=6FriPYrygaL1c5CMb0VWxnZvoJvgkeOLZdFAObgcnHI,7024
+flowerpower/plugins/io/saver/deltatable.py,sha256=9UWXkmbMvomFe8jWop5GRA9hyfhiAFWZbSNROXngWHQ,7040
 flowerpower/plugins/io/saver/duckdb.py,sha256=NpnoO5CffrNcCUPH_4KH4RFei8T40sHgjYySm0n1pV8,415
 flowerpower/plugins/io/saver/json.py,sha256=19a0K3aNQQxtSjskv4G6_5XfiFBoZnHGZAd_W3UgFSs,734
 flowerpower/plugins/io/saver/mqtt.py,sha256=vIlnRSMwPzyZzGbvi9TBX4j4Qn1iSbgUbrMU2giJSCU,573
@@ -93,9 +93,9 @@ flowerpower/utils/monkey.py,sha256=VPl3yimoWhwD9kI05BFsjNvtyQiDyLfY4Q85Bb6Ma0w,2
 flowerpower/utils/open_telemetry.py,sha256=fQWJWbIQFtKIxMBjAWeF12NGnqT0isO3A3j-DSOv_vE,949
 flowerpower/utils/scheduler.py,sha256=2zJ_xmLXpvXUQNF1XS2Gqm3Ogo907ctZ50GtvQB_rhE,9354
 flowerpower/utils/templates.py,sha256=ouyEeSDqa9PjW8c32fGpcINlpC0WToawRFZkMPtwsLE,1591
-flowerpower-0.11.5.6.dist-info/licenses/LICENSE,sha256=9AkLexxrmr0aBgSHiqxpJk9wgazpP1CTJyiDyr56J9k,1063
-flowerpower-0.11.5.6.dist-info/METADATA,sha256=eJclebBtAuEaUUe6Nvzu-YwiIIJ7N_ycN6aLNGIon0E,21612
-flowerpower-0.11.5.6.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-flowerpower-0.11.5.6.dist-info/entry_points.txt,sha256=61X11i5a2IwC9LBiP20XCDl5zMOigGCjMCx17B7bDbQ,52
-flowerpower-0.11.5.6.dist-info/top_level.txt,sha256=VraH4WtEUfSxs5L-rXwDQhzQb9eLHTUtgvmFZ2dAYnA,12
-flowerpower-0.11.5.6.dist-info/RECORD,,
+flowerpower-0.11.5.8.dist-info/licenses/LICENSE,sha256=9AkLexxrmr0aBgSHiqxpJk9wgazpP1CTJyiDyr56J9k,1063
+flowerpower-0.11.5.8.dist-info/METADATA,sha256=ADH8WX3HkFVTcdB-WmwvC1yN0mRcj_zp5EjyQYV3Zfk,21612
+flowerpower-0.11.5.8.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+flowerpower-0.11.5.8.dist-info/entry_points.txt,sha256=61X11i5a2IwC9LBiP20XCDl5zMOigGCjMCx17B7bDbQ,52
+flowerpower-0.11.5.8.dist-info/top_level.txt,sha256=VraH4WtEUfSxs5L-rXwDQhzQb9eLHTUtgvmFZ2dAYnA,12
+flowerpower-0.11.5.8.dist-info/RECORD,,