FlowerPower 0.11.5.6__tar.gz → 0.11.5.8__tar.gz

{flowerpower-0.11.5.6/src/FlowerPower.egg-info → flowerpower-0.11.5.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: FlowerPower
-Version: 0.11.5.6
+Version: 0.11.5.8
 Summary: A simple workflow framework. Hamilton + APScheduler = FlowerPower
 Author-email: "Volker L." <ligno.blades@gmail.com>
 Project-URL: Homepage, https://github.com/legout/flowerpower

{flowerpower-0.11.5.6 → flowerpower-0.11.5.8}/pyproject.toml

@@ -4,7 +4,7 @@ description = "A simple workflow framework. Hamilton + APScheduler = FlowerPower
 authors = [{ name = "Volker L.", email = "ligno.blades@gmail.com" }]
 readme = "README.md"
 requires-python = ">= 3.11"
-version = "0.11.5.6"
+version = "0.11.5.8"
 keywords = [
   "hamilton",
   "workflow",

{flowerpower-0.11.5.6 → flowerpower-0.11.5.8/src/FlowerPower.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: FlowerPower
-Version: 0.11.5.6
+Version: 0.11.5.8
 Summary: A simple workflow framework. Hamilton + APScheduler = FlowerPower
 Author-email: "Volker L." <ligno.blades@gmail.com>
 Project-URL: Homepage, https://github.com/legout/flowerpower

{flowerpower-0.11.5.6 → flowerpower-0.11.5.8}/src/flowerpower/plugins/io/base.py

@@ -365,7 +365,7 @@ class BaseFileReader(BaseFileIO, gc=False):
             df = [df.lazy() for df in self._to_polars_dataframe()]
 
         else:
-            df = self._to_polars_dataframe.lazy()
+            df = self._to_polars_dataframe().lazy()
         if metadata:
             metadata = get_dataframe_metadata(df, path=self.path, format=self.format)
             return df, metadata
@@ -1391,7 +1391,6 @@ class BaseDatasetWriter(BaseFileWriter, gc=False):
                 mode=mode or self.mode,
                 basename=basename or self.basename,
                 schema=schema or self.schema_,
-                partition_flavor=partitioning_flavor or self.partitioning_flavor,
                 partition_by=partition_by or self.partition_by,
                 compression=compression or self.compression,
                 row_group_size=row_group_size or self.row_group_size,

{flowerpower-0.11.5.6 → flowerpower-0.11.5.8}/src/flowerpower/plugins/io/metadata.py

@@ -1,5 +1,6 @@
 import datetime as dt
 import importlib
+import os
 
 import duckdb
 import pandas as pd
@@ -112,10 +113,8 @@ def get_dataframe_metadata(
         "num_columns": len(schema),
         "num_rows": num_rows,
         "num_files": num_files,
-        "name": kwargs.get("name", None),
-        "description": kwargs.get("description", None),
-        "id": kwargs.get("id", None),
     }
+    metadata.update(kwargs)
     return {k: v for k, v in metadata.items() if v is not None}
 
 
@@ -161,10 +160,8 @@ def get_duckdb_metadata(
         "num_columns": shape[1] if shape else None,
         "num_rows": shape[0] if shape else None,
         "num_files": len(fs.glob(path)) if include_num_files else None,
-        "name": kwargs.get("name", None),
-        "description": kwargs.get("description", None),
-        "id": kwargs.get("id", None),
     }
+    metadata.update(kwargs)
     return {k: v for k, v in metadata.items() if v is not None}
 
 
@@ -175,19 +172,19 @@ def get_pyarrow_dataset_metadata(
     **kwargs,
 ) -> dict:
     schema = get_serializable_schema(ds.schema)
+    files = ds.files
+
     metadata = {
-        "path": path,
+        "path": path or os.path.dirname(files[0]),
         "format": format,
         "timestamp": dt.datetime.now().timestamp(),
         "schema": schema,
         "partition_columns": ds.partitioning.schema.names if ds.partitioning else None,
         "num_columns": len(ds.schema),
         "num_rows": None,
-        "num_files": len(ds.files),
-        "name": kwargs.get("name", None),
-        "description": kwargs.get("description", None),
-        "id": kwargs.get("id", None),
+        "num_files": len(files),
     }
+    metadata.update(kwargs)
     return metadata
 
 

{flowerpower-0.11.5.6 → flowerpower-0.11.5.8}/src/flowerpower/plugins/io/saver/deltatable.py

@@ -60,14 +60,14 @@ class DeltaTableWriter(BaseDatasetWriter, gc=False):
         )
         | None = None,
         mode: str = "append",  # "overwrite" | "append" | "error | "ignore"
-        schema: pa.Schema | None = None,
+        # schema: pa.Schema | None = None,
         schema_mode: str | None = None,  # "merge" | "overwrite"
         partition_by: list[str] | None = None,
-        partition_filters: list[tuple[str, str, Any]] | None = None,
+        # partition_filters: list[tuple[str, str, Any]] | None = None,
         predicate: str | None = None,
         target_file_size: int | None = None,
-        large_dtypes: bool = False,
-        custom_metadata: dict[str, Any] | None = None,
+        # large_dtypes: bool = False,
+        # custom_metadata: dict[str, Any] | None = None,
         post_commithook_properties: PostCommitHookProperties | None = None,
         commit_properties: CommitProperties | None = None,
         # writerproperties
@@ -156,16 +156,16 @@ class DeltaTableWriter(BaseDatasetWriter, gc=False):
                 self._base_path,
                 data,
                 mode=mode,
-                schema=schema or self.schema_,
+                # schema=schema or self.schema_,
                 partition_by=partition_by or self.partition_by,
                 storage_options=self.storage_options.to_object_store_kwargs(),
                 description=self.description,
                 schema_mode=schema_mode,
-                partition_filters=partition_filters,
+                # partition_filters=partition_filters,
                 predicate=predicate,
                 target_file_size=target_file_size,
-                large_dtypes=large_dtypes,
-                custom_metadata=custom_metadata,
+                # large_dtypes=large_dtypes,
+                # custom_metadata=custom_metadata,
                 post_commithook_properties=post_commithook_properties,
                 commit_properties=commit_properties,
                 writer_properties=writer_properties,