FlowerPower 0.11.5.5__tar.gz → 0.11.5.6__tar.gz

{flowerpower-0.11.5.5/src/FlowerPower.egg-info → flowerpower-0.11.5.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: FlowerPower
-Version: 0.11.5.5
+Version: 0.11.5.6
 Summary: A simple workflow framework. Hamilton + APScheduler = FlowerPower
 Author-email: "Volker L." <ligno.blades@gmail.com>
 Project-URL: Homepage, https://github.com/legout/flowerpower

{flowerpower-0.11.5.5 → flowerpower-0.11.5.6}/pyproject.toml

@@ -4,7 +4,7 @@ description = "A simple workflow framework. Hamilton + APScheduler = FlowerPower
 authors = [{ name = "Volker L.", email = "ligno.blades@gmail.com" }]
 readme = "README.md"
 requires-python = ">= 3.11"
-version = "0.11.5.5"
+version = "0.11.5.6"
 keywords = [
   "hamilton",
   "workflow",

{flowerpower-0.11.5.5 → flowerpower-0.11.5.6/src/FlowerPower.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: FlowerPower
-Version: 0.11.5.5
+Version: 0.11.5.6
 Summary: A simple workflow framework. Hamilton + APScheduler = FlowerPower
 Author-email: "Volker L." <ligno.blades@gmail.com>
 Project-URL: Homepage, https://github.com/legout/flowerpower

{flowerpower-0.11.5.5 → flowerpower-0.11.5.6}/src/flowerpower/plugins/io/base.py

@@ -99,14 +99,6 @@ class BaseFileIO(msgspec.Struct, gc=False):
             else self.fs.fs.storage_options
         )
 
-        # self.path = (
-        #    self._raw_path.replace(protocol + "://", "")
-        #    .replace(f"**/*.{self.format}", "")
-        #    .replace("**", "")
-        #    .replace("*", "")
-        #    .rstrip("/")
-        # )
-
     @property
     def protocol(self):
         """Get the protocol of the filesystem."""

{flowerpower-0.11.5.5 → flowerpower-0.11.5.6}/src/flowerpower/plugins/io/loader/deltatable.py

@@ -42,11 +42,11 @@ class DeltaTableReader(BaseDatasetReader, gc=False):
     def _init_dt(self):
         try:
             self.delta_table = DeltaTable(
-                self._raw_path,
+                self._base_path,
                 storage_options=self.storage_options.to_object_store_kwargs(),
             )
         except TableNotFoundError:
-            logger.warning(f"Table {self._raw_path} not found.")
+            logger.warning(f"Table {self._base_path} not found.")
             self.delta_table = None
 
     @property
@@ -77,7 +77,7 @@ class DeltaTableReader(BaseDatasetReader, gc=False):
             self._dataset = self.delta_table.to_pyarrow_dataset()
         if metadata:
             metadata = get_pyarrow_dataset_metadata(
-                self._dataset, self._raw_path, "parquet"
+                self._dataset, self._base_path, "parquet"
             )
             return self._dataset, metadata
         return self._dataset
@@ -102,7 +102,7 @@ class DeltaTableReader(BaseDatasetReader, gc=False):
         if reload or not hasattr(self, "_data"):
             self._data = self.delta_table.to_pyarrow_table()
         if metadata:
-            metadata = get_dataframe_metadata(table, self._raw_path, "parquet")
+            metadata = get_dataframe_metadata(table, self._base_path, "parquet")
             return self._data, metadata
         return self._data
 
@@ -131,7 +131,7 @@ class DeltaTableReader(BaseDatasetReader, gc=False):
 
         if self.with_lock:
             with RedisLock(
-                lock_name=self._raw_path,
+                lock_name=self._base_path,
                 namespace="flowerpower",
                 client=self.redis,
                 expire=10,
@@ -169,7 +169,7 @@ class DeltaTableReader(BaseDatasetReader, gc=False):
 
         if self.with_lock:
             with RedisLock(
-                lock_name=self._raw_path,
+                lock_name=self._base_path,
                 namespace="flowerpower",
                 client=self.redis,
                 expire=10,
@@ -183,8 +183,8 @@ class DeltaTableReader(BaseDatasetReader, gc=False):
     @property
     def metadata(self) -> dict:
         if not hasattr(self, "_metadata"):
-            self._metadata = get_delta_metadata(self.delta_table, self._raw_path)
+            self._metadata = get_delta_metadata(self.delta_table, self._base_path)
         return self._metadata
         if not hasattr(self, "_metadata"):
-            self._metadata = get_delta_metadata(self.delta_table, self._raw_path)
+            self._metadata = get_delta_metadata(self.delta_table, self._base_path)
         return self._metadata

{flowerpower-0.11.5.5 → flowerpower-0.11.5.6}/src/flowerpower/plugins/io/saver/deltatable.py

@@ -134,7 +134,9 @@ class DeltaTableWriter(BaseDatasetWriter, gc=False):
         if isinstance(data[0], pa.Table):
             data = pa.concat_tables(data, promote_options="permissive")
 
-        metadata = get_dataframe_metadata(data, path=self._raw_path, format=self.format)
+        metadata = get_dataframe_metadata(
+            data, path=self._base_path, format=self.format
+        )
 
         writer_properties = WriterProperties(
             data_page_size_limit=data_page_size_limit,
@@ -151,7 +153,7 @@ class DeltaTableWriter(BaseDatasetWriter, gc=False):
 
         def _write():
             write_deltalake(
-                self._raw_path,
+                self._base_path,
                 data,
                 mode=mode,
                 schema=schema or self.schema_,
@@ -171,7 +173,7 @@ class DeltaTableWriter(BaseDatasetWriter, gc=False):
 
         if self.with_lock:
             with RedisLock(
-                lock_name=self._raw_path,
+                lock_name=self._base_path,
                 namespace="flowerpower",
                 client=self.redis,
                 expire=10,