FlowerPower 0.11.3__tar.gz → 0.11.5__tar.gz

{flowerpower-0.11.3/src/FlowerPower.egg-info → flowerpower-0.11.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: FlowerPower
-Version: 0.11.3
+Version: 0.11.5
 Summary: A simple workflow framework. Hamilton + APScheduler = FlowerPower
 Author-email: "Volker L." <ligno.blades@gmail.com>
 Project-URL: Homepage, https://github.com/legout/flowerpower

{flowerpower-0.11.3 → flowerpower-0.11.5}/pyproject.toml

@@ -4,7 +4,7 @@ description = "A simple workflow framework. Hamilton + APScheduler = FlowerPower
 authors = [{ name = "Volker L.", email = "ligno.blades@gmail.com" }]
 readme = "README.md"
 requires-python = ">= 3.11"
-version = "0.11.3"
+version = "0.11.5"
 keywords = [
   "hamilton",
   "workflow",

{flowerpower-0.11.3 → flowerpower-0.11.5/src/FlowerPower.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: FlowerPower
-Version: 0.11.3
+Version: 0.11.5
 Summary: A simple workflow framework. Hamilton + APScheduler = FlowerPower
 Author-email: "Volker L." <ligno.blades@gmail.com>
 Project-URL: Homepage, https://github.com/legout/flowerpower

{flowerpower-0.11.3 → flowerpower-0.11.5}/src/flowerpower/fs/base.py

@@ -608,7 +608,15 @@ def get_filesystem(
             return DirFileSystem(path=path, fs=fs)
 
     pp = infer_storage_options(str(path) if isinstance(path, Path) else path)
-    protocol = pp.get("protocol")
+    protocol = (
+        storage_options_kwargs.get("protocol", None)
+        or (
+            storage_options.get("protocol", None)
+            if isinstance(storage_options, dict)
+            else getattr(storage_options, "protocol", None)
+        )
+        or pp.get("protocol", "file")
+    )
 
     if protocol == "file" or protocol == "local":
         fs = filesystem(protocol)

{flowerpower-0.11.3 → flowerpower-0.11.5}/src/flowerpower/pipeline/manager.py

@@ -145,7 +145,11 @@ class PipelineManager:
                 cache_storage=cache_storage,
             )
         self._fs = fs
-        self._storage_options = storage_options or fs.storage_options
+        self._storage_options = (
+            storage_options or fs.storage_options
+            if fs.protocol != "dir"
+            else fs.fs.storage_options
+        )
 
         # Store overrides for ProjectConfig loading
         self._cfg_dir = cfg_dir

{flowerpower-0.11.3 → flowerpower-0.11.5}/src/flowerpower/plugins/io/base.py

@@ -1,4 +1,5 @@
 import importlib
+import os
 import posixpath
 from typing import Any, Generator
 
@@ -22,8 +23,9 @@ from sqlalchemy import create_engine, text
 from ...fs import get_filesystem
 from ...fs.ext import _dict_to_dataframe, path_to_glob
 from ...fs.storage_options import (AwsStorageOptions, AzureStorageOptions,
-                                   GcsStorageOptions, GitHubStorageOptions,
-                                   GitLabStorageOptions, StorageOptions)
+                                   BaseStorageOptions, GcsStorageOptions,
+                                   GitHubStorageOptions, GitLabStorageOptions,
+                                   StorageOptions)
 from ...utils.misc import convert_large_types_to_standard, to_pyarrow_table
 from .helpers.polars import pl
 from .helpers.sql import sql2polars_filter, sql2pyarrow_filter
@@ -75,67 +77,80 @@ class BaseFileIO(msgspec.Struct, gc=False):
     ) = field(default=None)
     fs: AbstractFileSystem | None = field(default=None)
     format: str | None = None
-    _raw_path: str | list[str] | None = field(default=None)
+    # _base_path: str | list[str] | None = field(default=None)
+    # _full_path: str | list[str] | None = field(default=None)
+    # _rel_path: str | list[str] | None = field(default=None)
+    # _glob_path
     _metadata: dict[str, Any] | None = field(default=None)
 
     def __post_init__(self):
-        self._raw_path = self.path
-        if isinstance(self.storage_options, dict):
-            if "protocol" not in self.storage_options:
-                self.storage_options["protocol"] = get_protocol(self.path)
-            self.storage_options = StorageOptions(
-                **self.storage_options
-            ).storage_options
-        if isinstance(self.storage_options, StorageOptions):
-            self.storage_options = self.storage_options.storage_options
+        # self._base_path = self.path if isinstance(self.path, str) else os.path.commonpath(self.path)
 
         if self.fs is None:
             self.fs = get_filesystem(
-                path=self.path if isinstance(self.path, str) else self.path[0],
+                path=self._base_path,
                 storage_options=self.storage_options,
                 fs=self.fs,
                 dirfs=True,
             )
+        self.storage_options = (
+            self.storage_options or self.fs.storage_options
+            if self.protocol != "dir"
+            else self.fs.fs.storage_options
+        )
 
-        if hasattr(self.storage_options, "protocol"):
-            protocol = self.storage_options.protocol
-        else:
-            protocol = self.fs.protocol
-            if protocol == "dir":
-                protocol = (
-                    self.fs.fs.protocol
-                    if isinstance(self.fs.fs.protocol, str)
-                    else self.fs.fs.protocol[0]
-                )
-            if isinstance(protocol, list | tuple):
-                protocol = protocol[0]
-
-        if isinstance(self.path, str):
-            self.path = (
-                self.path.replace(protocol + "://", "")
-                .replace(f"**/*.{self.format}", "")
-                .replace("**", "")
-                .replace("*", "")
-                .rstrip("/")
-            )
+        # self.path = (
+        #    self._raw_path.replace(protocol + "://", "")
+        #    .replace(f"**/*.{self.format}", "")
+        #    .replace("**", "")
+        #    .replace("*", "")
+        #    .rstrip("/")
+        # )
+
+    @property
+    def protocol(self):
+        """Get the protocol of the filesystem."""
+        protocol = (
+            self.fs.protocol if self.fs.protocol != "dir" else self.fs.fs.protocol
+        )
+        if isinstance(protocol, list | tuple):
+            protocol = protocol[0]
+        return protocol
+
+    @property
+    def _base_path(self) -> str:
+        """Get the base path for the filesystem."""
+
+        path = (
+            self.path if isinstance(self.path, str) else os.path.commonpath(self.path)
+        )
+        return path
 
     @property
-    def _path(self):
+    def _path(self) -> str | list[str]:
         if self.fs.protocol == "dir":
             if isinstance(self.path, list):
                 return [
-                    p.replace(self.fs.path.lstrip("/"), "").lstrip("/")
+                    p.replace(self._base_path.lstrip("/"), "").lstrip("/")
                     for p in self.path
                 ]
             else:
-                return self.path.replace(self.fs.path.lstrip("/"), "").lstrip("/")
+                return self.path.replace(self._base_path.lstrip("/"), "").lstrip("/")
         return self.path
 
     @property
-    def _glob_path(self):
+    def _glob_path(self) -> str | list[str]:
+        if isinstance(self._path, list):
+            return self._path
         return path_to_glob(self._path, self.format)
 
-    def list_files(self):
+    @property
+    def _root_path(self) -> str:
+        if self.fs.protocol == "dir":
+            return self._base_path.replace(self.fs.path, "")
+        return self._base_path
+
+    def list_files(self) -> list[str]:
         if isinstance(self._path, list):
             return self._path
 
@@ -276,7 +291,7 @@ class BaseFileReader(BaseFileIO, gc=False):
         return df
 
     def iter_pandas(
-        self, batch_size: int = 1, reload: bool = False, **kwargs
+        self, reload: bool = False, **kwargs
     ) -> Generator[pd.DataFrame, None, None]:
         """Iterate over Pandas DataFrames.
 
@@ -287,7 +302,10 @@ class BaseFileReader(BaseFileIO, gc=False):
         Returns:
             Generator[pd.DataFrame, None, None]: Generator of Pandas DataFrames.
         """
-        self._load(batch_size=batch_size, reload=reload, **kwargs)
+        if self.batch_size is None and "batch_size" not in kwargs:
+            self.batch_size = 1
+
+        self._load(reload=reload, **kwargs)
         if isinstance(self._data, list | Generator):
             for df in self._data:
                 yield df if isinstance(df, pd.DataFrame) else df.to_pandas()
@@ -324,14 +342,17 @@ class BaseFileReader(BaseFileIO, gc=False):
         return df
 
     def _iter_polars_dataframe(
-        self, batch_size: int = 1, reload: bool = False, **kwargs
+        self, reload: bool = False, **kwargs
     ) -> Generator[pl.DataFrame, None, None]:
         """Iterate over Polars DataFrames.
 
         Returns:
             Generator[pl.DataFrame, None, None]: Generator of Polars DataFrames.
         """
-        self._load(batch_size=batch_size, reload=reload, **kwargs)
+        if self.batch_size is None and "batch_size" not in kwargs:
+            self.batch_size = 1
+
+        self._load(reload=reload, **kwargs)
         if isinstance(self._data, list | Generator):
             for df in self._data:
                 yield df if isinstance(df, pl.DataFrame) else pl.from_arrow(df)
@@ -361,7 +382,7 @@ class BaseFileReader(BaseFileIO, gc=False):
         return df
 
     def _iter_polars_lazyframe(
-        self, batch_size: int = 1, reload: bool = False, **kwargs
+        self, reload: bool = False, **kwargs
     ) -> Generator[pl.LazyFrame, None, None]:
         """Iterate over Polars LazyFrames.
 
@@ -372,7 +393,9 @@ class BaseFileReader(BaseFileIO, gc=False):
         Returns:
             Generator[pl.LazyFrame, None, None]: Generator of Polars LazyFrames.
         """
-        self._load(batch_size=batch_size, reload=reload, **kwargs)
+        if self.batch_size is None and "batch_size" not in kwargs:
+            self.batch_size = 1
+        self._load(reload=reload, **kwargs)
         if isinstance(self._data, list | Generator):
             for df in self._data:
                 yield (
@@ -420,12 +443,11 @@ class BaseFileReader(BaseFileIO, gc=False):
     def iter_polars(
         self,
         lazy: bool = False,
-        batch_size: int = 1,
         **kwargs,
     ) -> Generator[pl.DataFrame | pl.LazyFrame, None, None]:
         if lazy:
-            yield from self._iter_polars_lazyframe(batch_size=batch_size, **kwargs)
-        yield from self._iter_polars_dataframe(batch_size=batch_size, **kwargs)
+            yield from self._iter_polars_lazyframe(**kwargs)
+        yield from self._iter_polars_dataframe(**kwargs)
 
     def to_pyarrow_table(
         self, metadata: bool = False, reload: bool = False, **kwargs
@@ -459,14 +481,17 @@ class BaseFileReader(BaseFileIO, gc=False):
         return df
 
     def iter_pyarrow_table(
-        self, batch_size: int = 1, reload: bool = False, **kwargs
+        self, reload: bool = False, **kwargs
     ) -> Generator[pa.Table, None, None]:
         """Iterate over PyArrow Tables.
 
         Returns:
             Generator[pa.Table, None, None]: Generator of PyArrow Tables.
         """
-        self._load(batch_size=batch_size, reload=reload, **kwargs)
+        if self.batch_size is None and "batch_size" not in kwargs:
+            self.batch_size = 1
+
+        self._load(reload=reload, **kwargs)
         if isinstance(self._data, list | Generator):
             for df in self._data:
                 yield df.to_arrow(**kwargs) if isinstance(df, pl.DataFrame) else df
@@ -752,7 +777,7 @@ class BaseDatasetReader(BaseFileReader, gc=False):
         Returns:
             pds.Dataset: PyArrow Dataset.
         """
-        if hasattr(self, "_dataset") and not reload:
+        if self._dataset is not None and not reload:
             if metadata:
                 return self._dataset, self._metadata
             return self._dataset
@@ -769,9 +794,9 @@ class BaseDatasetReader(BaseFileReader, gc=False):
                 self._dataset, path=self.path, format=self.format
             )
         elif self.format == "parquet":
-            if self.fs.exists(posixpath.join(self._path, "_metadata")):
+            if self.fs.exists(posixpath.join(self._root_path, "_metadata")):
                 self._dataset = self.fs.parquet_dataset(
-                    self._path,
+                    posixpath.join(self._root_path, "_metadata"),
                     schema=self.schema_,
                     partitioning=self.partitioning,
                     **kwargs,