FlowerPower 0.11.3__py3-none-any.whl → 0.11.5__py3-none-any.whl

flowerpower/fs/base.py

@@ -608,7 +608,15 @@ def get_filesystem(
             return DirFileSystem(path=path, fs=fs)
 
     pp = infer_storage_options(str(path) if isinstance(path, Path) else path)
-    protocol = pp.get("protocol")
+    protocol = (
+        storage_options_kwargs.get("protocol", None)
+        or (
+            storage_options.get("protocol", None)
+            if isinstance(storage_options, dict)
+            else getattr(storage_options, "protocol", None)
+        )
+        or pp.get("protocol", "file")
+    )
 
     if protocol == "file" or protocol == "local":
         fs = filesystem(protocol)

flowerpower/pipeline/manager.py

@@ -145,7 +145,11 @@ class PipelineManager:
                 cache_storage=cache_storage,
             )
         self._fs = fs
-        self._storage_options = storage_options or fs.storage_options
+        self._storage_options = (
+            storage_options or fs.storage_options
+            if fs.protocol != "dir"
+            else fs.fs.storage_options
+        )
 
         # Store overrides for ProjectConfig loading
         self._cfg_dir = cfg_dir

flowerpower/plugins/io/base.py

@@ -1,4 +1,5 @@
 import importlib
+import os
 import posixpath
 from typing import Any, Generator
 
@@ -22,8 +23,9 @@ from sqlalchemy import create_engine, text
 from ...fs import get_filesystem
 from ...fs.ext import _dict_to_dataframe, path_to_glob
 from ...fs.storage_options import (AwsStorageOptions, AzureStorageOptions,
-                                   GcsStorageOptions, GitHubStorageOptions,
-                                   GitLabStorageOptions, StorageOptions)
+                                   BaseStorageOptions, GcsStorageOptions,
+                                   GitHubStorageOptions, GitLabStorageOptions,
+                                   StorageOptions)
 from ...utils.misc import convert_large_types_to_standard, to_pyarrow_table
 from .helpers.polars import pl
 from .helpers.sql import sql2polars_filter, sql2pyarrow_filter
@@ -75,67 +77,80 @@ class BaseFileIO(msgspec.Struct, gc=False):
     ) = field(default=None)
     fs: AbstractFileSystem | None = field(default=None)
     format: str | None = None
-    _raw_path: str | list[str] | None = field(default=None)
+    # _base_path: str | list[str] | None = field(default=None)
+    # _full_path: str | list[str] | None = field(default=None)
+    # _rel_path: str | list[str] | None = field(default=None)
+    # _glob_path
     _metadata: dict[str, Any] | None = field(default=None)
 
     def __post_init__(self):
-        self._raw_path = self.path
-        if isinstance(self.storage_options, dict):
-            if "protocol" not in self.storage_options:
-                self.storage_options["protocol"] = get_protocol(self.path)
-            self.storage_options = StorageOptions(
-                **self.storage_options
-            ).storage_options
-        if isinstance(self.storage_options, StorageOptions):
-            self.storage_options = self.storage_options.storage_options
+        # self._base_path = self.path if isinstance(self.path, str) else os.path.commonpath(self.path)
 
         if self.fs is None:
             self.fs = get_filesystem(
-                path=self.path if isinstance(self.path, str) else self.path[0],
+                path=self._base_path,
                 storage_options=self.storage_options,
                 fs=self.fs,
                 dirfs=True,
             )
+        self.storage_options = (
+            self.storage_options or self.fs.storage_options
+            if self.protocol != "dir"
+            else self.fs.fs.storage_options
+        )
 
-        if hasattr(self.storage_options, "protocol"):
-            protocol = self.storage_options.protocol
-        else:
-            protocol = self.fs.protocol
-            if protocol == "dir":
-                protocol = (
-                    self.fs.fs.protocol
-                    if isinstance(self.fs.fs.protocol, str)
-                    else self.fs.fs.protocol[0]
-                )
-            if isinstance(protocol, list | tuple):
-                protocol = protocol[0]
-
-        if isinstance(self.path, str):
-            self.path = (
-                self.path.replace(protocol + "://", "")
-                .replace(f"**/*.{self.format}", "")
-                .replace("**", "")
-                .replace("*", "")
-                .rstrip("/")
-            )
+        # self.path = (
+        #    self._raw_path.replace(protocol + "://", "")
+        #    .replace(f"**/*.{self.format}", "")
+        #    .replace("**", "")
+        #    .replace("*", "")
+        #    .rstrip("/")
+        # )
+
+    @property
+    def protocol(self):
+        """Get the protocol of the filesystem."""
+        protocol = (
+            self.fs.protocol if self.fs.protocol != "dir" else self.fs.fs.protocol
+        )
+        if isinstance(protocol, list | tuple):
+            protocol = protocol[0]
+        return protocol
+
+    @property
+    def _base_path(self) -> str:
+        """Get the base path for the filesystem."""
+
+        path = (
+            self.path if isinstance(self.path, str) else os.path.commonpath(self.path)
+        )
+        return path
 
     @property
-    def _path(self):
+    def _path(self) -> str | list[str]:
         if self.fs.protocol == "dir":
             if isinstance(self.path, list):
                 return [
-                    p.replace(self.fs.path.lstrip("/"), "").lstrip("/")
+                    p.replace(self._base_path.lstrip("/"), "").lstrip("/")
                     for p in self.path
                 ]
             else:
-                return self.path.replace(self.fs.path.lstrip("/"), "").lstrip("/")
+                return self.path.replace(self._base_path.lstrip("/"), "").lstrip("/")
         return self.path
 
     @property
-    def _glob_path(self):
+    def _glob_path(self) -> str | list[str]:
+        if isinstance(self._path, list):
+            return self._path
         return path_to_glob(self._path, self.format)
 
-    def list_files(self):
+    @property
+    def _root_path(self) -> str:
+        if self.fs.protocol == "dir":
+            return self._base_path.replace(self.fs.path, "")
+        return self._base_path
+
+    def list_files(self) -> list[str]:
         if isinstance(self._path, list):
             return self._path
 
@@ -276,7 +291,7 @@ class BaseFileReader(BaseFileIO, gc=False):
         return df
 
     def iter_pandas(
-        self, batch_size: int = 1, reload: bool = False, **kwargs
+        self, reload: bool = False, **kwargs
     ) -> Generator[pd.DataFrame, None, None]:
         """Iterate over Pandas DataFrames.
 
@@ -287,7 +302,10 @@ class BaseFileReader(BaseFileIO, gc=False):
         Returns:
             Generator[pd.DataFrame, None, None]: Generator of Pandas DataFrames.
         """
-        self._load(batch_size=batch_size, reload=reload, **kwargs)
+        if self.batch_size is None and "batch_size" not in kwargs:
+            self.batch_size = 1
+
+        self._load(reload=reload, **kwargs)
         if isinstance(self._data, list | Generator):
             for df in self._data:
                 yield df if isinstance(df, pd.DataFrame) else df.to_pandas()
@@ -324,14 +342,17 @@ class BaseFileReader(BaseFileIO, gc=False):
         return df
 
     def _iter_polars_dataframe(
-        self, batch_size: int = 1, reload: bool = False, **kwargs
+        self, reload: bool = False, **kwargs
     ) -> Generator[pl.DataFrame, None, None]:
         """Iterate over Polars DataFrames.
 
         Returns:
             Generator[pl.DataFrame, None, None]: Generator of Polars DataFrames.
         """
-        self._load(batch_size=batch_size, reload=reload, **kwargs)
+        if self.batch_size is None and "batch_size" not in kwargs:
+            self.batch_size = 1
+
+        self._load(reload=reload, **kwargs)
         if isinstance(self._data, list | Generator):
             for df in self._data:
                 yield df if isinstance(df, pl.DataFrame) else pl.from_arrow(df)
@@ -361,7 +382,7 @@ class BaseFileReader(BaseFileIO, gc=False):
         return df
 
     def _iter_polars_lazyframe(
-        self, batch_size: int = 1, reload: bool = False, **kwargs
+        self, reload: bool = False, **kwargs
     ) -> Generator[pl.LazyFrame, None, None]:
         """Iterate over Polars LazyFrames.
 
@@ -372,7 +393,9 @@ class BaseFileReader(BaseFileIO, gc=False):
         Returns:
             Generator[pl.LazyFrame, None, None]: Generator of Polars LazyFrames.
         """
-        self._load(batch_size=batch_size, reload=reload, **kwargs)
+        if self.batch_size is None and "batch_size" not in kwargs:
+            self.batch_size = 1
+        self._load(reload=reload, **kwargs)
         if isinstance(self._data, list | Generator):
             for df in self._data:
                 yield (
@@ -420,12 +443,11 @@ class BaseFileReader(BaseFileIO, gc=False):
     def iter_polars(
         self,
         lazy: bool = False,
-        batch_size: int = 1,
         **kwargs,
     ) -> Generator[pl.DataFrame | pl.LazyFrame, None, None]:
         if lazy:
-            yield from self._iter_polars_lazyframe(batch_size=batch_size, **kwargs)
-        yield from self._iter_polars_dataframe(batch_size=batch_size, **kwargs)
+            yield from self._iter_polars_lazyframe(**kwargs)
+        yield from self._iter_polars_dataframe(**kwargs)
 
     def to_pyarrow_table(
         self, metadata: bool = False, reload: bool = False, **kwargs
@@ -459,14 +481,17 @@ class BaseFileReader(BaseFileIO, gc=False):
         return df
 
     def iter_pyarrow_table(
-        self, batch_size: int = 1, reload: bool = False, **kwargs
+        self, reload: bool = False, **kwargs
     ) -> Generator[pa.Table, None, None]:
         """Iterate over PyArrow Tables.
 
         Returns:
             Generator[pa.Table, None, None]: Generator of PyArrow Tables.
         """
-        self._load(batch_size=batch_size, reload=reload, **kwargs)
+        if self.batch_size is None and "batch_size" not in kwargs:
+            self.batch_size = 1
+
+        self._load(reload=reload, **kwargs)
         if isinstance(self._data, list | Generator):
             for df in self._data:
                 yield df.to_arrow(**kwargs) if isinstance(df, pl.DataFrame) else df
@@ -752,7 +777,7 @@ class BaseDatasetReader(BaseFileReader, gc=False):
         Returns:
             pds.Dataset: PyArrow Dataset.
         """
-        if hasattr(self, "_dataset") and not reload:
+        if self._dataset is not None and not reload:
             if metadata:
                 return self._dataset, self._metadata
             return self._dataset
@@ -769,9 +794,9 @@ class BaseDatasetReader(BaseFileReader, gc=False):
                 self._dataset, path=self.path, format=self.format
             )
         elif self.format == "parquet":
-            if self.fs.exists(posixpath.join(self._path, "_metadata")):
+            if self.fs.exists(posixpath.join(self._root_path, "_metadata")):
                 self._dataset = self.fs.parquet_dataset(
-                    self._path,
+                    posixpath.join(self._root_path, "_metadata"),
                     schema=self.schema_,
                     partitioning=self.partitioning,
                     **kwargs,

{flowerpower-0.11.3.dist-info → flowerpower-0.11.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: FlowerPower
-Version: 0.11.3
+Version: 0.11.5
 Summary: A simple workflow framework. Hamilton + APScheduler = FlowerPower
 Author-email: "Volker L." <ligno.blades@gmail.com>
 Project-URL: Homepage, https://github.com/legout/flowerpower

{flowerpower-0.11.3.dist-info → flowerpower-0.11.5.dist-info}/RECORD

@@ -17,7 +17,7 @@ flowerpower/cli/mqtt.py,sha256=GM5d6bzG01THZd7SwXFAC3j0cait642eXT50P3R22vk,6281
 flowerpower/cli/pipeline.py,sha256=60P6u_QOSgp0jJXEMxazEEo5Sh7-SWFo-Kkuaz21YuI,37845
 flowerpower/cli/utils.py,sha256=nDSSj_1nlYlMmj252kRZeohhFqHv9yvdgDEduQCyWOc,5152
 flowerpower/fs/__init__.py,sha256=uZaPXErEfQqQRbKRIjkB9yiygd45X5_psYn9-VVrBTQ,910
-flowerpower/fs/base.py,sha256=LPuIYVHSBvjv0ml2R9NiMlM0wLhlCDUAG8XAcvhUDX4,22090
+flowerpower/fs/base.py,sha256=bWfY_xPPZ9DhSpSGMC2SV6OdbYp5XyDrjxLYhTjJlqo,22352
 flowerpower/fs/ext.py,sha256=gsCJ87VxVdy22oVtNRaN4M-SLO8WORVf5JRyDeQjjEs,63834
 flowerpower/fs/storage_options.py,sha256=msq5TpxAU8tcE_Bxjw6SyxaFa75UjdYnR4-O9U2wmbk,48034
 flowerpower/job_queue/__init__.py,sha256=a25hIqv2xoFKb4JZlyUukS0ppZ9-2sJKH3XAvbk3rlk,10788
@@ -40,11 +40,11 @@ flowerpower/pipeline/__init__.py,sha256=xbEn_RN0vVNqLZMSFOCdV41ggUkYrghFVJYd_EC0
 flowerpower/pipeline/base.py,sha256=N3N0iqiVo2vUVli_WSADAQMq283mG9OdFql58LXeF2Q,3275
 flowerpower/pipeline/io.py,sha256=8Mlw4G7ehHHZEk4Qui-HcKBM3tBF4FuqUbjfNxK09iU,15963
 flowerpower/pipeline/job_queue.py,sha256=hl38-0QZCH5wujUf0qIqYznIPDLsJAoNDcOD7YGVQ6s,26114
-flowerpower/pipeline/manager.py,sha256=OMVznuQ0fSI4Jnw7fw0XTYdqYRqioJRC2B2kTxbmlN0,74361
+flowerpower/pipeline/manager.py,sha256=KVpOclUEUAETUNJamJJGuKt3oxCaLitQgxWxkE1q028,74460
 flowerpower/pipeline/registry.py,sha256=WWQoaxtgnlntFEIPQzYM1gk0zUXwrH2PmDLGbTzhrZs,18991
 flowerpower/pipeline/runner.py,sha256=dsSVYixFXqlxFk8EJfT4wV_7IwgkXq0ErwH_yf_NGS8,25654
 flowerpower/pipeline/visualizer.py,sha256=amjMrl5NetErE198HzZBPWVZBi_t5jj9ydxWpuNLoTI,5013
-flowerpower/plugins/io/base.py,sha256=l9bYYtXBYlD1e-eUI11EtoPlfNN-mdHLx4yf2wvgi4Y,78974
+flowerpower/plugins/io/base.py,sha256=Plr0v2y0MMf3azxqz7rIc2wk6CXFzpPczW6p5Qb9jiI,79464
 flowerpower/plugins/io/metadata.py,sha256=31FoVyRz6zdWMWda0f1GHq0xMokVB3SVYrLT2TejOH8,7271
 flowerpower/plugins/io/helpers/datetime.py,sha256=1WBUg2ywcsodJQwoF6JiIGc9yhVobvE2IErWp4i95m4,10649
 flowerpower/plugins/io/helpers/polars.py,sha256=VuksokWrsKk57-s5JkpsmzWCkaOIEiI4ONeAIO9LAdw,18071
@@ -93,9 +93,9 @@ flowerpower/utils/monkey.py,sha256=VPl3yimoWhwD9kI05BFsjNvtyQiDyLfY4Q85Bb6Ma0w,2
 flowerpower/utils/open_telemetry.py,sha256=fQWJWbIQFtKIxMBjAWeF12NGnqT0isO3A3j-DSOv_vE,949
 flowerpower/utils/scheduler.py,sha256=2zJ_xmLXpvXUQNF1XS2Gqm3Ogo907ctZ50GtvQB_rhE,9354
 flowerpower/utils/templates.py,sha256=ouyEeSDqa9PjW8c32fGpcINlpC0WToawRFZkMPtwsLE,1591
-flowerpower-0.11.3.dist-info/licenses/LICENSE,sha256=9AkLexxrmr0aBgSHiqxpJk9wgazpP1CTJyiDyr56J9k,1063
-flowerpower-0.11.3.dist-info/METADATA,sha256=JadCL-DzJIPNeZW39RYIuAZs8Ujm7AEdtEOarv-kF34,21610
-flowerpower-0.11.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-flowerpower-0.11.3.dist-info/entry_points.txt,sha256=61X11i5a2IwC9LBiP20XCDl5zMOigGCjMCx17B7bDbQ,52
-flowerpower-0.11.3.dist-info/top_level.txt,sha256=VraH4WtEUfSxs5L-rXwDQhzQb9eLHTUtgvmFZ2dAYnA,12
-flowerpower-0.11.3.dist-info/RECORD,,
+flowerpower-0.11.5.dist-info/licenses/LICENSE,sha256=9AkLexxrmr0aBgSHiqxpJk9wgazpP1CTJyiDyr56J9k,1063
+flowerpower-0.11.5.dist-info/METADATA,sha256=Cmyl1IhcjYUTdIrKw_VSGtjclhFUHzIcWlM7yknDL3I,21610
+flowerpower-0.11.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+flowerpower-0.11.5.dist-info/entry_points.txt,sha256=61X11i5a2IwC9LBiP20XCDl5zMOigGCjMCx17B7bDbQ,52
+flowerpower-0.11.5.dist-info/top_level.txt,sha256=VraH4WtEUfSxs5L-rXwDQhzQb9eLHTUtgvmFZ2dAYnA,12
+flowerpower-0.11.5.dist-info/RECORD,,