FlowerPower 0.11.4__tar.gz → 0.11.5.1__tar.gz

{flowerpower-0.11.4/src/FlowerPower.egg-info → flowerpower-0.11.5.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: FlowerPower
-Version: 0.11.4
+Version: 0.11.5.1
 Summary: A simple workflow framework. Hamilton + APScheduler = FlowerPower
 Author-email: "Volker L." <ligno.blades@gmail.com>
 Project-URL: Homepage, https://github.com/legout/flowerpower

{flowerpower-0.11.4 → flowerpower-0.11.5.1}/pyproject.toml

@@ -4,7 +4,7 @@ description = "A simple workflow framework. Hamilton + APScheduler = FlowerPower
 authors = [{ name = "Volker L.", email = "ligno.blades@gmail.com" }]
 readme = "README.md"
 requires-python = ">= 3.11"
-version = "0.11.4"
+version = "0.11.5.1"
 keywords = [
   "hamilton",
   "workflow",

{flowerpower-0.11.4 → flowerpower-0.11.5.1/src/FlowerPower.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: FlowerPower
-Version: 0.11.4
+Version: 0.11.5.1
 Summary: A simple workflow framework. Hamilton + APScheduler = FlowerPower
 Author-email: "Volker L." <ligno.blades@gmail.com>
 Project-URL: Homepage, https://github.com/legout/flowerpower

{flowerpower-0.11.4 → flowerpower-0.11.5.1}/src/flowerpower/fs/base.py

@@ -597,18 +597,29 @@ def get_filesystem(
         ... )
     """
     if fs is not None:
+        if dirfs:
+            if fs.protocol == "dir":
+                base_path = path.split("://")[1]
+                if base_path != fs.path:
+                    fs = DirFileSystem(
+                        path=posixpath.join(fs.path, base_path.replace(fs.path, "")),
+                        fs=fs.fs,
+                    )
         if cached:
             if fs.is_cache_fs:
                 return fs
             return MonitoredSimpleCacheFileSystem(fs=fs, cache_storage=cache_storage)
 
-        if dirfs:
-            if fs.protocol == "dir":
-                return fs
-            return DirFileSystem(path=path, fs=fs)
-
     pp = infer_storage_options(str(path) if isinstance(path, Path) else path)
-    protocol = pp.get("protocol")
+    protocol = (
+        storage_options_kwargs.get("protocol", None)
+        or (
+            storage_options.get("protocol", None)
+            if isinstance(storage_options, dict)
+            else getattr(storage_options, "protocol", None)
+        )
+        or pp.get("protocol", "file")
+    )
 
     if protocol == "file" or protocol == "local":
         fs = filesystem(protocol)
@@ -622,6 +633,8 @@ def get_filesystem(
     path = pp.get("path", "").lstrip("/")
     if len(host) and host not in path:
         path = posixpath.join(host, path)
+    if "." in path:
+        path = posixpath.dirname(path)
 
     if isinstance(storage_options, dict):
         storage_options = storage_options_from_dict(protocol, storage_options)

{flowerpower-0.11.4 → flowerpower-0.11.5.1}/src/flowerpower/pipeline/manager.py

@@ -145,7 +145,11 @@ class PipelineManager:
                 cache_storage=cache_storage,
             )
         self._fs = fs
-        self._storage_options = storage_options or fs.storage_options
+        self._storage_options = (
+            storage_options or fs.storage_options
+            if fs.protocol != "dir"
+            else fs.fs.storage_options
+        )
 
         # Store overrides for ProjectConfig loading
         self._cfg_dir = cfg_dir

{flowerpower-0.11.4 → flowerpower-0.11.5.1}/src/flowerpower/plugins/io/base.py

@@ -1,4 +1,5 @@
 import importlib
+import os
 import posixpath
 from typing import Any, Generator
 
@@ -22,8 +23,9 @@ from sqlalchemy import create_engine, text
 from ...fs import get_filesystem
 from ...fs.ext import _dict_to_dataframe, path_to_glob
 from ...fs.storage_options import (AwsStorageOptions, AzureStorageOptions,
-                                   GcsStorageOptions, GitHubStorageOptions,
-                                   GitLabStorageOptions, StorageOptions)
+                                   BaseStorageOptions, GcsStorageOptions,
+                                   GitHubStorageOptions, GitLabStorageOptions,
+                                   StorageOptions)
 from ...utils.misc import convert_large_types_to_standard, to_pyarrow_table
 from .helpers.polars import pl
 from .helpers.sql import sql2polars_filter, sql2pyarrow_filter
@@ -75,67 +77,80 @@ class BaseFileIO(msgspec.Struct, gc=False):
     ) = field(default=None)
     fs: AbstractFileSystem | None = field(default=None)
     format: str | None = None
-    _raw_path: str | list[str] | None = field(default=None)
+    # _base_path: str | list[str] | None = field(default=None)
+    # _full_path: str | list[str] | None = field(default=None)
+    # _rel_path: str | list[str] | None = field(default=None)
+    # _glob_path
     _metadata: dict[str, Any] | None = field(default=None)
 
     def __post_init__(self):
-        self._raw_path = self.path
-        if isinstance(self.storage_options, dict):
-            if "protocol" not in self.storage_options:
-                self.storage_options["protocol"] = get_protocol(self.path)
-            self.storage_options = StorageOptions(
-                **self.storage_options
-            ).storage_options
-        if isinstance(self.storage_options, StorageOptions):
-            self.storage_options = self.storage_options.storage_options
+        # self._base_path = self.path if isinstance(self.path, str) else os.path.commonpath(self.path)
 
         if self.fs is None:
             self.fs = get_filesystem(
-                path=self.path if isinstance(self.path, str) else self.path[0],
+                path=self._base_path,
                 storage_options=self.storage_options,
                 fs=self.fs,
                 dirfs=True,
             )
+        self.storage_options = (
+            self.storage_options or self.fs.storage_options
+            if self.protocol != "dir"
+            else self.fs.fs.storage_options
+        )
 
-        if hasattr(self.storage_options, "protocol"):
-            protocol = self.storage_options.protocol
-        else:
-            protocol = self.fs.protocol
-            if protocol == "dir":
-                protocol = (
-                    self.fs.fs.protocol
-                    if isinstance(self.fs.fs.protocol, str)
-                    else self.fs.fs.protocol[0]
-                )
-            if isinstance(protocol, list | tuple):
-                protocol = protocol[0]
-
-        if isinstance(self.path, str):
-            self.path = (
-                self.path.replace(protocol + "://", "")
-                .replace(f"**/*.{self.format}", "")
-                .replace("**", "")
-                .replace("*", "")
-                .rstrip("/")
-            )
+        # self.path = (
+        #    self._raw_path.replace(protocol + "://", "")
+        #    .replace(f"**/*.{self.format}", "")
+        #    .replace("**", "")
+        #    .replace("*", "")
+        #    .rstrip("/")
+        # )
 
     @property
-    def _path(self):
+    def protocol(self):
+        """Get the protocol of the filesystem."""
+        protocol = (
+            self.fs.protocol if self.fs.protocol != "dir" else self.fs.fs.protocol
+        )
+        if isinstance(protocol, list | tuple):
+            protocol = protocol[0]
+        return protocol
+
+    @property
+    def _base_path(self) -> str:
+        """Get the base path for the filesystem."""
+
+        path = (
+            self.path if isinstance(self.path, str) else os.path.commonpath(self.path)
+        )
+        return path
+
+    @property
+    def _path(self) -> str | list[str]:
         if self.fs.protocol == "dir":
             if isinstance(self.path, list):
                 return [
-                    p.replace(self.fs.path.lstrip("/"), "").lstrip("/")
+                    p.replace(self._base_path.lstrip("/"), "").lstrip("/")
                     for p in self.path
                 ]
             else:
-                return self.path.replace(self.fs.path.lstrip("/"), "").lstrip("/")
+                return self.path.replace(self._base_path.lstrip("/"), "").lstrip("/")
         return self.path
 
     @property
-    def _glob_path(self):
+    def _glob_path(self) -> str | list[str]:
+        if isinstance(self._path, list):
+            return self._path
         return path_to_glob(self._path, self.format)
 
-    def list_files(self):
+    @property
+    def _root_path(self) -> str:
+        if self.fs.protocol == "dir":
+            return self._base_path.replace(self.fs.path, "")
+        return self._base_path
+
+    def list_files(self) -> list[str]:
         if isinstance(self._path, list):
             return self._path
 
@@ -762,7 +777,7 @@ class BaseDatasetReader(BaseFileReader, gc=False):
         Returns:
             pds.Dataset: PyArrow Dataset.
         """
-        if hasattr(self, "_dataset") and not reload:
+        if self._dataset is not None and not reload:
             if metadata:
                 return self._dataset, self._metadata
             return self._dataset
@@ -779,9 +794,9 @@ class BaseDatasetReader(BaseFileReader, gc=False):
                 self._dataset, path=self.path, format=self.format
             )
         elif self.format == "parquet":
-            if self.fs.exists(posixpath.join(self._path, "_metadata")):
+            if self.fs.exists(posixpath.join(self._root_path, "_metadata")):
                 self._dataset = self.fs.parquet_dataset(
-                    self._path,
+                    posixpath.join(self._root_path, "_metadata"),
                     schema=self.schema_,
                     partitioning=self.partitioning,
                     **kwargs,