FlowerPower 0.11.6.20__py3-none-any.whl → 0.21.0__py3-none-any.whl

flowerpower/pipeline/base.py

@@ -3,11 +3,12 @@ import posixpath
 import sys
 from types import TracebackType
 
+from fsspec_utils import AbstractFileSystem, BaseStorageOptions, filesystem
 from loguru import logger
 from munch import Munch
 
+from ..settings import CONFIG_DIR, PIPELINES_DIR
 from ..cfg import PipelineConfig, ProjectConfig
-from ..fs import AbstractFileSystem, BaseStorageOptions, get_filesystem
 from ..utils.logging import setup_logging
 
 setup_logging()
@@ -40,26 +41,35 @@ class BasePipeline:
         base_dir: str | None = None,
         storage_options: dict | Munch | BaseStorageOptions = {},
         fs: AbstractFileSystem | None = None,
-        cfg_dir: str = "conf",
-        pipelines_dir: str = "pipelines",
-        job_queue_type: str | None = None,  # New parameter for worker backend
+        
     ):
         self._base_dir = base_dir
         self._storage_options = storage_options
         if fs is None:
-            fs = get_filesystem(self._base_dir, **self._storage_options)
+            fs = filesystem(self._base_dir, **self._storage_options)
         self._fs = fs
-        self._cfg_dir = cfg_dir
-        self._pipelines_dir = pipelines_dir
-        self._job_queue_type = job_queue_type
+        
 
+        self._setup_paths()
+        self._setup_directories()
+        self._add_modules_path()
+
+    def _setup_paths(self) -> None:
+        """Set up configuration and pipeline directory paths."""
+        self._cfg_dir = CONFIG_DIR
+        self._pipelines_dir = PIPELINES_DIR
+
+    def _setup_directories(self) -> None:
+        """Set up required directories with proper error handling."""
         try:
             self._fs.makedirs(f"{self._cfg_dir}/pipelines", exist_ok=True)
             self._fs.makedirs(self._pipelines_dir, exist_ok=True)
-        except Exception as e:
+        except (OSError, PermissionError) as e:
             logger.error(f"Error creating directories: {e}")
-
-        self._add_modules_path()
+            raise
+        except Exception as e:
+            logger.error(f"Unexpected error creating directories: {e}")
+            raise
 
     def __enter__(self) -> "BasePipeline":
         return self
@@ -98,7 +108,6 @@ class BasePipeline:
         """
         return ProjectConfig.load(
             base_dir=self._base_dir,
-            job_queue_type=self._job_queue_type,
             fs=self._fs,
             storage_options=self._storage_options,
         )

flowerpower/pipeline/io.py

@@ -7,16 +7,18 @@ Manages the import and export of pipelines.
 
 import posixpath
 
+from fsspec_utils import (AbstractFileSystem, BaseStorageOptions,
+                          DirFileSystem, filesystem)
 from loguru import logger
 from rich.console import Console
 
-# Import necessary config types and utility functions
-from ..fs.base import (AbstractFileSystem, BaseStorageOptions, DirFileSystem,
-                       get_filesystem)
 from ..settings import LOG_LEVEL
 from ..utils.logging import setup_logging
 from .registry import PipelineRegistry
 
+# Import necessary config types and utility functions
+
+
 console = Console()
 
 setup_logging(level=LOG_LEVEL)
@@ -41,6 +43,41 @@ class PipelineIOManager:
         self._cfg_dir = registry._cfg_dir
         self._pipelines_dir = registry._pipelines_dir
 
+    def _get_pipeline_files(self, name: str) -> list[str]:
+        """Get the list of files for a single pipeline."""
+        return [
+            "conf/project.yml",
+            f"conf/pipelines/{name}.yml",
+            f"pipelines/{name}.py",
+        ]
+
+    def _get_many_pipeline_files(self, names: list[str]) -> list[str]:
+        """Get the list of files for multiple pipelines."""
+        files = ["conf/project.yml"]
+        for name in names:
+            files.extend([
+                f"conf/pipelines/{name}.yml",
+                f"pipelines/{name}.py",
+            ])
+        return files
+
+    def _get_all_pipeline_files(self) -> list[str] | None:
+        """Get all pipeline files (returns None to let _sync_filesystem auto-discover)."""
+        return None
+
+    def _print_import_success(self, names: list[str], src_base_dir: str) -> None:
+        """Print success message for import operations."""
+        console.print(
+            f"✅ Imported pipelines [bold blue]{', '.join(names)}[/bold blue] from [green]{src_base_dir}[/green] to [bold blue]{self.project_cfg.name}[/bold blue]"
+        )
+
+    def _print_export_success(self, names: list[str] | None, dest_base_dir: str) -> None:
+        """Print success message for export operations."""
+        if names:
+            self._print_export_success(names, dest_base_dir)
+        else:
+            self._print_export_success(None, dest_base_dir)
+
     def _sync_filesystem(
         self,
         src_base_dir: str,
@@ -69,7 +106,7 @@ class PipelineIOManager:
 
         def _get_filesystem(base_dir, fs, storage_options):
             if fs is None:
-                fs = get_filesystem(base_dir, storage_options=storage_options)
+                fs = filesystem(base_dir, storage_options=storage_options)
             else:
                 if not isinstance(fs, AbstractFileSystem):
                     raise ValueError(
@@ -154,12 +191,7 @@ class PipelineIOManager:
             pm.import_pipeline("my_pipeline", "/path/to/pipeline")
             ```
         """
-        files = [
-            "conf/project.yml",
-            f"conf/pipelines/{name}.yml",
-            f"pipelines/{name}.py",
-        ]
-
+        files = self._get_pipeline_files(name)
         self._sync_filesystem(
             src_base_dir=src_base_dir,
             src_fs=src_fs,
@@ -171,10 +203,7 @@ class PipelineIOManager:
             overwrite=overwrite,
         )
 
-        # Use project_cfg.name directly
-        console.print(
-            f"✅ Imported pipeline [bold blue]{name}[/bold blue] from [green]{src_base_dir}[/green] to [bold blue]{self.project_cfg.name}[/bold blue]"
-        )
+        self._print_import_success([name], src_base_dir)
 
     def import_many(
         self,
@@ -208,17 +237,8 @@ class PipelineIOManager:
             ```
         """
 
-        files = ["conf/project.yml"]
+        files = self._get_many_pipeline_files(names)
 
-        for name in names:
-            files.extend(
-                [
-                    f"conf/pipelines/{name}.yml",
-                    f"pipelines/{name}.py",
-                ]
-            )
-
-        # Sync the filesystem
         self._sync_filesystem(
             src_base_dir=src_base_dir,
             src_fs=src_fs,
@@ -229,9 +249,7 @@ class PipelineIOManager:
             files=files,
             overwrite=overwrite,
         )
-        console.print(
-            f"✅ Imported pipelines [bold blue]{', '.join(names)}[/bold blue] from [green]{src_base_dir}[/green] to [bold blue]{self.project_cfg.name}[/bold blue]"
-        )
+        self._print_import_success(names, src_base_dir)
 
     def import_all(
         self,
@@ -260,6 +278,8 @@ class PipelineIOManager:
             # pm.import_all("s3://my-bucket/pipelines_backup", storage_options={"key": "...", "secret": "..."}, overwrite=False)
             ```
         """
+        files = self._get_all_pipeline_files()
+
         self._sync_filesystem(
             src_base_dir=src_base_dir,
             src_fs=src_fs,
@@ -267,12 +287,10 @@ class PipelineIOManager:
             dest_base_dir=".",
             dest_fs=self._fs,
             dest_storage_options=None,
-            files=None,
+            files=files,
             overwrite=overwrite,
         )
-        console.print(
-            f"✅ Imported all pipelines from [green]{src_base_dir}[/green] to [bold blue]{self.project_cfg.name}[/bold blue]"
-        )
+        self._print_import_success([], src_base_dir)
 
     def export_pipeline(
         self,
@@ -306,13 +324,8 @@ class PipelineIOManager:
             # pm.export("my_pipeline", "s3://my-bucket/exports", storage_options={"key": "...", "secret": "..."})
             ```
         """
-        files = [
-            "conf/project.yml",
-            f"conf/pipelines/{name}.yml",
-            f"pipelines/{name}.py",
-        ]
+        files = self._get_pipeline_files(name)
 
-        # Sync the filesystem
         self._sync_filesystem(
             src_base_dir=".",
             src_fs=self._fs,
@@ -324,9 +337,7 @@ class PipelineIOManager:
             overwrite=overwrite,
         )
 
-        console.print(
-            f"✅ Exported pipeline [bold blue]{self.project_cfg.name}.{name}[/bold blue] to [green]{dest_base_dir}[/green]"
-        )
+        self._print_export_success([name], dest_base_dir)
 
     def export_many(
         self,
@@ -356,23 +367,15 @@ class PipelineIOManager:
             pm.export_many(pipelines_to_export, "/path/to/export_dir", overwrite=True)
             ```
         """
-        files = [
-            "conf/project.yml",
-        ]
+        # Check if pipelines exist in the registry
         for name in names:
-            # Check if the pipeline exists in the registry
             if not self.registry.has_pipeline(name):
                 raise ValueError(
                     f"Pipeline {name} does not exist in the registry. Please check the name."
                 )
-            # Add pipeline files to the list
-            files.extend(
-                [
-                    f"conf/pipelines/{name}.yml",
-                    f"pipelines/{name}.py",
-                ]
-            )
-        # Sync the filesystem
+
+        files = self._get_many_pipeline_files(names)
+
         self._sync_filesystem(
             src_base_dir=".",
             src_fs=self._fs,
@@ -414,7 +417,8 @@ class PipelineIOManager:
             # pm.export_all("s3://my-bucket/pipelines_backup", storage_options={"key": "...", "secret": "..."}, overwrite=False)
             ```
         """
-        # sync the filesystem
+        files = self._get_all_pipeline_files()
+
         self._sync_filesystem(
             src_base_dir=".",
             src_fs=self._fs,
@@ -422,7 +426,7 @@ class PipelineIOManager:
             dest_base_dir=dest_base_dir,
             dest_fs=dest_fs,
             dest_storage_options=dest_storage_options,
-            files=None,
+            files=files,
             overwrite=overwrite,
         )
         console.print(