FlowerPower 0.30.0__py3-none-any.whl → 0.31.1__py3-none-any.whl

flowerpower/flowerpower.py

@@ -17,6 +17,9 @@ from .cfg.pipeline.adapter import AdapterConfig as PipelineAdapterConfig
 from .cfg.project.adapter import AdapterConfig as ProjectAdapterConfig
 from .pipeline import PipelineManager
 from .utils.logging import setup_logging
+from .utils.security import validate_pipeline_name
+from .utils.config import merge_run_config_with_kwargs
+from .utils.filesystem import FilesystemHelper
 
 setup_logging()
 
@@ -53,13 +56,8 @@ class FlowerPowerProject:
         self.name = self.pipeline_manager.project_cfg.name
 
     def _validate_pipeline_name(self, name: str) -> None:
-        """Validate the pipeline name argument."""
-        if not name or not isinstance(name, str):
-            raise ValueError("Pipeline 'name' must be a non-empty string")
-        if name.strip() != name:
-            raise ValueError(
-                "Pipeline 'name' cannot have leading or trailing whitespace"
-            )
+        """Validate the pipeline name argument using security utilities."""
+        validate_pipeline_name(name)  # Use secure validation function
 
     def _inject_dependencies(self):
         """Inject dependencies between managers for proper architecture.
@@ -71,76 +69,6 @@ class FlowerPowerProject:
         # This will be used when creating Pipeline instances
         self.pipeline_manager._project_context = self
 
-    def _merge_run_config_with_kwargs(self, run_config: RunConfig, kwargs: dict) -> RunConfig:
-        """Merge kwargs into a RunConfig object.
-        
-        This helper method updates the RunConfig object with values from kwargs,
-        handling different types of attributes appropriately.
-        
-        Args:
-            run_config: The RunConfig object to update
-            kwargs: Dictionary of additional parameters to merge
-            
-        Returns:
-            RunConfig: Updated RunConfig object
-        """
-        # Handle dictionary-like attributes with update or deep merge
-        if 'inputs' in kwargs and kwargs['inputs'] is not None:
-            if run_config.inputs is None:
-                run_config.inputs = kwargs['inputs']
-            else:
-                run_config.inputs.update(kwargs['inputs'])
-                
-        if 'config' in kwargs and kwargs['config'] is not None:
-            if run_config.config is None:
-                run_config.config = kwargs['config']
-            else:
-                run_config.config.update(kwargs['config'])
-                
-        if 'cache' in kwargs and kwargs['cache'] is not None:
-            run_config.cache = kwargs['cache']
-            
-        if 'adapter' in kwargs and kwargs['adapter'] is not None:
-            if run_config.adapter is None:
-                run_config.adapter = kwargs['adapter']
-            else:
-                run_config.adapter.update(kwargs['adapter'])
-        
-        # Handle executor_cfg - convert string/dict to ExecutorConfig if needed
-        if 'executor_cfg' in kwargs and kwargs['executor_cfg'] is not None:
-            executor_cfg = kwargs['executor_cfg']
-            if isinstance(executor_cfg, str):
-                run_config.executor = ExecutorConfig(type=executor_cfg)
-            elif isinstance(executor_cfg, dict):
-                run_config.executor = ExecutorConfig.from_dict(executor_cfg)
-            elif isinstance(executor_cfg, ExecutorConfig):
-                run_config.executor = executor_cfg
-        
-        # Handle adapter configurations
-        if 'with_adapter_cfg' in kwargs and kwargs['with_adapter_cfg'] is not None:
-            with_adapter_cfg = kwargs['with_adapter_cfg']
-            if isinstance(with_adapter_cfg, dict):
-                run_config.with_adapter = WithAdapterConfig.from_dict(with_adapter_cfg)
-            elif isinstance(with_adapter_cfg, WithAdapterConfig):
-                run_config.with_adapter = with_adapter_cfg
-                
-        if 'pipeline_adapter_cfg' in kwargs and kwargs['pipeline_adapter_cfg'] is not None:
-            run_config.pipeline_adapter_cfg = kwargs['pipeline_adapter_cfg']
-            
-        if 'project_adapter_cfg' in kwargs and kwargs['project_adapter_cfg'] is not None:
-            run_config.project_adapter_cfg = kwargs['project_adapter_cfg']
-        
-        # Handle simple attributes
-        simple_attrs = [
-            'final_vars', 'reload', 'log_level', 'max_retries', 'retry_delay',
-            'jitter_factor', 'retry_exceptions', 'on_success', 'on_failure'
-        ]
-        
-        for attr in simple_attrs:
-            if attr in kwargs and kwargs[attr] is not None:
-                setattr(run_config, attr, kwargs[attr])
-        
-        return run_config
 
     # --- Convenience Methods for Pipeline Operations ---
 
@@ -235,7 +163,7 @@ class FlowerPowerProject:
         
         # Merge kwargs into run_config
         if kwargs:
-            run_config = self._merge_run_config_with_kwargs(run_config, kwargs)
+            run_config = merge_run_config_with_kwargs(run_config, kwargs)
 
         return self.pipeline_manager.run(
             name=name,
@@ -363,6 +291,38 @@ class FlowerPowerProject:
         if log_level:
             setup_logging(level=log_level)
 
+        # Initialize project parameters
+        name, base_dir = cls._resolve_project_params(name, base_dir)
+
+        # Setup filesystem
+        fs = cls._setup_filesystem(base_dir, storage_options, fs)
+
+        # Handle existing project
+        cls._handle_existing_project(base_dir, fs, hooks_dir, overwrite)
+
+        # Create project structure
+        cls._create_project_structure(fs, hooks_dir)
+
+        # Initialize project configuration
+        cls._initialize_project_config(name, fs)
+
+        # Print success message and getting started guide
+        cls._print_success_message(name, base_dir)
+
+        return cls.load(
+            base_dir=base_dir,
+            storage_options=storage_options,
+            fs=fs,
+            log_level=log_level,
+        )
+
+    @classmethod
+    def _resolve_project_params(
+        cls,
+        name: str | None,
+        base_dir: str | None
+    ) -> tuple[str, str]:
+        """Resolve project name and base directory."""
         if name is None:
             name = str(Path.cwd().name)
             base_dir = posixpath.join(str(Path.cwd().parent), name)
@@ -370,54 +330,89 @@ class FlowerPowerProject:
         if base_dir is None:
             base_dir = posixpath.join(str(Path.cwd()), name)
 
+        return name, base_dir
+
+    @classmethod
+    def _setup_filesystem(
+        cls,
+        base_dir: str,
+        storage_options: dict | BaseStorageOptions | None,
+        fs: AbstractFileSystem | None
+    ) -> AbstractFileSystem:
+        """Setup filesystem for project operations."""
         if fs is None:
             fs = filesystem(
                 protocol_or_path=base_dir,
                 dirfs=True,
                 storage_options=storage_options,
             )
+        return fs
+
+    @classmethod
+    def _handle_existing_project(
+        cls,
+        base_dir: str,
+        fs: AbstractFileSystem,
+        hooks_dir: str,
+        overwrite: bool
+    ) -> None:
+        """Handle existing project directory."""
+        project_exists, _ = cls._check_project_exists(base_dir, fs)
 
-        # Check if project already exists
-        project_exists, message = cls._check_project_exists(base_dir, fs)
         if project_exists:
             if overwrite:
-                # Delete existing project files and directories
                 logger.info(f"Overwriting existing project at {base_dir}")
-                
-                # Remove directories recursively
-                config_path = f"{settings.CONFIG_DIR}"
-                pipelines_path = settings.PIPELINES_DIR
-                
-                if fs.exists(config_path):
-                    fs.rm(config_path, recursive=True)
-                if fs.exists(pipelines_path):
-                    fs.rm(pipelines_path, recursive=True)
-                if fs.exists(hooks_dir):
-                    fs.rm(hooks_dir, recursive=True)
-                
-                # Remove README.md file
-                if fs.exists("README.md"):
-                    fs.rm("README.md")
+
+                # Use FilesystemHelper to clean existing files
+                fs_helper = FilesystemHelper(base_dir)
+                fs_helper.clean_directory(
+                    fs,
+                    f"{settings.CONFIG_DIR}",
+                    settings.PIPELINES_DIR,
+                    hooks_dir,
+                    "README.md"
+                )
             else:
                 error_msg = f"Project already exists at {base_dir}. Use overwrite=True to overwrite the existing project."
                 rich.print(f"[red]{error_msg}[/red]")
                 logger.error(error_msg)
                 raise FileExistsError(error_msg)
 
+    @classmethod
+    def _create_project_structure(
+        cls,
+        fs: AbstractFileSystem,
+        hooks_dir: str
+    ) -> None:
+        """Create project directory structure."""
         fs.makedirs(f"{settings.CONFIG_DIR}/pipelines", exist_ok=True)
         fs.makedirs(settings.PIPELINES_DIR, exist_ok=True)
         fs.makedirs(hooks_dir, exist_ok=True)
 
+    @classmethod
+    def _initialize_project_config(
+        cls,
+        name: str,
+        fs: AbstractFileSystem
+    ) -> ProjectConfig:
+        """Initialize project configuration and create README."""
         # Load project configuration
         cfg = ProjectConfig.load(name=name, fs=fs)
 
+        # Create README file
         with fs.open("README.md", "w") as f:
             f.write(
                 f"# FlowerPower project {name.replace('_', ' ').upper()}\n\n"
                 f"**created on**\n\n*{dt.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}*\n\n"
             )
+
+        # Save configuration
         cfg.save(fs=fs)
+        return cfg
 
+    @classmethod
+    def _print_success_message(cls, name: str, base_dir: str) -> None:
+        """Print success message and getting started guide."""
         rich.print(
             f"\n✨ Initialized FlowerPower project [bold blue]{name}[/bold blue] "
             f"at [italic green]{base_dir}[/italic green]\n"
@@ -459,13 +454,6 @@ class FlowerPowerProject:
             )
         )
 
-        return cls.load(
-            base_dir=base_dir,
-            storage_options=storage_options,
-            fs=fs,
-            log_level=log_level,
-        )
-
 
 def initialize_project(
     name: str | None = None,
@@ -509,23 +497,9 @@ def create_project(
     fs: AbstractFileSystem | None = None,
     hooks_dir: str = settings.HOOKS_DIR,
 ) -> FlowerPowerProject:
-    """
-    Create or load a FlowerPower project.
-
-    If a project exists at the specified base_dir, it will be loaded.
-    Otherwise, a new project will be initialized.
-
-    Args:
-        name (str | None): The name of the project. If None, it defaults to the current directory name.
-        base_dir (str | None): The base directory where the project will be created or loaded from.
-                               If None, it defaults to the current working directory.
-        storage_options (dict | BaseStorageOptions | None): Storage options for the filesystem.
-        fs (AbstractFileSystem | None): An instance of AbstractFileSystem to use for file operations.
-        hooks_dir (str): The directory where the project hooks will be stored.
-
-    Returns:
-        FlowerPowerProject: An instance of FlowerPowerProject.
-    """
+    # Note: _check_project_exists expects base_dir to be a string.
+    # If base_dir is None, it will be handled by _check_project_exists or the load/init methods.
+    # We pass fs directly, as _check_project_exists can handle fs being None.
     # Note: _check_project_exists expects base_dir to be a string.
     # If base_dir is None, it will be handled by _check_project_exists or the load/init methods.
     # We pass fs directly, as _check_project_exists can handle fs being None.

flowerpower/pipeline/config_manager.py

@@ -0,0 +1,180 @@
+"""Configuration management for pipelines."""
+
+import os
+from typing import TYPE_CHECKING, Optional
+from fsspec_utils import AbstractFileSystem
+from ..utils.misc import get_filesystem
+from ..cfg import ProjectConfig, PipelineConfig
+from ..settings import CONFIG_DIR
+
+
+if TYPE_CHECKING:
+    from fsspec_utils import AbstractFileSystem
+
+
+class PipelineConfigManager:
+    """Handles loading, validation, and access to pipeline configurations.
+    
+    This class is responsible for:
+    - Loading project and pipeline configurations
+    - Validating configuration files
+    - Providing convenient access to configuration objects
+    - Managing configuration reload logic
+    """
+    
+    def __init__(
+        self, 
+        base_dir: str, 
+        fs: AbstractFileSystem, 
+        storage_options: dict,
+        cfg_dir: str = CONFIG_DIR
+    ):
+        """Initialize the configuration manager.
+        
+        Args:
+            base_dir: Base directory for the project
+            fs: Filesystem instance for file operations
+            storage_options: Storage options for filesystem
+            cfg_dir: Configuration directory name
+        """
+        self._base_dir = base_dir
+        self._fs = fs
+        self._storage_options = storage_options
+        self._cfg_dir = cfg_dir
+        self._project_cfg: Optional[ProjectConfig] = None
+        self._pipeline_cfg: Optional[PipelineConfig] = None
+        self._current_pipeline_name: Optional[str] = None
+    
+    def load_project_config(self, reload: bool = False) -> ProjectConfig:
+        """Load project configuration.
+        
+        Args:
+            reload: Whether to reload the configuration even if already loaded
+            
+        Returns:
+            ProjectConfig: The loaded project configuration
+        """
+        if self._project_cfg is None or reload:
+            from ..cfg import ProjectConfig
+            
+            # Construct config file path
+            cfg_path = f"{self._base_dir}/{self._cfg_dir}/project.yml"
+            
+            # Load configuration
+            fs = get_filesystem(fs=None, fs_type=self._base_dir)
+            self._project_cfg = ProjectConfig.from_yaml(path=f"{self._cfg_dir}/project.yml", fs=fs)
+            
+            # Add pipelines directory to Python path
+            self._add_modules_path(["pipelines"])
+        
+        return self._project_cfg
+    
+    def load_pipeline_config(self, name: str, reload: bool = False) -> PipelineConfig:
+        """Load pipeline configuration.
+
+        Args:
+            name: Name of the pipeline to load
+            reload: Whether to reload the configuration even if already loaded
+
+        Returns:
+            PipelineConfig: The loaded pipeline configuration
+        """
+        if (self._pipeline_cfg is None or
+            self._current_pipeline_name != name or
+            reload):
+
+            from ..cfg import PipelineConfig
+
+            # Ensure project config is loaded first
+            self.load_project_config(reload=reload)
+
+            # Create filesystem from storage_options
+            fs = get_filesystem(fs=None, fs_type=self._base_dir)
+
+            # Try different file locations and extensions
+            cfg_path = None
+            possible_paths = [
+                # Try .yml extension in pipelines/ subdirectory first
+                os.path.join(self._cfg_dir, "pipelines", f"{name}.yml"),
+                # Then try .yaml extension in pipelines/ subdirectory
+                os.path.join(self._cfg_dir, "pipelines", f"{name}.yaml"),
+                # Fallback to old paths for backward compatibility
+                os.path.join(self._cfg_dir, f"{name}.yml"),
+                os.path.join(self._cfg_dir, f"{name}.yaml"),
+            ]
+
+            for path in possible_paths:
+                try:
+                    if fs.exists(path):
+                        cfg_path = path
+                        break
+                except Exception:
+                    continue
+
+            if cfg_path is None:
+                raise FileNotFoundError(
+                    f"Pipeline configuration not found. Searched for: {possible_paths}"
+                )
+
+            # Load configuration
+            self._pipeline_cfg = PipelineConfig.from_yaml(
+                name=name,
+                path=cfg_path,
+                fs=fs,
+            )
+
+            # Update current pipeline name
+            self._current_pipeline_name = name
+
+        return self._pipeline_cfg
+    
+    @property
+    def project_config(self) -> ProjectConfig:
+        """Get the current project configuration.
+        
+        Returns:
+            ProjectConfig: The current project configuration
+            
+        Raises:
+            ValueError: If project configuration has not been loaded
+        """
+        if self._project_cfg is None:
+            raise ValueError("Project configuration not loaded. Call load_project_config() first.")
+        return self._project_cfg
+    
+    @property
+    def pipeline_config(self) -> PipelineConfig:
+        """Get the current pipeline configuration.
+        
+        Returns:
+            PipelineConfig: The current pipeline configuration
+            
+        Raises:
+            ValueError: If pipeline configuration has not been loaded
+        """
+        if self._pipeline_cfg is None:
+            raise ValueError("Pipeline configuration not loaded. Call load_pipeline_config() first.")
+        return self._pipeline_cfg
+    
+    @property
+    def current_pipeline_name(self) -> Optional[str]:
+        """Get the name of the currently loaded pipeline.
+        
+        Returns:
+            str | None: Name of the current pipeline, or None if none loaded
+        """
+        return self._current_pipeline_name
+    
+    def _add_modules_path(self, python_path: list[str]) -> None:
+        """Add module paths to Python path.
+        
+        Args:
+            python_path: List of paths to add to sys.path
+        """
+        import sys
+        from pathlib import Path
+        
+        for path in python_path:
+            path_obj = Path(self._base_dir) / path
+            if str(path_obj) not in sys.path:
+                sys.path.insert(0, str(path_obj))

flowerpower/pipeline/executor.py

@@ -0,0 +1,126 @@
+"""Pipeline execution handling."""
+
+from typing import TYPE_CHECKING, Any, Optional
+from fsspec_utils import AbstractFileSystem
+
+from ..cfg.pipeline.run import RunConfig
+from ..utils.config import merge_run_config_with_kwargs
+from ..utils.logging import setup_logging
+
+if TYPE_CHECKING:
+    from .config_manager import PipelineConfigManager
+    from .registry import PipelineRegistry
+
+
+class PipelineExecutor:
+    """Handles pipeline execution with comprehensive parameter handling.
+    
+    This class is responsible for:
+    - Executing pipelines with various configurations
+    - Merging runtime parameters with pipeline defaults
+    - Setting up execution environment (logging, etc.)
+    - Delegating to Pipeline objects for actual execution
+    """
+    
+    def __init__(
+        self,
+        config_manager: "PipelineConfigManager",
+        registry: "PipelineRegistry",
+        project_context: Optional[Any] = None
+    ):
+        """Initialize the pipeline executor.
+        
+        Args:
+            config_manager: Configuration manager for accessing pipeline configs
+            registry: Pipeline registry for accessing pipeline objects
+            project_context: Optional project context for execution
+        """
+        self._config_manager = config_manager
+        self._registry = registry
+        self._project_context = project_context
+    
+    def run(
+        self,
+        name: str,
+        run_config: Optional[RunConfig] = None,
+        **kwargs
+    ) -> dict[str, Any]:
+        """Execute a pipeline synchronously and return its results.
+        
+        This is the main method for running pipelines directly. It handles configuration
+        loading, adapter setup, and execution via Pipeline objects.
+        
+        Args:
+            name: Name of the pipeline to run. Must be a valid identifier.
+            run_config: Run configuration object containing all execution parameters.
+                If None, the default configuration from the pipeline will be used.
+            **kwargs: Additional parameters to override the run_config.
+                
+        Returns:
+            dict[str, Any]: Results of pipeline execution
+            
+        Raises:
+            ValueError: If pipeline configuration cannot be loaded
+            Exception: If pipeline execution fails
+        """
+        # Load pipeline configuration
+        pipeline_config = self._config_manager.load_pipeline_config(name=name)
+        
+        # Initialize run_config with pipeline defaults if not provided
+        run_config = run_config or pipeline_config.run
+        
+        # Merge kwargs into run_config
+        if kwargs:
+            run_config = merge_run_config_with_kwargs(run_config, kwargs)
+        
+        # Set up logging for this specific run if log_level is provided
+        if run_config.log_level is not None:
+            setup_logging(level=run_config.log_level)
+        
+        # Get the pipeline object from registry
+        pipeline = self._registry.get_pipeline(
+            name=name,
+            project_context=self._project_context,
+        )
+        
+        # Execute the pipeline
+        return pipeline.run(run_config=run_config)
+    
+    async def run_async(
+        self,
+        name: str,
+        run_config: Optional[RunConfig] = None,
+        **kwargs
+    ) -> dict[str, Any]:
+        """Execute a pipeline asynchronously and return its results.
+        
+        Args:
+            name: Name of the pipeline to run
+            run_config: Run configuration object
+            **kwargs: Additional parameters to override the run_config
+            
+        Returns:
+            dict[str, Any]: Results of pipeline execution
+        """
+        # Load pipeline configuration
+        pipeline_config = self._config_manager.load_pipeline_config(name=name)
+        
+        # Initialize run_config with pipeline defaults if not provided
+        run_config = run_config or pipeline_config.run
+        
+        # Merge kwargs into run_config
+        if kwargs:
+            run_config = merge_run_config_with_kwargs(run_config, kwargs)
+        
+        # Set up logging for this specific run if log_level is provided
+        if run_config.log_level is not None:
+            setup_logging(level=run_config.log_level)
+        
+        # Get the pipeline object from registry
+        pipeline = self._registry.get_pipeline(
+            name=name,
+            project_context=self._project_context,
+        )
+        
+        # Execute the pipeline asynchronously
+        return await pipeline.run_async(run_config=run_config)