FlowerPower 0.21.0__py3-none-any.whl → 0.31.0__py3-none-any.whl

flowerpower/utils/config.py

@@ -0,0 +1,306 @@
+"""
+Configuration utilities for FlowerPower.
+
+This module provides shared configuration handling utilities to avoid code duplication.
+"""
+
+from typing import Any, Dict
+from dataclasses import fields
+
+from ..cfg.pipeline.run import (
+    RunConfig,
+    ExecutorConfig,
+    WithAdapterConfig,
+)
+from .security import validate_config_dict, validate_callback_function
+def _merge_inputs(run_config: RunConfig, value):
+    """Merge inputs into run config."""
+    validate_config_dict(value)
+    if run_config.inputs is None:
+        run_config.inputs = value
+    else:
+        run_config.inputs.update(value)
+
+
+def _merge_config(run_config: RunConfig, value):
+    """Merge config into run config."""
+    validate_config_dict(value)
+    if run_config.config is None:
+        run_config.config = value
+    else:
+        run_config.config.update(value)
+
+
+def _set_cache(run_config: RunConfig, value):
+    """Set cache in run config."""
+    run_config.cache = value
+
+
+def _merge_adapter(run_config: RunConfig, value):
+    """Merge adapter into run config."""
+    if run_config.adapter is None:
+        run_config.adapter = value
+    else:
+        run_config.adapter.update(value)
+
+
+def _set_executor_cfg(run_config: RunConfig, value):
+    """Set executor config."""
+    if isinstance(value, str):
+        run_config.executor = ExecutorConfig(type=value)
+    elif isinstance(value, dict):
+        run_config.executor = ExecutorConfig.from_dict(value)
+    elif isinstance(value, ExecutorConfig):
+        run_config.executor = value
+
+
+def _set_with_adapter_cfg(run_config: RunConfig, value):
+    """Set with adapter config."""
+    if isinstance(value, dict):
+        run_config.with_adapter = WithAdapterConfig.from_dict(value)
+    elif isinstance(value, WithAdapterConfig):
+        run_config.with_adapter = value
+
+
+def _set_pipeline_adapter_cfg(run_config: RunConfig, value):
+    """Set pipeline adapter config."""
+    run_config.pipeline_adapter_cfg = value
+
+
+def _set_project_adapter_cfg(run_config: RunConfig, value):
+    """Set project adapter config."""
+    run_config.project_adapter_cfg = value
+
+
+_attr_handlers = {
+    'inputs': _merge_inputs,
+    'config': _merge_config,
+    'cache': _set_cache,
+    'adapter': _merge_adapter,
+    'executor_cfg': _set_executor_cfg,
+    'with_adapter_cfg': _set_with_adapter_cfg,
+    'pipeline_adapter_cfg': _set_pipeline_adapter_cfg,
+    'project_adapter_cfg': _set_project_adapter_cfg,
+}
+
+
+def merge_run_config_with_kwargs(run_config: RunConfig, kwargs: Dict[str, Any]) -> RunConfig:
+    """Merge kwargs into a RunConfig object.
+    
+    This utility function updates the RunConfig object with values from kwargs,
+    handling different types of attributes appropriately.
+    
+    Args:
+        run_config: The RunConfig object to update
+        kwargs: Dictionary of additional parameters to merge
+        
+    Returns:
+        RunConfig: Updated RunConfig object
+    """
+    # Handle complex attributes with specific logic
+    for attr, handler in _attr_handlers.items():
+        if attr in kwargs and kwargs[attr] is not None:
+            handler(run_config, kwargs[attr])
+
+    # Handle simple attributes
+    simple_attrs = [
+        'final_vars', 'reload', 'log_level', 'max_retries', 'retry_delay',
+        'jitter_factor', 'retry_exceptions', 'on_success', 'on_failure'
+    ]
+    
+    for attr in simple_attrs:
+        if attr in kwargs and kwargs[attr] is not None:
+            value = kwargs[attr]
+            # Validate callbacks for security
+            if attr in ['on_success', 'on_failure']:
+                validate_callback_function(value)
+            setattr(run_config, attr, value)
+    
+    return run_config
+
+
+class RunConfigBuilder:
+    """Builder pattern for constructing RunConfig objects with fluent interface."""
+    
+    def __init__(self, base_config: RunConfig | None = None):
+        self.config = base_config or RunConfig()
+    
+    def with_inputs(self, inputs: Dict[str, Any] | None) -> 'RunConfigBuilder':
+        """Set inputs configuration."""
+        if inputs is not None:
+            validate_config_dict(inputs)
+            self.config.inputs = inputs
+        return self
+    
+    def with_config(self, config: Dict[str, Any] | None) -> 'RunConfigBuilder':
+        """Set pipeline configuration."""
+        if config is not None:
+            validate_config_dict(config)
+            self.config.config = config
+        return self
+    
+    def with_cache(self, cache: bool | None) -> 'RunConfigBuilder':
+        """Set caching configuration."""
+        if cache is not None:
+            self.config.cache = cache
+        return self
+    
+    def with_adapter(self, adapter: Dict[str, Any] | None) -> 'RunConfigBuilder':
+        """Set adapter configuration."""
+        if adapter is not None:
+            if self.config.adapter is None:
+                self.config.adapter = adapter
+            else:
+                self.config.adapter.update(adapter)
+        return self
+    
+    def with_executor(self, executor_cfg: str | Dict[str, Any] | ExecutorConfig | None) -> 'RunConfigBuilder':
+        """Set executor configuration."""
+        if executor_cfg is not None:
+            if isinstance(executor_cfg, str):
+                self.config.executor = ExecutorConfig(type=executor_cfg)
+            elif isinstance(executor_cfg, dict):
+                self.config.executor = ExecutorConfig.from_dict(executor_cfg)
+            elif isinstance(executor_cfg, ExecutorConfig):
+                self.config.executor = executor_cfg
+        return self
+    
+    def with_retry_config(self, max_retries: int | None = None, retry_delay: float | None = None,
+                         jitter_factor: float | None = None, retry_exceptions: tuple | None = None) -> 'RunConfigBuilder':
+        """Set retry configuration."""
+        if max_retries is not None:
+            self.config.max_retries = max_retries
+        if retry_delay is not None:
+            self.config.retry_delay = retry_delay
+        if jitter_factor is not None:
+            self.config.jitter_factor = jitter_factor
+        if retry_exceptions is not None:
+            self.config.retry_exceptions = retry_exceptions
+        return self
+    
+    def with_logging(self, log_level: str | None = None) -> 'RunConfigBuilder':
+        """Set logging configuration."""
+        if log_level is not None:
+            self.config.log_level = log_level
+        return self
+    
+    def with_callbacks(self, on_success: str | None = None, on_failure: str | None = None) -> 'RunConfigBuilder':
+        """Set callback configurations."""
+        if on_success is not None:
+            validate_callback_function(on_success)
+            self.config.on_success = on_success
+        if on_failure is not None:
+            validate_callback_function(on_failure)
+            self.config.on_failure = on_failure
+        return self
+    
+    # Additional methods for backward compatibility with tests
+    def with_final_vars(self, final_vars: list[str] | None) -> 'RunConfigBuilder':
+        """Set final variables."""
+        if final_vars is not None:
+            self.config.final_vars = final_vars
+        return self
+    
+    def with_executor_cfg(self, executor_cfg: str | Dict[str, Any] | ExecutorConfig | None) -> 'RunConfigBuilder':
+        """Set executor configuration (alias for with_executor)."""
+        return self.with_executor(executor_cfg)
+    
+    def with_with_adapter_cfg(self, with_adapter_cfg: Dict[str, Any] | WithAdapterConfig | None) -> 'RunConfigBuilder':
+        """Set with_adapter configuration."""
+        if with_adapter_cfg is not None:
+            if isinstance(with_adapter_cfg, dict):
+                self.config.with_adapter = WithAdapterConfig.from_dict(with_adapter_cfg)
+            elif isinstance(with_adapter_cfg, WithAdapterConfig):
+                self.config.with_adapter = with_adapter_cfg
+        return self
+    
+    def with_pipeline_adapter_cfg(self, pipeline_adapter_cfg: Any | None) -> 'RunConfigBuilder':
+        """Set pipeline adapter configuration."""
+        if pipeline_adapter_cfg is not None:
+            self.config.pipeline_adapter_cfg = pipeline_adapter_cfg
+        return self
+    
+    def with_project_adapter_cfg(self, project_adapter_cfg: Any | None) -> 'RunConfigBuilder':
+        """Set project adapter configuration."""
+        if project_adapter_cfg is not None:
+            self.config.project_adapter_cfg = project_adapter_cfg
+        return self
+    
+    def with_reload(self, reload: bool | None) -> 'RunConfigBuilder':
+        """Set reload flag."""
+        if reload is not None:
+            self.config.reload = reload
+        return self
+    
+    def with_log_level(self, log_level: str | None) -> 'RunConfigBuilder':
+        """Set log level (alias for with_logging)."""
+        return self.with_logging(log_level)
+    
+    def with_max_retries(self, max_retries: int | None) -> 'RunConfigBuilder':
+        """Set max retries."""
+        if max_retries is not None:
+            self.config.max_retries = max_retries
+        return self
+    
+    def with_retry_delay(self, retry_delay: float | None) -> 'RunConfigBuilder':
+        """Set retry delay."""
+        if retry_delay is not None:
+            self.config.retry_delay = retry_delay
+        return self
+    
+    def with_jitter_factor(self, jitter_factor: float | None) -> 'RunConfigBuilder':
+        """Set jitter factor."""
+        if jitter_factor is not None:
+            self.config.jitter_factor = jitter_factor
+        return self
+    
+    def with_retry_exceptions(self, retry_exceptions: list | None) -> 'RunConfigBuilder':
+        """Set retry exceptions."""
+        if retry_exceptions is not None:
+            self.config.retry_exceptions = retry_exceptions
+        return self
+    
+    def with_on_success(self, on_success: Any | None) -> 'RunConfigBuilder':
+        """Set on_success callback."""
+        if on_success is not None:
+            self.config.on_success = on_success
+        return self
+    
+    def with_on_failure(self, on_failure: Any | None) -> 'RunConfigBuilder':
+        """Set on_failure callback."""
+        if on_failure is not None:
+            self.config.on_failure = on_failure
+        return self
+    
+    def reset(self) -> 'RunConfigBuilder':
+        """Reset builder to default values."""
+        self.config = RunConfig()
+        return self
+    
+    @classmethod
+    def from_config(cls, config: RunConfig) -> 'RunConfigBuilder':
+        """Create builder from existing config."""
+        return cls(base_config=config)
+    
+    def build(self) -> RunConfig:
+        """Build and return the RunConfig object."""
+        # Create a new copy to ensure immutability
+        return RunConfig(
+            inputs=self.config.inputs,
+            final_vars=self.config.final_vars,
+            config=self.config.config,
+            cache=self.config.cache,
+            executor=self.config.executor,
+            with_adapter=self.config.with_adapter,
+            pipeline_adapter_cfg=self.config.pipeline_adapter_cfg,
+            project_adapter_cfg=self.config.project_adapter_cfg,
+            adapter=self.config.adapter,
+            reload=self.config.reload,
+            log_level=self.config.log_level,
+            max_retries=self.config.max_retries,
+            retry_delay=self.config.retry_delay,
+            jitter_factor=self.config.jitter_factor,
+            retry_exceptions=self.config.retry_exceptions,
+            on_success=self.config.on_success,
+            on_failure=self.config.on_failure,
+        )

flowerpower/utils/executor.py

@@ -0,0 +1,178 @@
+"""
+Executor utilities for FlowerPower pipeline management.
+
+This module provides factory methods for creating executor instances
+with proper error handling and dependency management.
+"""
+
+from typing import Any, Dict, Optional, Union
+
+from loguru import logger
+
+# Lazy imports to avoid circular dependencies
+
+
+class ExecutorFactory:
+    """
+    Factory class for creating executor instances.
+
+    This class centralizes executor type selection and instance creation
+    to reduce complexity in the Pipeline class.
+    """
+
+    def __init__(self):
+        """Initialize the executor factory."""
+        self._executor_cache: Dict[str, Any] = {}
+
+    def create_executor(
+        self,
+        executor_cfg: Union[str, Dict[str, Any], Any, None]
+    ) -> Any:
+        """
+        Create an executor instance based on configuration.
+
+        Args:
+            executor_cfg: Executor configuration (string, dict, or ExecutorConfig)
+
+        Returns:
+            Executor instance
+        """
+        # Normalize configuration
+        executor_cfg = self._normalize_config(executor_cfg)
+
+        # Create executor based on type
+        executor_type = executor_cfg.type or "synchronous"
+        cache_key = f"{executor_type}_{hash(str(executor_cfg.to_dict()))}"
+
+        if cache_key in self._executor_cache:
+            return self._executor_cache[cache_key]
+
+        executor = self._create_executor_by_type(executor_cfg)
+        self._executor_cache[cache_key] = executor
+        return executor
+
+    def _normalize_config(
+        self,
+        executor_cfg: Union[str, Dict[str, Any], Any, None]
+    ) -> Any:
+        """Normalize executor configuration to ExecutorConfig instance."""
+        from ..cfg.pipeline.run import ExecutorConfig
+
+        if executor_cfg is None:
+            return ExecutorConfig()
+
+        if isinstance(executor_cfg, str):
+            return ExecutorConfig(type=executor_cfg)
+        elif isinstance(executor_cfg, dict):
+            return ExecutorConfig.from_dict(executor_cfg)
+        elif not isinstance(executor_cfg, ExecutorConfig):
+            raise TypeError(
+                "executor_cfg must be a string, dictionary, or ExecutorConfig instance."
+            )
+
+        return executor_cfg
+
+    def _create_executor_by_type(self, executor_cfg: Any) -> Any:
+        """Create executor based on type."""
+        executor_type = executor_cfg.type or "synchronous"
+
+        if executor_type in ("synchronous", None):
+            return self._create_synchronous_executor()
+        elif executor_type == "threadpool":
+            return self._create_threadpool_executor(executor_cfg)
+        elif executor_type == "processpool":
+            return self._create_processpool_executor(executor_cfg)
+        elif executor_type == "ray":
+            return self._create_ray_executor(executor_cfg)
+        elif executor_type == "dask":
+            return self._create_dask_executor(executor_cfg)
+        else:
+            logger.warning(
+                f"Unknown executor type: {executor_type}. Using local executor."
+            )
+            return self._create_synchronous_executor()
+
+    def _create_synchronous_executor(self) -> Any:
+        """Create synchronous/local executor."""
+        from hamilton.execution.executors import SynchronousLocalTaskExecutor
+        return SynchronousLocalTaskExecutor()
+
+    def _create_threadpool_executor(self, executor_cfg: Any) -> Any:
+        """Create thread pool executor."""
+        try:
+            from hamilton.plugins.h_threadpool import ThreadPoolExecutor
+
+            # Extract max workers from config
+            if executor_cfg.max_workers is not None:
+                return ThreadPoolExecutor(max_workers=executor_cfg.max_workers)
+            return ThreadPoolExecutor()
+        except ImportError:
+            logger.warning(
+                "ThreadPool executor dependencies not installed. Using local executor."
+            )
+            return self._create_synchronous_executor()
+
+    def _create_processpool_executor(self, executor_cfg: Any) -> Any:
+        """Create process pool executor."""
+        try:
+            from hamilton.execution.executors import ProcessPoolExecutor
+
+            # Extract max workers from config
+            if executor_cfg.max_workers is not None:
+                return ProcessPoolExecutor(max_workers=executor_cfg.max_workers)
+            return ProcessPoolExecutor()
+        except ImportError:
+            logger.warning(
+                "ProcessPool executor dependencies not installed. Using local executor."
+            )
+            return self._create_synchronous_executor()
+
+    def _create_ray_executor(self, executor_cfg: Any) -> Any:
+        """Create Ray executor."""
+        try:
+            from hamilton.plugins.h_ray import RayTaskExecutor
+
+            # Extract configuration
+            config = {}
+            if executor_cfg.num_cpus is not None:
+                config['num_cpus'] = executor_cfg.num_cpus
+            if config:
+                return RayTaskExecutor(**config)
+            return RayTaskExecutor()
+        except ImportError:
+            logger.warning(
+                "Ray executor dependencies not installed. Using local executor."
+            )
+            return self._create_synchronous_executor()
+
+    def _create_dask_executor(self, executor_cfg: Any) -> Any:
+        """Create Dask executor."""
+        try:
+            from hamilton.plugins.h_dask import DaskExecutor
+
+            # Extract configuration
+            config = {}
+            if executor_cfg.num_cpus is not None:
+                config['num_cpus'] = executor_cfg.num_cpus
+            if config:
+                return DaskExecutor(**config)
+            return DaskExecutor()
+        except ImportError:
+            logger.warning(
+                "Dask executor dependencies not installed. Using local executor."
+            )
+            return self._create_synchronous_executor()
+
+    def clear_cache(self) -> None:
+        """Clear the executor cache."""
+        self._executor_cache.clear()
+
+
+def create_executor_factory() -> ExecutorFactory:
+    """
+    Factory function to create an ExecutorFactory instance.
+
+    Returns:
+        ExecutorFactory: Configured factory instance
+    """
+    return ExecutorFactory()

flowerpower/utils/filesystem.py

@@ -0,0 +1,194 @@
+"""
+Filesystem utilities for FlowerPower pipeline management.
+
+This module provides helper classes and functions for common filesystem operations
+used throughout the FlowerPower codebase.
+"""
+
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+from fsspec_utils import AbstractFileSystem, filesystem
+from loguru import logger
+from .security import validate_file_path
+
+
+class FilesystemHelper:
+    """
+    Helper class for filesystem operations with caching and error handling.
+
+    This class provides centralized filesystem operations with proper error handling
+    and logging for common operations like directory creation, path resolution,
+    and filesystem initialization.
+    """
+
+    def __init__(self, base_dir: str, storage_options: Optional[Dict[str, Any]] = None):
+        """
+        Initialize the filesystem helper.
+
+        Args:
+            base_dir: Base directory for filesystem operations
+            storage_options: Storage options for filesystem access
+        """
+        # Validate base directory for security
+        validate_file_path(base_dir, allow_relative=True)
+        self._base_dir = base_dir
+        self._storage_options = storage_options or {}
+        self._fs_cache: Dict[str, AbstractFileSystem] = {}
+
+    def get_filesystem(self, cached: bool = False, cache_storage: Optional[str] = None) -> AbstractFileSystem:
+        """
+        Get a filesystem instance with optional caching.
+
+        Args:
+            cached: Whether to use cached filesystem
+            cache_storage: Storage path for cached filesystem
+
+        Returns:
+            AbstractFileSystem: Configured filesystem instance
+        """
+        cache_key = f"{self._base_dir}_{cached}_{cache_storage}"
+
+        if cache_key not in self._fs_cache:
+            if cached and cache_storage:
+                # Ensure cache storage directory exists
+                cache_path = Path(cache_storage)
+                cache_path.mkdir(parents=True, exist_ok=True)
+
+            self._fs_cache[cache_key] = filesystem(
+                self._base_dir,
+                storage_options=self._storage_options,
+                cached=cached,
+                cache_storage=cache_storage,
+            )
+
+        return self._fs_cache[cache_key]
+
+    def ensure_directories_exist(
+        self,
+        fs: AbstractFileSystem,
+        *directories: str,
+        exist_ok: bool = True
+    ) -> None:
+        """
+        Ensure that the specified directories exist.
+
+        Args:
+            fs: Filesystem instance
+            *directories: Directory paths to create
+            exist_ok: Whether to ignore existing directories
+
+        Raises:
+            RuntimeError: If directory creation fails
+        """
+        for directory in directories:
+            # Validate directory path for security
+            validate_file_path(directory, allow_relative=True)
+            try:
+                fs.makedirs(directory, exist_ok=exist_ok)
+            except (OSError, PermissionError) as e:
+                logger.error(f"Error creating directory {directory}: {e}")
+                raise RuntimeError(f"Failed to create directory {directory}: {e}") from e
+            except Exception as e:
+                logger.error(f"Unexpected error creating directory {directory}: {e}")
+                raise RuntimeError(f"Unexpected filesystem error for {directory}: {e}") from e
+
+    def resolve_path(self, fs: AbstractFileSystem, *path_parts: str) -> str:
+        """
+        Resolve a path in the filesystem.
+
+        Args:
+            fs: Filesystem instance
+            *path_parts: Path components to join
+
+        Returns:
+            str: Resolved path
+        """
+        if hasattr(fs, 'path'):
+            base_path = fs.path
+        else:
+            base_path = self._base_dir
+
+        resolved_path = fs.join(base_path, *path_parts)
+        # Validate resolved path for security
+        validate_file_path(resolved_path, allow_relative=True)
+        return resolved_path
+
+    def clean_directory(
+        self,
+        fs: AbstractFileSystem,
+        *paths: str,
+        recursive: bool = True
+    ) -> None:
+        """
+        Clean specified paths if they exist.
+
+        Args:
+            fs: Filesystem instance
+            *paths: Paths to clean
+            recursive: Whether to remove recursively
+        """
+        for path in paths:
+            # Validate path for security before cleaning
+            validate_file_path(path, allow_relative=True)
+            if fs.exists(path):
+                try:
+                    fs.rm(path, recursive=recursive)
+                except Exception as e:
+                    logger.warning(f"Failed to clean path {path}: {e}")
+
+    def sync_filesystem(self, fs: AbstractFileSystem) -> None:
+        """
+        Sync filesystem cache if applicable.
+
+        Args:
+            fs: Filesystem instance to sync
+        """
+        if hasattr(fs, 'is_cache_fs') and fs.is_cache_fs:
+            fs.sync_cache()
+
+            # Log sync information if available
+            if hasattr(fs, '_mapper') and hasattr(fs, 'cache_path'):
+                logger.debug(
+                    f"Synced filesystem cache: {fs._mapper.directory} -> {fs.cache_path}"
+                )
+
+    def get_project_path(self, fs: AbstractFileSystem) -> str:
+        """
+        Get the project path for the filesystem.
+
+        Args:
+            fs: Filesystem instance
+
+        Returns:
+            str: Project path
+        """
+        if hasattr(fs, 'is_cache_fs') and fs.is_cache_fs:
+            project_path = fs._mapper.directory
+        else:
+            project_path = getattr(fs, 'path', self._base_dir)
+        
+        # Validate project path for security
+        validate_file_path(project_path, allow_relative=True)
+        return project_path
+
+    def clear_cache(self) -> None:
+        """Clear the filesystem cache."""
+        self._fs_cache.clear()
+
+
+def create_filesystem_helper(
+    base_dir: str,
+    storage_options: Optional[Dict[str, Any]] = None
+) -> FilesystemHelper:
+    """
+    Factory function to create a FilesystemHelper instance.
+
+    Args:
+        base_dir: Base directory for filesystem operations
+        storage_options: Storage options for filesystem access
+
+    Returns:
+        FilesystemHelper: Configured helper instance
+    """
+    return FilesystemHelper(base_dir, storage_options)