FlowerPower 0.21.0__py3-none-any.whl → 0.31.0__py3-none-any.whl

flowerpower/pipeline/pipeline.py

@@ -9,6 +9,7 @@ import importlib.util
 import random
 import time
 from typing import TYPE_CHECKING, Any, Callable
+from requests.exceptions import HTTPError, ConnectionError, Timeout # Example exception
 
 import humanize
 import msgspec
@@ -20,6 +21,8 @@ from hamilton_sdk.api.clients import UnauthorizedException
 from requests.exceptions import ConnectionError, HTTPError
 
 from .. import settings
+from ..utils.adapter import create_adapter_manager
+from ..utils.executor import create_executor_factory
 
 if importlib.util.find_spec("opentelemetry"):
     from hamilton.plugins import h_opentelemetry
@@ -57,8 +60,9 @@ else:
 
 from ..cfg import PipelineConfig, ProjectConfig
 from ..cfg.pipeline.adapter import AdapterConfig as PipelineAdapterConfig
-from ..cfg.pipeline.run import ExecutorConfig, RunConfig, WithAdapterConfig
+from ..cfg.pipeline.run import ExecutorConfig, RunConfig
 from ..cfg.project.adapter import AdapterConfig as ProjectAdapterConfig
+from ..utils.config import merge_run_config_with_kwargs
 
 if TYPE_CHECKING:
     from ..flowerpower import FlowerPowerProject
@@ -82,83 +86,20 @@ class Pipeline(msgspec.Struct):
     config: PipelineConfig
     module: Any
     project_context: FlowerPowerProject
+    _adapter_manager: Any = None
+    _executor_factory: Any = None
 
     def __post_init__(self):
-        """Initialize Hamilton settings."""
+        """Initialize Hamilton settings and utility managers."""
         if not settings.HAMILTON_TELEMETRY_ENABLED:
             disable_telemetry()
         if not settings.HAMILTON_AUTOLOAD_EXTENSIONS:
             disable_autoload()
 
-    def _merge_run_config_with_kwargs(self, run_config: RunConfig, kwargs: dict) -> RunConfig:
-        """Merge kwargs into the run_config object.
-        
-        Args:
-            run_config: The base RunConfig object to merge into
-            kwargs: Additional parameters to merge into the run_config
-            
-        Returns:
-            Updated RunConfig object with merged kwargs
-        """
-        from copy import deepcopy
-        
-        # Create a deep copy of the run_config to avoid modifying the original
-        merged_config = deepcopy(run_config)
-        
-        # Handle each possible kwarg
-        for key, value in kwargs.items():
-            if key == 'inputs' and value is not None:
-                if merged_config.inputs is None:
-                    merged_config.inputs = {}
-                merged_config.inputs.update(value)
-            elif key == 'final_vars' and value is not None:
-                if merged_config.final_vars is None:
-                    merged_config.final_vars = []
-                merged_config.final_vars = value
-            elif key == 'config' and value is not None:
-                if merged_config.config is None:
-                    merged_config.config = {}
-                merged_config.config.update(value)
-            elif key == 'cache' and value is not None:
-                merged_config.cache = value
-            elif key == 'executor_cfg' and value is not None:
-                if isinstance(value, str):
-                    merged_config.executor = ExecutorConfig(type=value)
-                elif isinstance(value, dict):
-                    merged_config.executor = ExecutorConfig.from_dict(value)
-                elif isinstance(value, ExecutorConfig):
-                    merged_config.executor = value
-            elif key == 'with_adapter_cfg' and value is not None:
-                if isinstance(value, dict):
-                    merged_config.with_adapter = WithAdapterConfig.from_dict(value)
-                elif isinstance(value, WithAdapterConfig):
-                    merged_config.with_adapter = value
-            elif key == 'pipeline_adapter_cfg' and value is not None:
-                merged_config.pipeline_adapter_cfg = value
-            elif key == 'project_adapter_cfg' and value is not None:
-                merged_config.project_adapter_cfg = value
-            elif key == 'adapter' and value is not None:
-                if merged_config.adapter is None:
-                    merged_config.adapter = {}
-                merged_config.adapter.update(value)
-            elif key == 'reload' and value is not None:
-                merged_config.reload = value
-            elif key == 'log_level' and value is not None:
-                merged_config.log_level = value
-            elif key == 'max_retries' and value is not None:
-                merged_config.max_retries = value
-            elif key == 'retry_delay' and value is not None:
-                merged_config.retry_delay = value
-            elif key == 'jitter_factor' and value is not None:
-                merged_config.jitter_factor = value
-            elif key == 'retry_exceptions' and value is not None:
-                merged_config.retry_exceptions = value
-            elif key == 'on_success' and value is not None:
-                merged_config.on_success = value
-            elif key == 'on_failure' and value is not None:
-                merged_config.on_failure = value
-        
-        return merged_config
+        # Initialize utility managers
+        self._adapter_manager = create_adapter_manager()
+        self._executor_factory = create_executor_factory()
+
 
     def run(
         self,
@@ -182,7 +123,7 @@ class Pipeline(msgspec.Struct):
         
         # Merge kwargs into the run_config
         if kwargs:
-            run_config = self._merge_run_config_with_kwargs(run_config, kwargs)
+            run_config = merge_run_config_with_kwargs(run_config, kwargs)
 
         # Reload module if requested
         if run_config.reload:
@@ -222,23 +163,33 @@ class Pipeline(msgspec.Struct):
         # Convert string exceptions to actual exception classes
         if retry_exceptions and isinstance(retry_exceptions, (list, tuple)):
             converted_exceptions = []
+            # Safe mapping of exception names to classes
+            exception_mapping = {
+                'Exception': Exception,
+                'ValueError': ValueError,
+                'TypeError': TypeError,
+                'RuntimeError': RuntimeError,
+                'FileNotFoundError': FileNotFoundError,
+                'PermissionError': PermissionError,
+                'ConnectionError': ConnectionError,
+                'TimeoutError': TimeoutError,
+                'KeyError': KeyError,
+                'AttributeError': AttributeError,
+                'ImportError': ImportError,
+                'OSError': OSError,
+                'IOError': IOError,
+                'HTTPError': HTTPError,
+                'ConnectionError': ConnectionError,
+                'Timeout': Timeout,
+            }
             for exc in retry_exceptions:
                 if isinstance(exc, str):
-                    try:
-                        exc_class = eval(exc)
-                        # Ensure it's actually an exception class
-                        if isinstance(exc_class, type) and issubclass(
-                            exc_class, BaseException
-                        ):
-                            converted_exceptions.append(exc_class)
-                        else:
-                            logger.warning(
-                                f"'{exc}' is not an exception class, using Exception"
-                            )
-                            converted_exceptions.append(Exception)
-                    except (NameError, AttributeError):
+                    exc_class = exception_mapping.get(exc)
+                    if exc_class is not None:
+                        converted_exceptions.append(exc_class)
+                    else:
                         logger.warning(
-                            f"Unknown exception type: {exc}, using Exception"
+                            f"Unknown exception '{exc}', using Exception as fallback"
                         )
                         converted_exceptions.append(Exception)
                 elif isinstance(exc, type) and issubclass(exc, BaseException):
@@ -379,6 +330,7 @@ class Pipeline(msgspec.Struct):
         """Get the executor based on the provided configuration."""
         logger.debug("Setting up executor...")
 
+        # Merge with default configuration
         if executor_cfg:
             if isinstance(executor_cfg, str):
                 executor_cfg = ExecutorConfig(type=executor_cfg)
@@ -393,60 +345,25 @@ class Pipeline(msgspec.Struct):
         else:
             executor_cfg = self.config.run.executor
 
-        if executor_cfg.type is None or executor_cfg.type == "synchronous":
-            logger.debug("Using SynchronousLocalTaskExecutor as default.")
-            return executors.SynchronousLocalTaskExecutor(), None
+        # Create executor using factory
+        executor = self._executor_factory.create_executor(executor_cfg)
 
-        if executor_cfg.type == "threadpool":
-            logger.debug(
-                f"Using MultiThreadingExecutor with max_workers={executor_cfg.max_workers}"
-            )
-            return executors.MultiThreadingExecutor(
-                max_tasks=executor_cfg.max_workers
-            ), None
-        elif executor_cfg.type == "processpool":
-            logger.debug(
-                f"Using MultiProcessingExecutor with max_workers={executor_cfg.max_workers}"
-            )
-            return executors.MultiProcessingExecutor(
-                max_tasks=executor_cfg.max_workers
-            ), None
-        elif executor_cfg.type == "ray":
-            if h_ray:
-                logger.debug(
-                    f"Using RayTaskExecutor with num_cpus={executor_cfg.num_cpus}"
-                )
-
-                # Handle temporary case where project_context is PipelineManager
-                project_cfg = getattr(
-                    self.project_context, "project_cfg", None
-                ) or getattr(self.project_context, "_project_cfg", None)
+        # Handle special cleanup for certain executor types
+        cleanup_fn = None
+        if executor_cfg.type == "ray" and h_ray:
+            # Handle temporary case where project_context is PipelineManager
+            project_cfg = getattr(
+                self.project_context, "project_cfg", None
+            ) or getattr(self.project_context, "_project_cfg", None)
 
-                return (
-                    h_ray.RayTaskExecutor(
-                        num_cpus=executor_cfg.num_cpus,
-                        ray_init_config=project_cfg.adapter.ray.ray_init_config,
-                    ),
+            if project_cfg and hasattr(project_cfg.adapter, 'ray'):
+                cleanup_fn = (
                     ray.shutdown
                     if project_cfg.adapter.ray.shutdown_ray_on_completion
-                    else None,
+                    else None
                 )
-            else:
-                logger.warning("Ray is not installed. Using local executor.")
-                return executors.SynchronousLocalTaskExecutor(), None
-        elif executor_cfg.type == "dask":
-            if distributed:
-                cluster = distributed.LocalCluster()
-                client = distributed.Client(cluster)
-                return h_dask.DaskExecutor(client=client), cluster.close
-            else:
-                logger.warning("Dask is not installed. Using local executor.")
-                return executors.SynchronousLocalTaskExecutor(), None
-        else:
-            logger.warning(
-                f"Unknown executor type: {executor_cfg.type}. Using local executor."
-            )
-            return executors.SynchronousLocalTaskExecutor(), None
+
+        return executor, cleanup_fn
 
     def _get_adapters(
         self,
@@ -458,152 +375,23 @@ class Pipeline(msgspec.Struct):
         """Set up the adapters for the pipeline."""
         logger.debug("Setting up adapters...")
 
-        # Resolve adapter configurations
-        if with_adapter_cfg:
-            if isinstance(with_adapter_cfg, dict):
-                with_adapter_cfg = WithAdapterConfig.from_dict(with_adapter_cfg)
-            elif not isinstance(with_adapter_cfg, WithAdapterConfig):
-                raise TypeError(
-                    "with_adapter must be a dictionary or WithAdapterConfig instance."
-                )
-
-            with_adapter_cfg = self.config.run.with_adapter.merge(with_adapter_cfg)
-        else:
-            with_adapter_cfg = self.config.run.with_adapter
-
-        if pipeline_adapter_cfg:
-            if isinstance(pipeline_adapter_cfg, dict):
-                pipeline_adapter_cfg = PipelineAdapterConfig.from_dict(
-                    pipeline_adapter_cfg
-                )
-            elif not isinstance(pipeline_adapter_cfg, PipelineAdapterConfig):
-                raise TypeError(
-                    "pipeline_adapter_cfg must be a dictionary or PipelineAdapterConfig instance."
-                )
-
-            pipeline_adapter_cfg = self.config.adapter.merge(pipeline_adapter_cfg)
-        else:
-            pipeline_adapter_cfg = self.config.adapter
-
-        if project_adapter_cfg:
-            if isinstance(project_adapter_cfg, dict):
-                project_adapter_cfg = ProjectAdapterConfig.from_dict(
-                    project_adapter_cfg
-                )
-            elif not isinstance(project_adapter_cfg, ProjectAdapterConfig):
-                raise TypeError(
-                    "project_adapter_cfg must be a dictionary or ProjectAdapterConfig instance."
-                )
-
-            # Handle temporary case where project_context is PipelineManager
-            manager_project_cfg = getattr(
-                self.project_context, "project_cfg", None
-            ) or getattr(self.project_context, "_project_cfg", None)
-            if manager_project_cfg and hasattr(manager_project_cfg, "adapter"):
-                project_adapter_cfg = manager_project_cfg.adapter.merge(
-                    project_adapter_cfg
-                )
-            else:
-                # Use project context directly if it's FlowerPowerProject
-                if hasattr(self.project_context, "pipeline_manager"):
-                    pm_cfg = getattr(
-                        self.project_context.pipeline_manager, "project_cfg", None
-                    ) or getattr(
-                        self.project_context.pipeline_manager, "_project_cfg", None
-                    )
-                    base_cfg = pm_cfg.adapter if pm_cfg else None
-                    if base_cfg:
-                        project_adapter_cfg = base_cfg.merge(project_adapter_cfg)
-                    else:
-                        from ..cfg.project.adapter import \
-                            AdapterConfig as ProjectAdapterConfig
-
-                        project_adapter_cfg = ProjectAdapterConfig()
-                else:
-                    from ..cfg.project.adapter import \
-                        AdapterConfig as ProjectAdapterConfig
-
-                    project_adapter_cfg = ProjectAdapterConfig()
-        else:
-            # Handle temporary case where project_context is PipelineManager
-            manager_project_cfg = getattr(
-                self.project_context, "project_cfg", None
-            ) or getattr(self.project_context, "_project_cfg", None)
-            if manager_project_cfg and hasattr(manager_project_cfg, "adapter"):
-                project_adapter_cfg = manager_project_cfg.adapter
-            else:
-                # Use project context directly if it's FlowerPowerProject
-                if hasattr(self.project_context, "pipeline_manager"):
-                    pm_cfg = getattr(
-                        self.project_context.pipeline_manager, "project_cfg", None
-                    ) or getattr(
-                        self.project_context.pipeline_manager, "_project_cfg", None
-                    )
-                    project_adapter_cfg = pm_cfg.adapter if pm_cfg else None
-                else:
-                    project_adapter_cfg = None
-
-            # Create default adapter config if none found
-            if project_adapter_cfg is None:
-                from ..cfg.project.adapter import \
-                    AdapterConfig as ProjectAdapterConfig
-
-                project_adapter_cfg = ProjectAdapterConfig()
-
-        adapters = []
-
-        # Hamilton Tracker adapter
-        if with_adapter_cfg.hamilton_tracker:
-            tracker_kwargs = project_adapter_cfg.hamilton_tracker.to_dict()
-            tracker_kwargs.update(pipeline_adapter_cfg.hamilton_tracker.to_dict())
-            tracker_kwargs["hamilton_api_url"] = tracker_kwargs.pop("api_url", None)
-            tracker_kwargs["hamilton_ui_url"] = tracker_kwargs.pop("ui_url", None)
+        # Resolve adapter configurations using the adapter manager
+        with_adapter_cfg = self._adapter_manager.resolve_with_adapter_config(
+            with_adapter_cfg, self.config.run.with_adapter
+        )
 
-            constants.MAX_DICT_LENGTH_CAPTURE = (
-                tracker_kwargs.pop("max_dict_length_capture", None)
-                or settings.HAMILTON_MAX_DICT_LENGTH_CAPTURE
-            )
-            constants.MAX_LIST_LENGTH_CAPTURE = (
-                tracker_kwargs.pop("max_list_length_capture", None)
-                or settings.HAMILTON_MAX_LIST_LENGTH_CAPTURE
-            )
-            constants.CAPTURE_DATA_STATISTICS = (
-                tracker_kwargs.pop("capture_data_statistics", None)
-                or settings.HAMILTON_CAPTURE_DATA_STATISTICS
-            )
+        pipeline_adapter_cfg = self._adapter_manager.resolve_pipeline_adapter_config(
+            pipeline_adapter_cfg, self.config.adapter
+        )
 
-            tracker = HamiltonTracker(**tracker_kwargs)
-            adapters.append(tracker)
+        project_adapter_cfg = self._adapter_manager.resolve_project_adapter_config(
+            project_adapter_cfg, self.project_context
+        )
 
-        # MLFlow adapter
-        if with_adapter_cfg.mlflow:
-            if h_mlflow is None:
-                logger.warning("MLFlow is not installed. Skipping MLFlow adapter.")
-            else:
-                mlflow_kwargs = project_adapter_cfg.mlflow.to_dict()
-                mlflow_kwargs.update(pipeline_adapter_cfg.mlflow.to_dict())
-                mlflow_adapter = h_mlflow.MLFlowTracker(**mlflow_kwargs)
-                adapters.append(mlflow_adapter)
-
-        # OpenTelemetry adapter
-        if with_adapter_cfg.opentelemetry:
-            if h_opentelemetry is None:
-                logger.warning(
-                    "OpenTelemetry is not installed. Skipping OpenTelemetry adapter."
-                )
-            else:
-                otel_kwargs = project_adapter_cfg.opentelemetry.to_dict()
-                otel_kwargs.update(pipeline_adapter_cfg.opentelemetry.to_dict())
-                init_tracer()
-                otel_adapter = h_opentelemetry.OpenTelemetryTracker(**otel_kwargs)
-                adapters.append(otel_adapter)
-
-        # Progress bar adapter
-        if with_adapter_cfg.progressbar:
-            progressbar_kwargs = project_adapter_cfg.progressbar.to_dict()
-            progressbar_kwargs.update(pipeline_adapter_cfg.progressbar.to_dict())
-            progressbar_adapter = h_rich.ProgressBar(**progressbar_kwargs)
-            adapters.append(progressbar_adapter)
+        # Create adapters
+        adapters = self._adapter_manager.create_adapters(
+            with_adapter_cfg, pipeline_adapter_cfg, project_adapter_cfg
+        )
 
         # Add any additional adapters
         if adapter:

flowerpower/pipeline/registry.py

@@ -8,6 +8,8 @@ import sys
 from dataclasses import dataclass
 from typing import TYPE_CHECKING, Any, Dict
 
+import msgspec
+
 import rich
 from fsspec_utils import AbstractFileSystem, filesystem
 from loguru import logger
@@ -48,8 +50,7 @@ class HookType(str, Enum):
         return self.value
 
 
-@dataclass
-class CachedPipelineData:
+class CachedPipelineData(msgspec.Struct):
     """Container for cached pipeline data."""
     pipeline: "Pipeline"
     config: PipelineConfig
@@ -832,5 +833,45 @@ class PipelineRegistry:
             f.write(template.format(function_name=function_name))
 
         rich.print(
-            f"🔧 Added hook [bold blue]{type.value}[/bold blue] to {to} as {function_name} for {name}"
-        )
+            f"🔧 Added hook [bold blue]{type.value}[/bold blue] to {to} as {function_name} for {name}")
+       
+
+    def create_pipeline(
+        self,
+        name: str,
+        overwrite: bool = False,
+        template: str | None = None,
+        tags: list[str] | None = None,
+        description: str | None = None
+    ) -> None:
+        """Create a new pipeline.
+
+        This method provides compatibility with the lifecycle manager interface.
+        Additional parameters (template, tags, description) are currently not used.
+
+        Args:
+            name: Name of the pipeline to create
+            overwrite: Whether to overwrite existing pipeline
+            template: Template to use (not currently implemented)
+            tags: Tags for the pipeline (not currently implemented)
+            description: Description of the pipeline (not currently implemented)
+        """
+        self.new(name=name, overwrite=overwrite)
+
+    def delete_pipeline(
+        self,
+        name: str,
+        cfg: bool = True,
+        module: bool = False
+    ) -> None:
+        """Delete a pipeline.
+
+        This method provides compatibility with the lifecycle manager interface.
+
+        Args:
+            name: Name of the pipeline to delete
+            cfg: Whether to delete configuration files
+            module: Whether to delete module files
+        """
+        self.delete(name=name, cfg=cfg, module=module)
+        

flowerpower/utils/__init__.py

@@ -0,0 +1,19 @@
+"""
+Utility modules for FlowerPower.
+
+This package contains utility classes and functions that help simplify
+the main codebase by centralizing common operations.
+"""
+
+from .adapter import AdapterManager, create_adapter_manager
+from .executor import ExecutorFactory, create_executor_factory
+from .filesystem import FilesystemHelper, create_filesystem_helper
+
+__all__ = [
+    "AdapterManager",
+    "create_adapter_manager",
+    "ExecutorFactory",
+    "create_executor_factory",
+    "FilesystemHelper",
+    "create_filesystem_helper",
+]