FlowerPower 0.11.6.20__py3-none-any.whl → 0.21.0__py3-none-any.whl

flowerpower/cfg/pipeline/builder.py

@@ -0,0 +1,377 @@
+import copy
+from typing import Any, Callable, Optional, Union
+
+from fsspec_utils import AbstractFileSystem, BaseStorageOptions, filesystem
+
+from ... import settings
+from ..base import BaseConfig
+from .adapter import AdapterConfig as PipelineAdapterConfig
+from .run import ExecutorConfig, RunConfig, WithAdapterConfig
+from ..project.adapter import AdapterConfig as ProjectAdapterConfig
+
+
+class RunConfigBuilder:
+    """A fluent builder for creating RunConfig objects.
+    
+    This builder provides a clean interface for constructing RunConfig objects
+    with proper configuration merging from project and pipeline defaults.
+    """
+    
+    def __init__(
+        self, 
+        pipeline_name: str, 
+        base_dir: str | None = None,
+        fs: AbstractFileSystem | None = None,
+        storage_options: dict | BaseStorageOptions | None = {}
+    ):
+        """Initialize the RunConfigBuilder.
+        
+        Args:
+            pipeline_name: Name of the pipeline to build config for
+            base_dir: Base directory for the project (defaults to current directory)
+            fs: Optional filesystem instance
+            storage_options: Options for filesystem access
+        """
+        self.pipeline_name = pipeline_name
+        self.base_dir = base_dir or "."
+        self._fs = fs
+        self._storage_options = storage_options
+        
+        # Initialize with empty config
+        self._config = RunConfig()
+        
+        # Load defaults from pipeline and project configs
+        self._load_defaults()
+    
+    def _load_defaults(self):
+        """Load default configuration from pipeline and project YAML files."""
+        if self._fs is None:
+            self._fs = filesystem(
+                self.base_dir, 
+                cached=False, 
+                dirfs=True, 
+                storage_options=self._storage_options
+            )
+        
+        # Load pipeline configuration
+        try:
+            from .. import PipelineConfig
+            pipeline_cfg = PipelineConfig.load(
+                base_dir=self.base_dir,
+                name=self.pipeline_name,
+                fs=self._fs,
+                storage_options=self._storage_options
+            )
+            if pipeline_cfg and pipeline_cfg.run:
+                self._config = copy.deepcopy(pipeline_cfg.run)
+        except Exception:
+            # If pipeline config doesn't exist, use defaults
+            pass
+        
+        # Load project configuration for adapter defaults
+        try:
+            from .. import ProjectConfig
+            project_cfg = ProjectConfig.load(
+                base_dir=self.base_dir,
+                fs=self._fs,
+                storage_options=self._storage_options
+            )
+            if project_cfg and project_cfg.adapter:
+                # Store project adapter config for merging
+                self._project_adapter_cfg = project_cfg.adapter
+            else:
+                self._project_adapter_cfg = ProjectAdapterConfig()
+        except Exception:
+            self._project_adapter_cfg = ProjectAdapterConfig()
+    
+    def with_inputs(self, inputs: dict) -> "RunConfigBuilder":
+        """Set pipeline input values.
+        
+        Args:
+            inputs: Dictionary of input values to override defaults
+            
+        Returns:
+            Self for method chaining
+        """
+        if inputs:
+            if self._config.inputs is None:
+                self._config.inputs = {}
+            self._config.inputs.update(inputs)
+        return self
+    
+    def with_final_vars(self, final_vars: list[str]) -> "RunConfigBuilder":
+        """Set the final output variables.
+        
+        Args:
+            final_vars: List of variable names to return from execution
+            
+        Returns:
+            Self for method chaining
+        """
+        self._config.final_vars = final_vars
+        return self
+    
+    def with_config(self, config: dict) -> "RunConfigBuilder":
+        """Set Hamilton driver configuration.
+        
+        Args:
+            config: Dictionary of configuration values for Hamilton
+            
+        Returns:
+            Self for method chaining
+        """
+        if config:
+            if self._config.config is None:
+                self._config.config = {}
+            self._config.config.update(config)
+        return self
+    
+    def with_cache(self, cache: Union[dict, bool]) -> "RunConfigBuilder":
+        """Set cache configuration.
+        
+        Args:
+            cache: Cache configuration (dict) or enable/disable flag (bool)
+            
+        Returns:
+            Self for method chaining
+        """
+        self._config.cache = cache
+        return self
+    
+    def with_executor(self, executor_type: str, **kwargs) -> "RunConfigBuilder":
+        """Set executor configuration.
+        
+        Args:
+            executor_type: Type of executor ('synchronous', 'threadpool', 'processpool', 'ray', 'dask')
+            **kwargs: Additional executor configuration options
+            
+        Returns:
+            Self for method chaining
+        """
+        if not self._config.executor:
+            self._config.executor = ExecutorConfig()
+        
+        self._config.executor.type = executor_type
+        
+        # Apply additional executor options
+        for key, value in kwargs.items():
+            if hasattr(self._config.executor, key):
+                setattr(self._config.executor, key, value)
+        
+        return self
+    
+    def with_adapter(self, adapter_name: str, **kwargs) -> "RunConfigBuilder":
+        """Enable and configure a specific adapter.
+        
+        Args:
+            adapter_name: Name of the adapter ('hamilton_tracker', 'mlflow', 'opentelemetry', etc.)
+            **kwargs: Adapter-specific configuration options
+            
+        Returns:
+            Self for method chaining
+        """
+        if not self._config.with_adapter:
+            self._config.with_adapter = WithAdapterConfig()
+        
+        # Enable the adapter
+        if hasattr(self._config.with_adapter, adapter_name):
+            setattr(self._config.with_adapter, adapter_name, True)
+            
+            # Store adapter configuration for merging
+            if not hasattr(self, '_adapter_configs'):
+                self._adapter_configs = {}
+            self._adapter_configs[adapter_name] = kwargs
+        
+        return self
+    
+    def with_retries(
+        self, 
+        max_attempts: int = 3, 
+        delay: float = 1.0, 
+        jitter: float = 0.1,
+        exceptions: Optional[list] = None
+    ) -> "RunConfigBuilder":
+        """Configure retry behavior.
+        
+        Args:
+            max_attempts: Maximum number of retry attempts
+            delay: Base delay between retries in seconds
+            jitter: Random jitter factor to add to retry delay
+            exceptions: List of exception types that should trigger retries
+            
+        Returns:
+            Self for method chaining
+        """
+        self._config.max_retries = max_attempts
+        self._config.retry_delay = delay
+        self._config.jitter_factor = jitter
+        
+        if exceptions:
+            self._config.retry_exceptions = exceptions
+        
+        return self
+    
+    def with_callbacks(
+        self, 
+        on_success: Optional[Callable] = None, 
+        on_failure: Optional[Callable] = None
+    ) -> "RunConfigBuilder":
+        """Set success and failure callbacks.
+        
+        Args:
+            on_success: Callback function to execute on successful completion
+            on_failure: Callback function to execute on failure
+            
+        Returns:
+            Self for method chaining
+        """
+        if on_success:
+            self._config.on_success = on_success
+        if on_failure:
+            self._config.on_failure = on_failure
+        
+        return self
+    
+    def with_log_level(self, log_level: str) -> "RunConfigBuilder":
+        """Set the log level for execution.
+        
+        Args:
+            log_level: Log level ('DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL')
+            
+        Returns:
+            Self for method chaining
+        """
+        self._config.log_level = log_level
+        return self
+    
+    def with_reload(self, reload: bool = True) -> "RunConfigBuilder":
+        """Set whether to reload the pipeline module.
+        
+        Args:
+            reload: Whether to force reload of the pipeline module
+            
+        Returns:
+            Self for method chaining
+        """
+        self._config.reload = reload
+        return self
+    
+    def with_pipeline_adapter_config(self, config: dict) -> "RunConfigBuilder":
+        """Set pipeline-specific adapter configuration.
+        
+        Args:
+            config: Pipeline adapter configuration dictionary
+            
+        Returns:
+            Self for method chaining
+        """
+        if config:
+            if self._config.pipeline_adapter_cfg is None:
+                self._config.pipeline_adapter_cfg = {}
+            self._config.pipeline_adapter_cfg.update(config)
+        return self
+    
+    def with_project_adapter_config(self, config: dict) -> "RunConfigBuilder":
+        """Set project-level adapter configuration.
+        
+        Args:
+            config: Project adapter configuration dictionary
+            
+        Returns:
+            Self for method chaining
+        """
+        if config:
+            if self._config.project_adapter_cfg is None:
+                self._config.project_adapter_cfg = {}
+            self._config.project_adapter_cfg.update(config)
+        return self
+    
+    def with_custom_adapter(self, name: str, adapter: Any) -> "RunConfigBuilder":
+        """Add a custom adapter instance.
+        
+        Args:
+            name: Name/identifier for the adapter
+            adapter: Adapter instance
+            
+        Returns:
+            Self for method chaining
+        """
+        if self._config.adapter is None:
+            self._config.adapter = {}
+        self._config.adapter[name] = adapter
+        return self
+    
+    def build(self) -> RunConfig:
+        """Build the final RunConfig object.
+        
+        This method merges all configurations and validates the final result.
+        
+        Returns:
+            Fully configured RunConfig object
+            
+        Raises:
+            ValueError: If configuration is invalid
+        """
+        # Create a deep copy to avoid modifying the internal state
+        final_config = copy.deepcopy(self._config)
+        
+        # Merge adapter configurations
+        if hasattr(self, '_adapter_configs') and self._adapter_configs:
+            self._merge_adapter_configs(final_config)
+        
+        # Validate configuration
+        self._validate_config(final_config)
+        
+        return final_config
+    
+    def _merge_adapter_configs(self, config: RunConfig):
+        """Merge adapter configurations from builder with project/pipeline configs."""
+        if not config.pipeline_adapter_cfg:
+            config.pipeline_adapter_cfg = {}
+        
+        if not config.project_adapter_cfg:
+            config.project_adapter_cfg = {}
+        
+        # Merge project adapter defaults
+        for adapter_name, adapter_config in self._adapter_configs.items():
+            if adapter_name in ['hamilton_tracker', 'mlflow', 'opentelemetry']:
+                # Merge with project config
+                if hasattr(self._project_adapter_cfg, adapter_name):
+                    project_config = getattr(self._project_adapter_cfg, adapter_name).to_dict()
+                    adapter_config = {**project_config, **adapter_config}
+                
+                # Store in pipeline adapter config
+                if adapter_name not in config.pipeline_adapter_cfg:
+                    config.pipeline_adapter_cfg[adapter_name] = {}
+                config.pipeline_adapter_cfg[adapter_name].update(adapter_config)
+    
+    def _validate_config(self, config: RunConfig):
+        """Validate the final configuration.
+        
+        Args:
+            config: RunConfig object to validate
+            
+        Raises:
+            ValueError: If configuration is invalid
+        """
+        # Validate retry configuration
+        if config.max_retries < 0:
+            raise ValueError("max_retries must be non-negative")
+        
+        if config.retry_delay < 0:
+            raise ValueError("retry_delay must be non-negative")
+        
+        if config.jitter_factor is not None and config.jitter_factor < 0:
+            raise ValueError("jitter_factor must be non-negative")
+        
+        # Validate executor configuration
+        if config.executor and config.executor.type:
+            valid_executors = ['synchronous', 'threadpool', 'processpool', 'ray', 'dask']
+            if config.executor.type not in valid_executors:
+                raise ValueError(f"Invalid executor type: {config.executor.type}")
+        
+        # Validate log level
+        if config.log_level:
+            valid_levels = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']
+            if config.log_level.upper() not in valid_levels:
+                raise ValueError(f"Invalid log level: {config.log_level}")

flowerpower/cfg/pipeline/run.py

@@ -1,5 +1,6 @@
 import msgspec
 from munch import munchify
+from typing import Any, Callable
 
 from ... import settings
 from ..base import BaseConfig
@@ -33,6 +34,13 @@ class RunConfig(BaseConfig):
     retry_delay: int | float = msgspec.field(default=1)
     jitter_factor: float | None = msgspec.field(default=0.1)
     retry_exceptions: list[str] = msgspec.field(default_factory=lambda: ["Exception"])
+    # New fields for comprehensive configuration
+    pipeline_adapter_cfg: dict | None = msgspec.field(default=None)
+    project_adapter_cfg: dict | None = msgspec.field(default=None)
+    adapter: dict[str, Any] | None = msgspec.field(default=None)
+    reload: bool = msgspec.field(default=False)
+    on_success: Callable | tuple[Callable, tuple | None, dict | None] | None = msgspec.field(default=None)
+    on_failure: Callable | tuple[Callable, tuple | None, dict | None] | None = msgspec.field(default=None)
 
     def __post_init__(self):
         if isinstance(self.inputs, dict):
@@ -45,3 +53,31 @@ class RunConfig(BaseConfig):
             self.with_adapter = WithAdapterConfig.from_dict(self.with_adapter)
         if isinstance(self.executor, dict):
             self.executor = ExecutorConfig.from_dict(self.executor)
+        if isinstance(self.pipeline_adapter_cfg, dict):
+            from ..pipeline.adapter import AdapterConfig as PipelineAdapterConfig
+            self.pipeline_adapter_cfg = PipelineAdapterConfig.from_dict(self.pipeline_adapter_cfg)
+        if isinstance(self.project_adapter_cfg, dict):
+            from ..project.adapter import AdapterConfig as ProjectAdapterConfig
+            self.project_adapter_cfg = ProjectAdapterConfig.from_dict(self.project_adapter_cfg)
+        if isinstance(self.adapter, dict):
+            # Convert adapter instances if needed
+            pass
+        if isinstance(self.retry_exceptions, list):
+            # Convert string exceptions to actual exception classes
+            converted_exceptions = []
+            for exc in self.retry_exceptions:
+                if isinstance(exc, str):
+                    try:
+                        exc_class = eval(exc)
+                        # Ensure it's actually an exception class
+                        if isinstance(exc_class, type) and issubclass(exc_class, BaseException):
+                            converted_exceptions.append(exc_class)
+                        else:
+                            converted_exceptions.append(Exception)
+                    except (NameError, AttributeError):
+                        converted_exceptions.append(Exception)
+                elif isinstance(exc, type) and issubclass(exc, BaseException):
+                    converted_exceptions.append(exc)
+                else:
+                    converted_exceptions.append(Exception)
+            self.retry_exceptions = converted_exceptions

flowerpower/cfg/project/__init__.py

@@ -1,20 +1,20 @@
 import msgspec
+from fsspec_utils import AbstractFileSystem, BaseStorageOptions, filesystem
+import posixpath
 
-from ...fs import AbstractFileSystem, BaseStorageOptions, get_filesystem
+from ...settings import CONFIG_DIR
 from ..base import BaseConfig
 from .adapter import AdapterConfig
-from .job_queue import JobQueueConfig
 
 
 class ProjectConfig(BaseConfig):
     """A configuration class for managing project-level settings in FlowerPower.
 
-    This class handles project-wide configuration including job queue and adapter settings.
+    This class handles project-wide configuration including adapter settings.
     It supports loading from and saving to YAML files, with filesystem abstraction.
 
     Attributes:
         name (str | None): The name of the project.
-        job_queue (JobQueueConfig): Configuration for the job queue component.
         adapter (AdapterConfig): Configuration for the adapter component.
 
     Example:
@@ -31,12 +31,9 @@ class ProjectConfig(BaseConfig):
     """
 
     name: str | None = msgspec.field(default=None)
-    job_queue: JobQueueConfig = msgspec.field(default_factory=JobQueueConfig)
     adapter: AdapterConfig = msgspec.field(default_factory=AdapterConfig)
 
     def __post_init__(self):
-        if isinstance(self.job_queue, dict):
-            self.job_queue = JobQueueConfig.from_dict(self.job_queue)
         if isinstance(self.adapter, dict):
             self.adapter = AdapterConfig.from_dict(self.adapter)
 
@@ -45,7 +42,6 @@ class ProjectConfig(BaseConfig):
         cls,
         base_dir: str = ".",
         name: str | None = None,
-        job_queue_type: str | None = None,
         fs: AbstractFileSystem | None = None,
         storage_options: dict | BaseStorageOptions | None = {},
     ):
@@ -54,7 +50,6 @@ class ProjectConfig(BaseConfig):
         Args:
             base_dir (str, optional): Base directory for the project. Defaults to ".".
             name (str | None, optional): Project name. Defaults to None.
-            job_queue_type (str | None, optional): Type of job queue to use. Defaults to None.
             fs (AbstractFileSystem | None, optional): Filesystem to use. Defaults to None.
             storage_options (dict | Munch, optional): Options for filesystem. Defaults to empty Munch.
 
@@ -65,22 +60,18 @@ class ProjectConfig(BaseConfig):
             ```python
             project = ProjectConfig.load(
                 base_dir="my_project",
-                name="pipeline1",
-                job_queue_type="rq"
+                name="pipeline1"
                 )
             ```
         """
         if fs is None:
-            fs = get_filesystem(
+            fs = filesystem(
                 base_dir, cached=False, dirfs=True, storage_options=storage_options
             )
         if fs.exists("conf/project.yml"):
             project = ProjectConfig.from_yaml(path="conf/project.yml", fs=fs)
         else:
             project = ProjectConfig(name=name)
-        if job_queue_type is not None:
-            if job_queue_type != project.job_queue.type:
-                project.job_queue.update_type(job_queue_type)
 
         return project
 
@@ -103,18 +94,17 @@ class ProjectConfig(BaseConfig):
             ```
         """
         if fs is None:
-            fs = get_filesystem(
+            fs = filesystem(
                 base_dir, cached=True, dirfs=True, storage_options=storage_options
             )
 
-        fs.makedirs("conf", exist_ok=True)
-        self.to_yaml(path="conf/project.yml", fs=fs)
+        fs.makedirs(CONFIG_DIR, exist_ok=True)
+        self.to_yaml(path=posixpath.join(CONFIG_DIR, "project.yml"), fs=fs)
 
 
 def init_project_config(
     base_dir: str = ".",
     name: str | None = None,
-    job_queue_type: str | None = None,
     fs: AbstractFileSystem | None = None,
     storage_options: dict | BaseStorageOptions | None = {},
 ):
@@ -125,7 +115,6 @@ def init_project_config(
     Args:
         base_dir (str, optional): Base directory for the project. Defaults to ".".
         name (str | None, optional): Project name. Defaults to None.
-        job_queue_type (str | None, optional): Type of job queue to use. Defaults to None.
         fs (AbstractFileSystem | None, optional): Filesystem to use. Defaults to None.
         storage_options (dict | Munch, optional): Options for filesystem. Defaults to empty Munch.
 
@@ -136,17 +125,15 @@ def init_project_config(
         ```python
         project = init_project_config(
             base_dir="my_project",
-            name="test_project",
-            job_queue_type="rq"
+            name="test_project"
         )
         ```
     """
     project = ProjectConfig.load(
         base_dir=base_dir,
         name=name,
-        job_queue_type=job_queue_type,
         fs=fs,
         storage_options=storage_options,
     )
     project.save(base_dir=base_dir, fs=fs, storage_options=storage_options)
-    return project
+    return project

flowerpower/cfg/project/adapter.py

@@ -24,18 +24,6 @@ class OpenTelemetryConfig(BaseConfig):
     port: int = msgspec.field(default=6831)
 
 
-# class OpenLineageConfig(BaseConfig):
-#     from openlineage.client import OpenLineageClientOptions
-#     from openlineage.client.transport import Transport
-#     from openlineage.client.transport import TransportFactory
-#     url: str | None = msgspec.field(default=None)
-#     options: OpenLineageClientOptions | None = msgspec.field(
-#         default=None)
-#     transport: Transport | None = msgspec.field(default=None)
-#     factory: TransportFactory | None = msgspec.field(
-#         default=None)
-#     config: dict | None = msgspec.field(default=None)
-
 
 class RayConfig(BaseConfig):
     ray_init_config: dict | None = msgspec.field(default=None)

flowerpower/cli/__init__.py

@@ -4,7 +4,7 @@ import os
 import typer
 from loguru import logger
 
-from ..flowerpower import init as init_
+from ..flowerpower import FlowerPowerProject
 from .pipeline import app as pipeline_app
 from .utils import parse_dict_or_list_param
 
@@ -18,14 +18,6 @@ app.add_typer(
     pipeline_app, name="pipeline", help="Manage and execute FlowerPower pipelines"
 )
 
-if importlib.util.find_spec("apscheduler") or importlib.util.find_spec("rq"):
-    from .job_queue import app as job_queue_app
-
-    app.add_typer(
-        job_queue_app,
-        name="job-queue",
-        help="Manage job queue workers and scheduled tasks",
-    )
 
 if importlib.util.find_spec("paho"):
     from .mqtt import app as mqtt_app
@@ -49,12 +41,6 @@ def init(
     storage_options: str = typer.Option(
         None, "--storage-options", "-s", help="Storage options as a JSON or dict string"
     ),
-    job_queue_type: str = typer.Option(
-        "rq",
-        "--job-queue-type",
-        "-q",
-        help="Job queue backend type to use (rq, apscheduler)",
-    ),
 ):
     """
     Initialize a new FlowerPower project.
@@ -69,7 +55,6 @@ def init(
         base_dir: Base directory where the project will be created. If not provided,
                   the current directory's parent will be used
         storage_options: Storage options for filesystem access, as a JSON or dict string
-        job_queue_type: Type of job queue backend to use (rq, apscheduler)
 
     Examples:
         # Create a project in the current directory using its name
@@ -80,9 +65,6 @@ def init(
 
         # Create a project in a specific location
         $ flowerpower init --name my-project --base-dir /path/to/projects
-
-        # Create a project with APScheduler as the job queue backend
-        $ flowerpower init --job-queue-type apscheduler
     """
     parsed_storage_options = {}
     if storage_options:
@@ -95,11 +77,10 @@ def init(
             raise typer.Exit(code=1)
 
     try:
-        init_(
+        FlowerPowerProject.new(
             name=project_name,
             base_dir=base_dir,
             storage_options=parsed_storage_options,
-            job_queue_type=job_queue_type,
         )
     except Exception as e:
         logger.error(f"Error initializing project: {e}")

flowerpower/cli/cfg.py

@@ -1,8 +1,5 @@
 import typer
 
-from ..cfg import Config
-from ..cli.utils import parse_dict_or_list_param
-
 app = typer.Typer(help="Config management commands")
 
 

flowerpower/cli/mqtt.py

@@ -69,7 +69,6 @@ def run_pipeline_on_message(
     with_opentelemetry: bool = False,
     with_progressbar: bool = False,
     storage_options: str | None = None,
-    as_job: bool = False,
     host: str | None = None,
     port: int | None = None,
     username: str | None = None,
@@ -107,7 +106,6 @@ def run_pipeline_on_message(
         with_opentelemetry: Enable OpenTelemetry tracing
         with_progressbar: Enable progress bar
         storage_options: Storage options as JSON, dict string or key=value pairs
-        as_job: Run as a job in the scheduler
         host: MQTT broker host
         port: MQTT broker port
         username: MQTT broker username
@@ -128,9 +126,6 @@ def run_pipeline_on_message(
         # Configure retries for resilience
         $ flowerpower mqtt run-pipeline-on-message my_pipeline --topic sensors/data --max-retries 5 --retry-delay 2.0
 
-        # Run as a job with custom MQTT settings
-        $ flowerpower mqtt run-pipeline-on-message my_pipeline --topic events/process --as-job --qos 2 --host mqtt.example.com
-
         # Use a config hook to process messages
         $ flowerpower mqtt run-pipeline-on-message my_pipeline --topic data/incoming --config-hook process_message
 
@@ -158,7 +153,6 @@ def run_pipeline_on_message(
         with_opentelemetry=with_opentelemetry,
         with_progressbar=with_progressbar,
         storage_options=parsed_storage_options,
-        as_job=as_job,
         host=host,
         port=port,
         username=username,