FlowerPower 0.20.0__py3-none-any.whl → 0.21.0__py3-none-any.whl

flowerpower/__init__.py

@@ -1,22 +1,18 @@
 import importlib.metadata
 
 from .cfg import Config, PipelineConfig, ProjectConfig
-from .flowerpower import FlowerPower, FlowerPowerProject
-from .flowerpower import init as init_project  # noqa: E402
-from .job_queue import JobQueueManager  # noqa: E402
+from .flowerpower import FlowerPower, FlowerPowerProject, create_project
 from .pipeline import PipelineManager
 
 __version__ = importlib.metadata.version("FlowerPower")
 
 __all__ = [
     "__version__",
-    "init_project",
+    "create_project",
     "FlowerPower",
     "FlowerPowerProject",
     "PipelineManager",
-    "JobQueueManager",
     "Config",
     "ProjectConfig",
     "PipelineConfig",
-    "PipelineConfig",
 ]

flowerpower/cfg/__init__.py

@@ -4,6 +4,7 @@ import msgspec
 from fsspec_utils import AbstractFileSystem, BaseStorageOptions, filesystem
 from munch import Munch
 
+from ..settings import CONFIG_DIR, PIPELINES_DIR
 from .base import BaseConfig
 from .pipeline import PipelineConfig, init_pipeline_config
 from .project import ProjectConfig, init_project_config
@@ -49,7 +50,6 @@ class Config(BaseConfig):
         base_dir: str = ".",
         name: str | None = None,
         pipeline_name: str | None = None,
-        job_queue_type: str | None = None,
         fs: AbstractFileSystem | None = None,
         storage_options: dict | BaseStorageOptions | None = {},
     ):
@@ -59,7 +59,6 @@ class Config(BaseConfig):
             base_dir (str, optional): Base directory for configurations. Defaults to ".".
             name (str | None, optional): Project name. Defaults to None.
             pipeline_name (str | None, optional): Pipeline name. Defaults to None.
-            job_queue_type (str | None, optional): Type of job queue to use. Defaults to None.
             fs (AbstractFileSystem | None, optional): Filesystem to use. Defaults to None.
             storage_options (dict | Munch, optional): Options for filesystem. Defaults to empty Munch.
 
@@ -72,7 +71,6 @@ class Config(BaseConfig):
                 base_dir="my_project",
                 name="test_project",
                 pipeline_name="etl",
-                job_queue_type="rq"
             )
             ```
         """
@@ -83,7 +81,6 @@ class Config(BaseConfig):
         project = ProjectConfig.load(
             base_dir=base_dir,
             name=name,
-            job_queue_type=job_queue_type,
             fs=fs,
             storage_options=storage_options,
         )
@@ -127,11 +124,11 @@ class Config(BaseConfig):
                 self.base_dir, cached=True, dirfs=True, **storage_options
             )
 
-        if not self.fs.exists("conf"):
-            self.fs.makedirs("conf")
+        if not self.fs.exists(CONFIG_DIR):
+            self.fs.makedirs(CONFIG_DIR)
 
         if pipeline:
-            self.fs.makedirs("conf/pipelines", exist_ok=True)
+            self.fs.makedirs(PIPELINES_DIR, exist_ok=True)
             h_params = self.pipeline.pop("h_params") if self.pipeline.h_params else None
             self.pipeline.to_yaml(
                 path=f"conf/pipelines/{self.pipeline.name}.yml", fs=self.fs
@@ -210,7 +207,6 @@ def init_config(
     base_dir: str = ".",
     name: str | None = None,
     pipeline_name: str | None = None,
-    job_queue_type: str | None = None,
     fs: AbstractFileSystem | None = None,
     storage_options: dict | BaseStorageOptions | None = {},
 ):
@@ -223,7 +219,6 @@ def init_config(
         base_dir (str, optional): Base directory for configurations. Defaults to ".".
         name (str | None, optional): Project name. Defaults to None.
         pipeline_name (str | None, optional): Pipeline name. Defaults to None.
-        job_queue_type (str | None, optional): Type of job queue to use. Defaults to None.
         fs (AbstractFileSystem | None, optional): Filesystem to use. Defaults to None.
         storage_options (dict | Munch, optional): Options for filesystem. Defaults to empty Munch.
 
@@ -236,7 +231,6 @@ def init_config(
             base_dir="my_project",
             name="test_project",
             pipeline_name="data-pipeline",
-            job_queue_type="rq"
         )
         ```
     """
@@ -249,7 +243,6 @@ def init_config(
     project_cfg = init_project_config(
         base_dir=base_dir,
         name=name,
-        job_queue_type=job_queue_type,
         fs=fs,
         storage_options=storage_options,
     )

flowerpower/cfg/base.py

@@ -2,7 +2,8 @@ import copy
 from typing import Any, Self
 
 import msgspec
-from fsspec import AbstractFileSystem, filesystem
+from fsspec_utils import AbstractFileSystem, filesystem
+from ..utils.misc import get_filesystem
 
 
 class BaseConfig(msgspec.Struct, kw_only=True):
@@ -20,12 +21,10 @@ class BaseConfig(msgspec.Struct, kw_only=True):
         Raises:
             NotImplementedError: If the filesystem does not support writing files.
         """
-        if fs is None:
-            fs = filesystem("file")
+        fs = get_filesystem(fs)
         try:
             with fs.open(path, "wb") as f:
                 f.write(msgspec.yaml.encode(self, order="deterministic"))
-                # yaml.dump(self.to_dict(), f, default_flow_style=False)
         except NotImplementedError:
             raise NotImplementedError("The filesystem does not support writing files.")
 
@@ -54,29 +53,42 @@ class BaseConfig(msgspec.Struct, kw_only=True):
             An instance of the class with the values from the YAML file.
 
         """
-        if fs is None:
-            fs = filesystem("file")
+        fs = get_filesystem(fs)
         with fs.open(path) as f:
-            # data = yaml.full_load(f)
-            # return cls.from_dict(data)
             return msgspec.yaml.decode(f.read(), type=cls, strict=False)
 
-    def update(self, d: dict[str, Any]) -> None:
+    def _apply_dict_updates(self, target: Self, d: dict[str, Any]) -> None:
+        """
+        Helper method to apply dictionary updates to a target instance.
+        
+        Args:
+            target: The target instance to apply updates to.
+            d: The dictionary containing updates to apply.
+        """
         for k, v in d.items():
-            if hasattr(self, k):
-                current_value = getattr(self, k)
+            if hasattr(target, k):
+                current_value = getattr(target, k)
                 if isinstance(current_value, dict) and isinstance(v, dict):
                     current_value.update(v)
                 else:
-                    setattr(self, k, v)
+                    setattr(target, k, v)
             else:
-                setattr(self, k, v)
+                # Use object.__setattr__ to bypass msgspec.Struct's restrictions
+                object.__setattr__(target, k, v)
+
+    def update(self, d: dict[str, Any]) -> None:
+        """
+        Updates this instance with values from the provided dictionary.
+        
+        Args:
+            d: The dictionary containing updates to apply.
+        """
+        self._apply_dict_updates(self, d)
 
     def merge_dict(self, d: dict[str, Any]) -> Self:
         """
         Creates a copy of this instance and updates the copy with values
-        from the provided dictionary, only if the dictionary field's value is not
-        its default value. The original instance (self) is not modified.
+        from the provided dictionary. The original instance (self) is not modified.
 
         Args:
             d: The dictionary to get values from.
@@ -84,16 +96,8 @@ class BaseConfig(msgspec.Struct, kw_only=True):
         Returns:
             A new instance of the struct with updated values.
         """
-        self_copy = copy.copy(self)
-        for k, v in d.items():
-            if hasattr(self_copy, k):
-                current_value = getattr(self_copy, k)
-                if isinstance(current_value, dict) and isinstance(v, dict):
-                    current_value.update(v)
-                else:
-                    setattr(self_copy, k, v)
-            else:
-                setattr(self_copy, k, v)
+        self_copy = copy.deepcopy(self)
+        self._apply_dict_updates(self_copy, d)
         return self_copy
 
     def merge(self, source: Self) -> Self:

flowerpower/cfg/pipeline/__init__.py

@@ -9,7 +9,7 @@ from .adapter import AdapterConfig
 from .run import ExecutorConfig as ExecutorConfig
 from .run import RunConfig
 from .run import WithAdapterConfig as WithAdapterConfig
-from .schedule import ScheduleConfig
+#from .schedule import ScheduleConfig
 
 
 class PipelineConfig(BaseConfig):
@@ -22,7 +22,7 @@ class PipelineConfig(BaseConfig):
     Attributes:
         name (str | None): The name of the pipeline.
         run (RunConfig): Configuration for pipeline execution.
-        schedule (ScheduleConfig): Configuration for pipeline scheduling.
+        schedule (ScheduleConfig): Configuration for pipeline scheduling. DEPRECATED.
         params (dict): Pipeline parameters.
         adapter (AdapterConfig): Configuration for the pipeline adapter.
         h_params (dict): Hamilton-formatted parameters.
@@ -45,7 +45,7 @@ class PipelineConfig(BaseConfig):
 
     name: str | None = msgspec.field(default=None)
     run: RunConfig = msgspec.field(default_factory=RunConfig)
-    schedule: ScheduleConfig = msgspec.field(default_factory=ScheduleConfig)
+    #: ScheduleConfig = msgspec.field(default_factory=ScheduleConfig)
     params: dict = msgspec.field(default_factory=dict)
     adapter: AdapterConfig = msgspec.field(default_factory=AdapterConfig)
     h_params: dict = msgspec.field(default_factory=dict)

flowerpower/cfg/pipeline/_schedule.py

@@ -0,0 +1,32 @@
+import datetime as dt
+
+import msgspec
+from munch import munchify
+
+from ..base import BaseConfig
+
+
+class ScheduleConfig(BaseConfig):
+    cron: str | dict | None = msgspec.field(default=None)
+    interval: str | int | dict | None = msgspec.field(default=None)
+    date: str | None = msgspec.field(default=None)
+
+    def __post_init__(self):
+        if isinstance(self.date, str):
+            try:
+                self.date = dt.datetime.fromisoformat(self.date)
+            except ValueError:
+                raise ValueError(
+                    f"Invalid date format: {self.date}. Expected ISO format."
+                )
+        if isinstance(self.cron, dict):
+            self.cron = munchify(self.cron)
+        if isinstance(self.interval, dict):
+            self.interval = munchify(self.interval)
+
+
+# class ScheduleConfig(BaseConfig):
+#     run: ScheduleRunConfig = msgspec.field(default_factory=ScheduleRunConfig)
+#     trigger: ScheduleTriggerConfig = msgspec.field(
+#         default_factory=ScheduleTriggerConfig
+#     )

flowerpower/cfg/pipeline/adapter.py

@@ -39,17 +39,12 @@ class MLFlowConfig(BaseConfig):
             self.run_tags = munchify(self.run_tags)
 
 
-# class OpenLineageConfig(BaseConfig):
-#     namespace : str | None = msgspec.field(default=None)
-#     job_name : str | None = msgspec.field(default=None)
-
 
 class AdapterConfig(BaseConfig):
     hamilton_tracker: HamiltonTracerConfig = msgspec.field(
         default_factory=HamiltonTracerConfig
     )
     mlflow: MLFlowConfig = msgspec.field(default_factory=MLFlowConfig)
-    # openlineage: OpenLineageConfig | dict = msgspec.field(default_factory=OpenLineageConfig)
 
     def __post_init__(self):
         if isinstance(self.hamilton_tracker, dict):

flowerpower/cfg/pipeline/builder.py

@@ -0,0 +1,377 @@
+import copy
+from typing import Any, Callable, Optional, Union
+
+from fsspec_utils import AbstractFileSystem, BaseStorageOptions, filesystem
+
+from ... import settings
+from ..base import BaseConfig
+from .adapter import AdapterConfig as PipelineAdapterConfig
+from .run import ExecutorConfig, RunConfig, WithAdapterConfig
+from ..project.adapter import AdapterConfig as ProjectAdapterConfig
+
+
+class RunConfigBuilder:
+    """A fluent builder for creating RunConfig objects.
+    
+    This builder provides a clean interface for constructing RunConfig objects
+    with proper configuration merging from project and pipeline defaults.
+    """
+    
+    def __init__(
+        self, 
+        pipeline_name: str, 
+        base_dir: str | None = None,
+        fs: AbstractFileSystem | None = None,
+        storage_options: dict | BaseStorageOptions | None = {}
+    ):
+        """Initialize the RunConfigBuilder.
+        
+        Args:
+            pipeline_name: Name of the pipeline to build config for
+            base_dir: Base directory for the project (defaults to current directory)
+            fs: Optional filesystem instance
+            storage_options: Options for filesystem access
+        """
+        self.pipeline_name = pipeline_name
+        self.base_dir = base_dir or "."
+        self._fs = fs
+        self._storage_options = storage_options
+        
+        # Initialize with empty config
+        self._config = RunConfig()
+        
+        # Load defaults from pipeline and project configs
+        self._load_defaults()
+    
+    def _load_defaults(self):
+        """Load default configuration from pipeline and project YAML files."""
+        if self._fs is None:
+            self._fs = filesystem(
+                self.base_dir, 
+                cached=False, 
+                dirfs=True, 
+                storage_options=self._storage_options
+            )
+        
+        # Load pipeline configuration
+        try:
+            from .. import PipelineConfig
+            pipeline_cfg = PipelineConfig.load(
+                base_dir=self.base_dir,
+                name=self.pipeline_name,
+                fs=self._fs,
+                storage_options=self._storage_options
+            )
+            if pipeline_cfg and pipeline_cfg.run:
+                self._config = copy.deepcopy(pipeline_cfg.run)
+        except Exception:
+            # If pipeline config doesn't exist, use defaults
+            pass
+        
+        # Load project configuration for adapter defaults
+        try:
+            from .. import ProjectConfig
+            project_cfg = ProjectConfig.load(
+                base_dir=self.base_dir,
+                fs=self._fs,
+                storage_options=self._storage_options
+            )
+            if project_cfg and project_cfg.adapter:
+                # Store project adapter config for merging
+                self._project_adapter_cfg = project_cfg.adapter
+            else:
+                self._project_adapter_cfg = ProjectAdapterConfig()
+        except Exception:
+            self._project_adapter_cfg = ProjectAdapterConfig()
+    
+    def with_inputs(self, inputs: dict) -> "RunConfigBuilder":
+        """Set pipeline input values.
+        
+        Args:
+            inputs: Dictionary of input values to override defaults
+            
+        Returns:
+            Self for method chaining
+        """
+        if inputs:
+            if self._config.inputs is None:
+                self._config.inputs = {}
+            self._config.inputs.update(inputs)
+        return self
+    
+    def with_final_vars(self, final_vars: list[str]) -> "RunConfigBuilder":
+        """Set the final output variables.
+        
+        Args:
+            final_vars: List of variable names to return from execution
+            
+        Returns:
+            Self for method chaining
+        """
+        self._config.final_vars = final_vars
+        return self
+    
+    def with_config(self, config: dict) -> "RunConfigBuilder":
+        """Set Hamilton driver configuration.
+        
+        Args:
+            config: Dictionary of configuration values for Hamilton
+            
+        Returns:
+            Self for method chaining
+        """
+        if config:
+            if self._config.config is None:
+                self._config.config = {}
+            self._config.config.update(config)
+        return self
+    
+    def with_cache(self, cache: Union[dict, bool]) -> "RunConfigBuilder":
+        """Set cache configuration.
+        
+        Args:
+            cache: Cache configuration (dict) or enable/disable flag (bool)
+            
+        Returns:
+            Self for method chaining
+        """
+        self._config.cache = cache
+        return self
+    
+    def with_executor(self, executor_type: str, **kwargs) -> "RunConfigBuilder":
+        """Set executor configuration.
+        
+        Args:
+            executor_type: Type of executor ('synchronous', 'threadpool', 'processpool', 'ray', 'dask')
+            **kwargs: Additional executor configuration options
+            
+        Returns:
+            Self for method chaining
+        """
+        if not self._config.executor:
+            self._config.executor = ExecutorConfig()
+        
+        self._config.executor.type = executor_type
+        
+        # Apply additional executor options
+        for key, value in kwargs.items():
+            if hasattr(self._config.executor, key):
+                setattr(self._config.executor, key, value)
+        
+        return self
+    
+    def with_adapter(self, adapter_name: str, **kwargs) -> "RunConfigBuilder":
+        """Enable and configure a specific adapter.
+        
+        Args:
+            adapter_name: Name of the adapter ('hamilton_tracker', 'mlflow', 'opentelemetry', etc.)
+            **kwargs: Adapter-specific configuration options
+            
+        Returns:
+            Self for method chaining
+        """
+        if not self._config.with_adapter:
+            self._config.with_adapter = WithAdapterConfig()
+        
+        # Enable the adapter
+        if hasattr(self._config.with_adapter, adapter_name):
+            setattr(self._config.with_adapter, adapter_name, True)
+            
+            # Store adapter configuration for merging
+            if not hasattr(self, '_adapter_configs'):
+                self._adapter_configs = {}
+            self._adapter_configs[adapter_name] = kwargs
+        
+        return self
+    
+    def with_retries(
+        self, 
+        max_attempts: int = 3, 
+        delay: float = 1.0, 
+        jitter: float = 0.1,
+        exceptions: Optional[list] = None
+    ) -> "RunConfigBuilder":
+        """Configure retry behavior.
+        
+        Args:
+            max_attempts: Maximum number of retry attempts
+            delay: Base delay between retries in seconds
+            jitter: Random jitter factor to add to retry delay
+            exceptions: List of exception types that should trigger retries
+            
+        Returns:
+            Self for method chaining
+        """
+        self._config.max_retries = max_attempts
+        self._config.retry_delay = delay
+        self._config.jitter_factor = jitter
+        
+        if exceptions:
+            self._config.retry_exceptions = exceptions
+        
+        return self
+    
+    def with_callbacks(
+        self, 
+        on_success: Optional[Callable] = None, 
+        on_failure: Optional[Callable] = None
+    ) -> "RunConfigBuilder":
+        """Set success and failure callbacks.
+        
+        Args:
+            on_success: Callback function to execute on successful completion
+            on_failure: Callback function to execute on failure
+            
+        Returns:
+            Self for method chaining
+        """
+        if on_success:
+            self._config.on_success = on_success
+        if on_failure:
+            self._config.on_failure = on_failure
+        
+        return self
+    
+    def with_log_level(self, log_level: str) -> "RunConfigBuilder":
+        """Set the log level for execution.
+        
+        Args:
+            log_level: Log level ('DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL')
+            
+        Returns:
+            Self for method chaining
+        """
+        self._config.log_level = log_level
+        return self
+    
+    def with_reload(self, reload: bool = True) -> "RunConfigBuilder":
+        """Set whether to reload the pipeline module.
+        
+        Args:
+            reload: Whether to force reload of the pipeline module
+            
+        Returns:
+            Self for method chaining
+        """
+        self._config.reload = reload
+        return self
+    
+    def with_pipeline_adapter_config(self, config: dict) -> "RunConfigBuilder":
+        """Set pipeline-specific adapter configuration.
+        
+        Args:
+            config: Pipeline adapter configuration dictionary
+            
+        Returns:
+            Self for method chaining
+        """
+        if config:
+            if self._config.pipeline_adapter_cfg is None:
+                self._config.pipeline_adapter_cfg = {}
+            self._config.pipeline_adapter_cfg.update(config)
+        return self
+    
+    def with_project_adapter_config(self, config: dict) -> "RunConfigBuilder":
+        """Set project-level adapter configuration.
+        
+        Args:
+            config: Project adapter configuration dictionary
+            
+        Returns:
+            Self for method chaining
+        """
+        if config:
+            if self._config.project_adapter_cfg is None:
+                self._config.project_adapter_cfg = {}
+            self._config.project_adapter_cfg.update(config)
+        return self
+    
+    def with_custom_adapter(self, name: str, adapter: Any) -> "RunConfigBuilder":
+        """Add a custom adapter instance.
+        
+        Args:
+            name: Name/identifier for the adapter
+            adapter: Adapter instance
+            
+        Returns:
+            Self for method chaining
+        """
+        if self._config.adapter is None:
+            self._config.adapter = {}
+        self._config.adapter[name] = adapter
+        return self
+    
+    def build(self) -> RunConfig:
+        """Build the final RunConfig object.
+        
+        This method merges all configurations and validates the final result.
+        
+        Returns:
+            Fully configured RunConfig object
+            
+        Raises:
+            ValueError: If configuration is invalid
+        """
+        # Create a deep copy to avoid modifying the internal state
+        final_config = copy.deepcopy(self._config)
+        
+        # Merge adapter configurations
+        if hasattr(self, '_adapter_configs') and self._adapter_configs:
+            self._merge_adapter_configs(final_config)
+        
+        # Validate configuration
+        self._validate_config(final_config)
+        
+        return final_config
+    
+    def _merge_adapter_configs(self, config: RunConfig):
+        """Merge adapter configurations from builder with project/pipeline configs."""
+        if not config.pipeline_adapter_cfg:
+            config.pipeline_adapter_cfg = {}
+        
+        if not config.project_adapter_cfg:
+            config.project_adapter_cfg = {}
+        
+        # Merge project adapter defaults
+        for adapter_name, adapter_config in self._adapter_configs.items():
+            if adapter_name in ['hamilton_tracker', 'mlflow', 'opentelemetry']:
+                # Merge with project config
+                if hasattr(self._project_adapter_cfg, adapter_name):
+                    project_config = getattr(self._project_adapter_cfg, adapter_name).to_dict()
+                    adapter_config = {**project_config, **adapter_config}
+                
+                # Store in pipeline adapter config
+                if adapter_name not in config.pipeline_adapter_cfg:
+                    config.pipeline_adapter_cfg[adapter_name] = {}
+                config.pipeline_adapter_cfg[adapter_name].update(adapter_config)
+    
+    def _validate_config(self, config: RunConfig):
+        """Validate the final configuration.
+        
+        Args:
+            config: RunConfig object to validate
+            
+        Raises:
+            ValueError: If configuration is invalid
+        """
+        # Validate retry configuration
+        if config.max_retries < 0:
+            raise ValueError("max_retries must be non-negative")
+        
+        if config.retry_delay < 0:
+            raise ValueError("retry_delay must be non-negative")
+        
+        if config.jitter_factor is not None and config.jitter_factor < 0:
+            raise ValueError("jitter_factor must be non-negative")
+        
+        # Validate executor configuration
+        if config.executor and config.executor.type:
+            valid_executors = ['synchronous', 'threadpool', 'processpool', 'ray', 'dask']
+            if config.executor.type not in valid_executors:
+                raise ValueError(f"Invalid executor type: {config.executor.type}")
+        
+        # Validate log level
+        if config.log_level:
+            valid_levels = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']
+            if config.log_level.upper() not in valid_levels:
+                raise ValueError(f"Invalid log level: {config.log_level}")