FlowerPower 0.20.0__py3-none-any.whl → 0.30.0__py3-none-any.whl

flowerpower/cfg/pipeline/run.py

@@ -1,5 +1,6 @@
 import msgspec
 from munch import munchify
+from typing import Any, Callable
 
 from ... import settings
 from ..base import BaseConfig
@@ -33,6 +34,39 @@ class RunConfig(BaseConfig):
     retry_delay: int | float = msgspec.field(default=1)
     jitter_factor: float | None = msgspec.field(default=0.1)
     retry_exceptions: list[str] = msgspec.field(default_factory=lambda: ["Exception"])
+    # New fields for comprehensive configuration
+    pipeline_adapter_cfg: dict | None = msgspec.field(default=None)
+    project_adapter_cfg: dict | None = msgspec.field(default=None)
+    adapter: dict[str, Any] | None = msgspec.field(default=None)
+    reload: bool = msgspec.field(default=False)
+
+
+class CallbackSpec(msgspec.Struct):
+    """Specification for a callback function with optional arguments."""
+    func: Callable
+    args: tuple | None = None
+    kwargs: dict | None = None
+
+
+class RunConfig(BaseConfig):
+    inputs: dict | None = msgspec.field(default_factory=dict)
+    final_vars: list[str] | None = msgspec.field(default_factory=list)
+    config: dict | None = msgspec.field(default_factory=dict)
+    cache: dict | bool | None = msgspec.field(default=False)
+    with_adapter: WithAdapterConfig = msgspec.field(default_factory=WithAdapterConfig)
+    executor: ExecutorConfig = msgspec.field(default_factory=ExecutorConfig)
+    log_level: str | None = msgspec.field(default="INFO")
+    max_retries: int = msgspec.field(default=3)
+    retry_delay: int | float = msgspec.field(default=1)
+    jitter_factor: float | None = msgspec.field(default=0.1)
+    retry_exceptions: list[str] = msgspec.field(default_factory=lambda: ["Exception"])
+    # New fields for comprehensive configuration
+    pipeline_adapter_cfg: dict | None = msgspec.field(default=None)
+    project_adapter_cfg: dict | None = msgspec.field(default=None)
+    adapter: dict[str, Any] | None = msgspec.field(default=None)
+    reload: bool = msgspec.field(default=False)
+    on_success: CallbackSpec | None = msgspec.field(default=None)
+    on_failure: CallbackSpec | None = msgspec.field(default=None)
 
     def __post_init__(self):
         if isinstance(self.inputs, dict):
@@ -45,3 +79,58 @@ class RunConfig(BaseConfig):
             self.with_adapter = WithAdapterConfig.from_dict(self.with_adapter)
         if isinstance(self.executor, dict):
             self.executor = ExecutorConfig.from_dict(self.executor)
+        if isinstance(self.pipeline_adapter_cfg, dict):
+            from ..pipeline.adapter import AdapterConfig as PipelineAdapterConfig
+            self.pipeline_adapter_cfg = PipelineAdapterConfig.from_dict(self.pipeline_adapter_cfg)
+        if isinstance(self.project_adapter_cfg, dict):
+            from ..project.adapter import AdapterConfig as ProjectAdapterConfig
+            self.project_adapter_cfg = ProjectAdapterConfig.from_dict(self.project_adapter_cfg)
+        if isinstance(self.adapter, dict):
+            # Convert adapter instances if needed
+            pass
+        if isinstance(self.retry_exceptions, list):
+            # Convert string exceptions to actual exception classes
+            converted_exceptions = []
+            for exc in self.retry_exceptions:
+                if isinstance(exc, str):
+                    try:
+                        exc_class = eval(exc)
+                        # Ensure it's actually an exception class
+                        if isinstance(exc_class, type) and issubclass(exc_class, BaseException):
+                            converted_exceptions.append(exc_class)
+                        else:
+                            converted_exceptions.append(Exception)
+                    except (NameError, AttributeError):
+                        converted_exceptions.append(Exception)
+                elif isinstance(exc, type) and issubclass(exc, BaseException):
+                    converted_exceptions.append(exc)
+                else:
+                    converted_exceptions.append(Exception)
+            self.retry_exceptions = converted_exceptions
+
+        # Handle callback conversions
+        if self.on_success is not None and not isinstance(self.on_success, CallbackSpec):
+            if callable(self.on_success):
+                self.on_success = CallbackSpec(func=self.on_success)
+            elif isinstance(self.on_success, tuple) and len(self.on_success) == 3:
+                func, args, kwargs = self.on_success
+                self.on_success = CallbackSpec(func=func, args=args, kwargs=kwargs)
+            else:
+                self.on_success = None
+                warnings.warn(
+                    "Invalid on_success format, must be Callable or (Callable, args, kwargs)",
+                    RuntimeWarning
+                )
+
+        if self.on_failure is not None and not isinstance(self.on_failure, CallbackSpec):
+            if callable(self.on_failure):
+                self.on_failure = CallbackSpec(func=self.on_failure)
+            elif isinstance(self.on_failure, tuple) and len(self.on_failure) == 3:
+                func, args, kwargs = self.on_failure
+                self.on_failure = CallbackSpec(func=func, args=args, kwargs=kwargs)
+            else:
+                self.on_failure = None
+                warnings.warn(
+                    "Invalid on_failure format, must be Callable or (Callable, args, kwargs)",
+                    RuntimeWarning
+                )

flowerpower/cfg/project/__init__.py

@@ -1,20 +1,20 @@
 import msgspec
 from fsspec_utils import AbstractFileSystem, BaseStorageOptions, filesystem
+import posixpath
 
+from ...settings import CONFIG_DIR
 from ..base import BaseConfig
 from .adapter import AdapterConfig
-from .job_queue import JobQueueConfig
 
 
 class ProjectConfig(BaseConfig):
     """A configuration class for managing project-level settings in FlowerPower.
 
-    This class handles project-wide configuration including job queue and adapter settings.
+    This class handles project-wide configuration including adapter settings.
     It supports loading from and saving to YAML files, with filesystem abstraction.
 
     Attributes:
         name (str | None): The name of the project.
-        job_queue (JobQueueConfig): Configuration for the job queue component.
         adapter (AdapterConfig): Configuration for the adapter component.
 
     Example:
@@ -31,12 +31,9 @@ class ProjectConfig(BaseConfig):
     """
 
     name: str | None = msgspec.field(default=None)
-    job_queue: JobQueueConfig = msgspec.field(default_factory=JobQueueConfig)
     adapter: AdapterConfig = msgspec.field(default_factory=AdapterConfig)
 
     def __post_init__(self):
-        if isinstance(self.job_queue, dict):
-            self.job_queue = JobQueueConfig.from_dict(self.job_queue)
         if isinstance(self.adapter, dict):
             self.adapter = AdapterConfig.from_dict(self.adapter)
 
@@ -45,7 +42,6 @@ class ProjectConfig(BaseConfig):
         cls,
         base_dir: str = ".",
         name: str | None = None,
-        job_queue_type: str | None = None,
         fs: AbstractFileSystem | None = None,
         storage_options: dict | BaseStorageOptions | None = {},
     ):
@@ -54,7 +50,6 @@ class ProjectConfig(BaseConfig):
         Args:
             base_dir (str, optional): Base directory for the project. Defaults to ".".
             name (str | None, optional): Project name. Defaults to None.
-            job_queue_type (str | None, optional): Type of job queue to use. Defaults to None.
             fs (AbstractFileSystem | None, optional): Filesystem to use. Defaults to None.
             storage_options (dict | Munch, optional): Options for filesystem. Defaults to empty Munch.
 
@@ -65,8 +60,7 @@ class ProjectConfig(BaseConfig):
             ```python
             project = ProjectConfig.load(
                 base_dir="my_project",
-                name="pipeline1",
-                job_queue_type="rq"
+                name="pipeline1"
                 )
             ```
         """
@@ -78,9 +72,6 @@ class ProjectConfig(BaseConfig):
             project = ProjectConfig.from_yaml(path="conf/project.yml", fs=fs)
         else:
             project = ProjectConfig(name=name)
-        if job_queue_type is not None:
-            if job_queue_type != project.job_queue.type:
-                project.job_queue.update_type(job_queue_type)
 
         return project
 
@@ -107,14 +98,13 @@ class ProjectConfig(BaseConfig):
                 base_dir, cached=True, dirfs=True, storage_options=storage_options
             )
 
-        fs.makedirs("conf", exist_ok=True)
-        self.to_yaml(path="conf/project.yml", fs=fs)
+        fs.makedirs(CONFIG_DIR, exist_ok=True)
+        self.to_yaml(path=posixpath.join(CONFIG_DIR, "project.yml"), fs=fs)
 
 
 def init_project_config(
     base_dir: str = ".",
     name: str | None = None,
-    job_queue_type: str | None = None,
     fs: AbstractFileSystem | None = None,
     storage_options: dict | BaseStorageOptions | None = {},
 ):
@@ -125,7 +115,6 @@ def init_project_config(
     Args:
         base_dir (str, optional): Base directory for the project. Defaults to ".".
         name (str | None, optional): Project name. Defaults to None.
-        job_queue_type (str | None, optional): Type of job queue to use. Defaults to None.
         fs (AbstractFileSystem | None, optional): Filesystem to use. Defaults to None.
         storage_options (dict | Munch, optional): Options for filesystem. Defaults to empty Munch.
 
@@ -136,17 +125,15 @@ def init_project_config(
         ```python
         project = init_project_config(
             base_dir="my_project",
-            name="test_project",
-            job_queue_type="rq"
+            name="test_project"
         )
         ```
     """
     project = ProjectConfig.load(
         base_dir=base_dir,
         name=name,
-        job_queue_type=job_queue_type,
         fs=fs,
         storage_options=storage_options,
     )
     project.save(base_dir=base_dir, fs=fs, storage_options=storage_options)
-    return project
+    return project

flowerpower/cfg/project/adapter.py

@@ -24,18 +24,6 @@ class OpenTelemetryConfig(BaseConfig):
     port: int = msgspec.field(default=6831)
 
 
-# class OpenLineageConfig(BaseConfig):
-#     from openlineage.client import OpenLineageClientOptions
-#     from openlineage.client.transport import Transport
-#     from openlineage.client.transport import TransportFactory
-#     url: str | None = msgspec.field(default=None)
-#     options: OpenLineageClientOptions | None = msgspec.field(
-#         default=None)
-#     transport: Transport | None = msgspec.field(default=None)
-#     factory: TransportFactory | None = msgspec.field(
-#         default=None)
-#     config: dict | None = msgspec.field(default=None)
-
 
 class RayConfig(BaseConfig):
     ray_init_config: dict | None = msgspec.field(default=None)

flowerpower/cli/__init__.py

@@ -4,7 +4,7 @@ import os
 import typer
 from loguru import logger
 
-from ..flowerpower import init as init_
+from ..flowerpower import FlowerPowerProject
 from .pipeline import app as pipeline_app
 from .utils import parse_dict_or_list_param
 
@@ -18,21 +18,6 @@ app.add_typer(
     pipeline_app, name="pipeline", help="Manage and execute FlowerPower pipelines"
 )
 
-if importlib.util.find_spec("rq"):
-    from .job_queue import app as job_queue_app
-
-    app.add_typer(
-        job_queue_app,
-        name="job-queue",
-        help="Manage job queue workers and scheduled tasks",
-    )
-
-if importlib.util.find_spec("paho"):
-    from .mqtt import app as mqtt_app
-
-    app.add_typer(
-        mqtt_app, name="mqtt", help="Connect pipelines to MQTT message brokers"
-    )
 
 
 @app.command()
@@ -49,12 +34,6 @@ def init(
     storage_options: str = typer.Option(
         None, "--storage-options", "-s", help="Storage options as a JSON or dict string"
     ),
-    job_queue_type: str = typer.Option(
-        "rq",
-        "--job-queue-type",
-        "-q",
-        help="Job queue backend type to use (rq)",
-    ),
 ):
     """
     Initialize a new FlowerPower project.
@@ -69,7 +48,6 @@ def init(
         base_dir: Base directory where the project will be created. If not provided,
                   the current directory's parent will be used
         storage_options: Storage options for filesystem access, as a JSON or dict string
-        job_queue_type: Type of job queue backend to use (rq)
 
     Examples:
         # Create a project in the current directory using its name
@@ -80,9 +58,6 @@ def init(
 
         # Create a project in a specific location
         $ flowerpower init --name my-project --base-dir /path/to/projects
-
-        # Create a project with RQ as the job queue backend (default)
-        $ flowerpower init --job-queue-type rq
     """
     parsed_storage_options = {}
     if storage_options:
@@ -95,11 +70,10 @@ def init(
             raise typer.Exit(code=1)
 
     try:
-        init_(
+        FlowerPowerProject.new(
             name=project_name,
             base_dir=base_dir,
             storage_options=parsed_storage_options,
-            job_queue_type=job_queue_type,
         )
     except Exception as e:
         logger.error(f"Error initializing project: {e}")

flowerpower/cli/pipeline.py

@@ -5,6 +5,7 @@ from typing_extensions import Annotated
 
 from ..flowerpower import FlowerPowerProject
 from ..pipeline.manager import HookType, PipelineManager
+from ..cfg.pipeline.run import RunConfig
 from ..utils.logging import setup_logging
 from .utils import parse_dict_or_list_param
 
@@ -114,18 +115,23 @@ def run(
         raise typer.Exit(1)
 
     try:
-        _ = project.run(
-            name=name,
+        # Construct RunConfig object from parsed CLI arguments
+        run_config = RunConfig(
             inputs=parsed_inputs,
             final_vars=parsed_final_vars,
             config=parsed_config,
             cache=parsed_cache,
-            executor_cfg=executor,
-            with_adapter_cfg=parsed_with_adapter,
+            with_adapter=parsed_with_adapter,
             max_retries=max_retries,
             retry_delay=retry_delay,
             jitter_factor=jitter_factor,
         )
+        
+        # Handle executor configuration
+        if executor is not None:
+            run_config.executor.type = executor
+
+        _ = project.run(name=name, run_config=run_config)
         logger.info(f"Pipeline '{name}' finished running.")
     except Exception as e:
         logger.error(f"Pipeline execution failed: {e}")