FlowerPower 0.11.6.19__py3-none-any.whl → 0.20.0__py3-none-any.whl

flowerpower/pipeline/manager.py

@@ -2,12 +2,12 @@ import datetime as dt
 import os
 import posixpath
 import sys
+import warnings
 from pathlib import Path
 from types import TracebackType
 from typing import Any, Callable, TypeVar, Union
 from uuid import UUID
 
-import duration_parser
 from loguru import logger
 from munch import Munch
 
@@ -16,18 +16,16 @@ try:
 except ImportError:
     Digraph = Any  # Type alias for when graphviz isn't installed
 
+from fsspec_utils import AbstractFileSystem, BaseStorageOptions, filesystem
+
 from .. import settings
 from ..cfg import PipelineConfig, ProjectConfig
 from ..cfg.pipeline.adapter import AdapterConfig as PipelineAdapterConfig
 from ..cfg.pipeline.run import ExecutorConfig, WithAdapterConfig
 from ..cfg.project.adapter import AdapterConfig as ProjectAdapterConfig
-from ..fs import AbstractFileSystem, BaseStorageOptions, get_filesystem
-from ..utils.callback import run_with_callback
 from ..utils.logging import setup_logging
 from .io import PipelineIOManager
-from .job_queue import PipelineJobQueue
 from .registry import HookType, PipelineRegistry
-from .runner import run_pipeline
 from .visualizer import PipelineVisualizer
 
 setup_logging(level=settings.LOG_LEVEL)
@@ -98,7 +96,7 @@ class PipelineManager:
             pipelines_dir: Override default pipelines directory name ('pipelines').
                 Example: "flows" or "dags".
             job_queue_type: Override worker type from project config/settings.
-                Valid values: "rq", "apscheduler", or "huey".
+                Valid values: "rq".
             log_level: Set logging level for the manager.
                 Valid values: "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"
 
@@ -138,7 +136,7 @@ class PipelineManager:
             cached = False
             cache_storage = None
         if not fs:
-            fs = get_filesystem(
+            fs = filesystem(
                 self._base_dir,
                 storage_options=storage_options,
                 cached=cached,
@@ -175,22 +173,12 @@ class PipelineManager:
         self.registry = PipelineRegistry(
             project_cfg=self.project_cfg,
             fs=self._fs,
-            cfg_dir=self._cfg_dir,
-            pipelines_dir=self._pipelines_dir,
-        )
-        pipeline_job_queue = PipelineJobQueue(
-            project_cfg=self.project_cfg,
-            fs=self._fs,
-            cfg_dir=self._cfg_dir,
-            pipelines_dir=self._pipelines_dir,
+            base_dir=self._base_dir,
+            storage_options=self._storage_options,
         )
-        if pipeline_job_queue.job_queue is None:
-            logger.warning(
-                "Job queue backend is unavailable. Some features may not work."
-            )
-            self.jqm = None
-        else:
-            self.jqm = pipeline_job_queue
+
+        # Initialize project context (will be injected by FlowerPowerProject)
+        self._project_context = None
         self.visualizer = PipelineVisualizer(project_cfg=self.project_cfg, fs=self._fs)
         self.io = PipelineIOManager(registry=self.registry)
 
@@ -241,49 +229,6 @@ class PipelineManager:
         # Add cleanup code if needed
         pass
 
-    def _get_run_func(
-        self,
-        name: str,
-        reload: bool = False,
-        on_success: Callable | tuple[Callable, tuple | None, dict | None] | None = None,
-        on_failure: Callable | tuple[Callable, tuple | None, dict | None] | None = None,
-    ) -> Callable:
-        """Create a PipelineRunner instance and return its run method.
-
-        This internal helper method ensures that each job gets a fresh runner
-        with the correct configuration state.
-
-        Args:
-            name: Name of the pipeline to create runner for
-            reload: Whether to reload pipeline configuration
-
-        Returns:
-            Callable: Bound run method from a fresh PipelineRunner instance
-
-        Example:
-            >>> # Internal usage
-            >>> manager = PipelineManager()
-            >>> run_func = manager._get_run_func_for_job("data_pipeline")
-            >>> result = run_func(inputs={"date": "2025-04-28"})
-        """
-        if (
-            name == self._current_pipeline_name and not reload
-            # and hasattr(self, "_runner")
-        ):
-            # run_pipeline_ = partial(run_pipeline, project_cfg=self.project_cfg, pipeline_cfg=self._pipeline_cfg)
-            run_func = run_with_callback(on_success=on_success, on_failure=on_failure)(
-                run_pipeline
-            )
-            return run_func
-
-        _ = self.load_pipeline(name=name, reload=reload)
-        # run_pipeline_ = partial(run_pipeline, project_cfg=self.project_cfg, pipeline_cfg=pipeline_cfg)
-
-        run_func = run_with_callback(on_success=on_success, on_failure=on_failure)(
-            run_pipeline
-        )
-        return run_func
-
     def _add_modules_path(self) -> None:
         """Add pipeline module paths to Python path.
 
@@ -543,14 +488,16 @@ class PipelineManager:
             ...     reload=True
             ... )
         """
-        # pipeline_cfg = self._load_pipeline_cfg(name=name, reload=reload)
-        run_func = self._get_run_func(
-            name=name, reload=reload, on_success=on_success, on_failure=on_failure
+        # Use injected project context, fallback to self for backward compatibility
+        project_context = getattr(self, "_project_context", self)
+
+        # Get Pipeline instance from registry
+        pipeline = self.registry.get_pipeline(
+            name=name, project_context=project_context, reload=reload
         )
 
-        res = run_func(
-            project_cfg=self._project_cfg,
-            pipeline_cfg=self._pipeline_cfg,
+        # Execute pipeline using its own run method
+        return pipeline.run(
             inputs=inputs,
             final_vars=final_vars,
             config=config,
@@ -560,16 +507,16 @@ class PipelineManager:
             pipeline_adapter_cfg=pipeline_adapter_cfg,
             project_adapter_cfg=project_adapter_cfg,
             adapter=adapter,
-            # reload=reload,  # Runner handles module reload if needed
+            reload=reload,
             log_level=log_level,
             max_retries=max_retries,
             retry_delay=retry_delay,
             jitter_factor=jitter_factor,
             retry_exceptions=retry_exceptions,
+            on_success=on_success,
+            on_failure=on_failure,
         )
 
-        return res
-
     # --- Delegated Methods ---
 
     # Registry Delegations
@@ -1213,571 +1160,3 @@ class PipelineManager:
         return self.visualizer.show_dag(
             name=name, format=format, reload=reload, raw=raw
         )
-
-    def run_job(
-        self,
-        name: str,
-        inputs: dict | None = None,
-        final_vars: list[str] | None = None,
-        config: dict | None = None,
-        cache: bool | dict = False,
-        executor_cfg: str | dict | ExecutorConfig | None = None,
-        with_adapter_cfg: dict | WithAdapterConfig | None = None,
-        pipeline_adapter_cfg: dict | PipelineAdapterConfig | None = None,
-        project_adapter_cfg: dict | ProjectAdapterConfig | None = None,
-        adapter: dict[str, Any] | None = None,
-        reload: bool = False,
-        log_level: str | None = None,
-        max_retries: int | None = None,
-        retry_delay: float | None = None,
-        jitter_factor: float | None = None,
-        retry_exceptions: tuple | list | None = None,
-        on_success: Callable | tuple[Callable, tuple | None, dict | None] | None = None,
-        on_failure: Callable | tuple[Callable, tuple | None, dict | None] | None = None,
-        on_success_pipeline: Callable
-        | tuple[Callable, tuple | None, dict | None]
-        | None = None,
-        on_failure_pipeline: Callable
-        | tuple[Callable, tuple | None, dict | None]
-        | None = None,
-        **kwargs: Any,
-    ) -> dict[str, Any] | None:
-        """Execute a pipeline job immediately through the job queue.
-
-        Unlike the run() method which executes synchronously, this method runs
-        the pipeline through the configured worker system (RQ, APScheduler, etc.).
-
-        If the job queue is not configured, it logs an error and returns None.
-
-        Args:
-            name (str): Name of the pipeline to run. Must be a valid identifier.
-            inputs (dict | None): Override pipeline input values. Example: {"data_date": "2025-04-28"}
-            final_vars (list[str] | None): Specify which output variables to return.
-                Example: ["model", "metrics"]
-            config (dict | None): Configuration for Hamilton pipeline executor.
-                Example: {"model": "LogisticRegression"}
-            cache (dict | None): Cache configuration for results. Example: {"recompute": ["node1", "final_node"]}
-            executor_cfg (str | dict | ExecutorConfig | None): Execution configuration, can be:
-                - str: Executor name, e.g. "threadpool", "local"
-                - dict: Raw config, e.g. {"type": "threadpool", "max_workers": 4}
-                - ExecutorConfig: Structured config object
-            with_adapter_cfg (dict | WithAdapterConfig | None): Adapter settings for pipeline execution.
-                Example: {"opentelemetry": True, "tracker": False}
-             pipeline_adapter_cfg (dict | PipelineAdapterConfig | None): Pipeline-specific adapter settings.
-                Example: {"tracker": {"project_id": "123", "tags": {"env": "prod"}}}
-            project_adapter_cfg (dict | ProjectAdapterConfig | None): Project-level adapter settings.
-                Example: {"opentelemetry": {"host": "http://localhost:4317"}}
-            adapter (dict[str, Any] | None): Custom adapter instance for pipeline
-                Example: {"ray_graph_adapter": RayGraphAdapter()}
-            reload (bool): Force reload of pipeline configuration.
-            log_level (str | None): Logging level for the execution. Default None uses project config.
-                Valid values: "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"
-            max_retries (int): Maximum number of retries for execution.
-            retry_delay (float): Delay between retries in seconds.
-            jitter_factor (float): Random jitter factor to add to retry delay
-            retry_exceptions (tuple): Exceptions that trigger a retry.
-            on_success (Callable | tuple[Callable, tuple | None, dict | None] | None): Callback to run on successful job execution.
-                This runs after the pipeline execution through the job queue was executed successfully.
-            on_failure (Callable | tuple[Callable, tuple | None, dict | None] | None): Callback to run on job execution failure.
-                This runs if the job creation or the pipeline execution through the job queue fails or raises an exception.
-            on_success_pipeline (Callable | tuple[Callable, tuple | None, dict | None] | None): Callback to run on successful pipeline execution.
-                This runs after the pipeline completes successfully.
-            on_failure_pipeline (Callable | tuple[Callable, tuple | None, dict | None] | None): Callback to run on pipeline execution failure.
-                This runs if the pipeline fails or raises an exception.
-
-            **kwargs: JobQueue-specific arguments
-                For RQ:
-                    - queue_name: Queue to use (str)
-                    - retry: Number of retries (int)
-                    - result_ttl: Time to live for the job result (float or timedelta)
-                    - ttl: Time to live for the job (float or timedelta)
-                    - timeout: Time to wait for the job to complete (float or timedelta)
-                    - repeat: Repeat count (int or dict)
-                    - rq_on_failure: Callback function on failure (callable)
-                    - rq_on_success: Callback function on success (callable)
-                    - rq_on_stopped: Callback function on stop (callable)
-                For APScheduler:
-                    - job_executor: Executor type (str)
-
-        Returns:
-            dict[str, Any] | None: Job execution results if successful, otherwise None.
-
-        Raises:
-            ValueError: If pipeline or configuration is invalid
-            RuntimeError: If job execution fails
-
-        Example:
-            >>> from flowerpower.pipeline import PipelineManager
-            >>>
-            >>> manager = PipelineManager()
-            >>>
-            >>> # Simple job execution
-            >>> result = manager.run_job("data_pipeline")
-            >>>
-            >>> # Complex job with retry logic
-            >>> result = manager.run_job(
-            ...     name="ml_training",
-            ...     inputs={"training_date": "2025-04-28"},
-            ...     executor_cfg={"type": "async"},
-            ...     with_adapter_cfg={"enable_tracking": True},
-            ...     retry=3,
-            ...     queue_name="ml_jobs"
-            ... )
-        """
-        if self.jqm is None:
-            logger.error(
-                "This PipelineManager instance does not have a job queue configured. Skipping job execution."
-            )
-            return None
-
-        kwargs["on_success"] = kwargs.get("rq_on_success", None)
-        kwargs["on_failure"] = kwargs.get("rq_on_failure", None)
-        kwargs["on_stopped"] = kwargs.get("rq_on_stopped", None)
-
-        run_func = self._get_run_func(
-            name=name,
-            reload=reload,
-            on_success=on_success_pipeline,
-            on_failure=on_failure_pipeline,
-        )
-        # run_func = run_with_callback(on_success=on_success_pipeline, on_failure=on_failure_pipeline)(
-        #    run_func_
-        # )
-        run_job = run_with_callback(on_success=on_success, on_failure=on_failure)(
-            self.jqm.run_job
-        )
-
-        return run_job(
-            run_func=run_func,
-            pipeline_cfg=self._pipeline_cfg,
-            name=name,
-            inputs=inputs,
-            final_vars=final_vars,
-            config=config,
-            cache=cache,
-            executor_cfg=executor_cfg,
-            with_adapter_cfg=with_adapter_cfg,
-            pipeline_adapter_cfg=pipeline_adapter_cfg,
-            project_adapter_cfg=project_adapter_cfg,
-            adapter=adapter,
-            log_level=log_level,
-            max_retries=max_retries,
-            retry_delay=retry_delay,
-            jitter_factor=jitter_factor,
-            retry_exceptions=retry_exceptions,
-            **kwargs,
-        )
-
-    def add_job(
-        self,
-        name: str,
-        inputs: dict | None = None,
-        final_vars: list[str] | None = None,
-        config: dict | None = None,
-        cache: bool | dict = False,
-        executor_cfg: str | dict | ExecutorConfig | None = None,
-        with_adapter_cfg: dict | WithAdapterConfig | None = None,
-        pipeline_adapter_cfg: dict | PipelineAdapterConfig | None = None,
-        project_adapter_cfg: dict | ProjectAdapterConfig | None = None,
-        adapter: dict[str, Any] | None = None,
-        reload: bool = False,  # Reload config/module before creating run_func
-        log_level: str | None = None,
-        result_ttl: int | dt.timedelta = 0,
-        run_at: dt.datetime | str | None = None,
-        run_in: dt.datetime | str | None = None,
-        max_retries: int = 3,
-        retry_delay: float = 1.0,
-        jitter_factor: float = 0.1,
-        retry_exceptions: tuple = (Exception,),
-        on_success: Callable | tuple[Callable, tuple | None, dict | None] | None = None,
-        on_failure: Callable | tuple[Callable, tuple | None, dict | None] | None = None,
-        on_success_pipeline: Callable
-        | tuple[Callable, tuple | None, dict | None]
-        | None = None,
-        on_failure_pipeline: Callable
-        | tuple[Callable, tuple | None, dict | None]
-        | None = None,
-        **kwargs,  # JobQueue specific args
-    ) -> str | UUID | None:
-        """Adds a job to the job queue.
-
-        If the job queue is not configured, it logs an error and returns None.
-
-        Args:
-            name (str): Name of the pipeline to run. Must be a valid identifier.
-            inputs (dict | None): Override pipeline input values. Example: {"data_date": "2025-04-28"}
-            final_vars (list[str] | None): Specify which output variables to return.
-                Example: ["model", "metrics"]
-            config (dict | None): Configuration for Hamilton pipeline executor.
-                Example: {"model": "LogisticRegression"}
-            cache (dict | None): Cache configuration for results. Example: {"recompute": ["node1", "final_node"]}
-            executor_cfg (str | dict | ExecutorConfig | None): Execution configuration, can be:
-                - str: Executor name, e.g. "threadpool", "local"
-                - dict: Raw config, e.g. {"type": "threadpool", "max_workers": 4}
-                - ExecutorConfig: Structured config object
-            with_adapter_cfg (dict | WithAdapterConfig | None): Adapter settings for pipeline execution.
-                Example: {"opentelemetry": True, "tracker": False}
-             pipeline_adapter_cfg (dict | PipelineAdapterConfig | None): Pipeline-specific adapter settings.
-                Example: {"tracker": {"project_id": "123", "tags": {"env": "prod"}}}
-            project_adapter_cfg (dict | ProjectAdapterConfig | None): Project-level adapter settings.
-                Example: {"opentelemetry": {"host": "http://localhost:4317"}}
-            adapter (dict[str, Any] | None): Custom adapter instance for pipeline
-                Example: {"ray_graph_adapter": RayGraphAdapter()}
-            reload (bool): Force reload of pipeline configuration.
-            run_at (dt.datetime | str | None): Future date to run the job.
-                Example: datetime(2025, 4, 28, 12, 0)
-                Example str: "2025-04-28T12:00:00" (ISO format)
-            run_in (dt.datetime | str | None): Time interval to run the job.
-                Example: 3600 (every hour in seconds)
-                Example: datetime.timedelta(days=1)
-                Example str: "1d" (1 day)
-            result_ttl (int | dt.timedelta): Time to live for the job result.
-                Example: 3600 (1 hour in seconds)
-            log_level (str | None): Logging level for the execution. Default None uses project config.
-                Valid values: "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"
-            max_retries (int): Maximum number of retries for execution.
-            retry_delay (float): Delay between retries in seconds.
-            jitter_factor (float): Random jitter factor to add to retry delay
-            retry_exceptions (tuple): Exceptions that trigger a retry.
-            on_success (Callable | tuple[Callable, tuple | None, dict | None] | None): Callback to run on successful job creation.
-            on_failure (Callable | tuple[Callable, tuple | None, dict | None] | None): Callback to run on job creation failure.
-            on_success_pipeline (Callable | tuple[Callable, tuple | None, dict | None] | None): Callback to run on successful pipeline execution.
-            on_failure_pipeline (Callable | tuple[Callable, tuple | None, dict | None] | None): Callback to run on pipeline execution failure.
-            **kwargs: Additional keyword arguments passed to the worker's add_job method.
-                For RQ this includes:
-                    - result_ttl: Time to live for the job result (float or timedelta)
-                    - ttl: Time to live for the job (float or timedelta)
-                    - timeout: Time to wait for the job to complete (float or timedelta)
-                    - queue_name: Name of the queue to use (str)
-                    - retry: Number of retries (int)
-                    - repeat: Repeat count (int or dict)
-                    - rq_on_failure: Callback function on failure (callable)
-                    - rq_on_success: Callback function on success (callable)
-                    - rq_on_stopped: Callback function on stop (callable)
-                For APScheduler, this includes:
-                    - job_executor: Job executor to use (str)
-
-        Returns:
-            str | UUID | None: The ID of the job that was added to the job queue, or None if the job queue is not configured.
-
-        Raises:
-            ValueError: If the job ID is not valid or if the job cannot be scheduled.
-
-        Example:
-            >>> from flowerpower.pipeline import PipelineManager
-            >>> pm = PipelineManager()
-            >>> job_id = pm.add_job("example_pipeline", inputs={"input1": 42})
-
-        """
-        if self.jqm is None:
-            logger.error(
-                "This PipelineManager instance does not have a job queue configured. Skipping job execution."
-            )
-            return None
-
-        kwargs["on_success"] = kwargs.get("rq_on_success", None)
-        kwargs["on_failure"] = kwargs.get("rq_on_failure", None)
-        kwargs["on_stopped"] = kwargs.get("rq_on_stopped", None)
-
-        run_func = self._get_run_func(
-            name=name,
-            reload=reload,
-            on_success=on_success_pipeline,
-            on_failure=on_failure_pipeline,
-        )
-
-        run_in = (
-            duration_parser.parse(run_in) if isinstance(run_in, str) else run_in
-        )  # convert to seconds
-        run_at = (
-            dt.datetime.fromisoformat(run_at) if isinstance(run_at, str) else run_at
-        )
-
-        add_job = run_with_callback(on_success=on_success, on_failure=on_failure)(
-            self.jqm.add_job
-        )
-        return add_job(
-            run_func=run_func,
-            pipeline_cfg=self._pipeline_cfg,
-            name=name,  # Pass name for logging
-            # Pass run parameters
-            inputs=inputs,
-            final_vars=final_vars,
-            config=config,
-            cache=cache,
-            executor_cfg=executor_cfg,
-            with_adapter_cfg=with_adapter_cfg,
-            pipeline_adapter_cfg=pipeline_adapter_cfg,
-            project_adapter_cfg=project_adapter_cfg,
-            adapter=adapter,
-            # reload=reload,  # Note: reload already happened
-            log_level=log_level,
-            result_ttl=result_ttl,
-            run_at=run_at,
-            run_in=run_in,
-            max_retries=max_retries,
-            retry_delay=retry_delay,
-            jitter_factor=jitter_factor,
-            retry_exceptions=retry_exceptions,
-            **kwargs,  # Pass worker args
-        )
-
-    def schedule(
-        self,
-        name: str,
-        inputs: dict | None = None,
-        final_vars: list[str] | None = None,
-        config: dict | None = None,
-        cache: bool | dict = False,
-        executor_cfg: str | dict | ExecutorConfig | None = None,
-        with_adapter_cfg: dict | WithAdapterConfig | None = None,
-        pipeline_adapter_cfg: dict | PipelineAdapterConfig | None = None,
-        project_adapter_cfg: dict | ProjectAdapterConfig | None = None,
-        adapter: dict[str, Any] | None = None,
-        reload: bool = False,
-        log_level: str | None = None,
-        cron: str | dict[str, str | int] | None = None,
-        interval: int | str | dict[str, str | int] | None = None,
-        date: dt.datetime | str | None = None,
-        overwrite: bool = False,
-        schedule_id: str | None = None,
-        max_retries: int | None = None,
-        retry_delay: float | None = None,
-        jitter_factor: float | None = None,
-        retry_exceptions: tuple | list | None = None,
-        on_success: Callable | tuple[Callable, tuple | None, dict | None] | None = None,
-        on_failure: Callable | tuple[Callable, tuple | None, dict | None] | None = None,
-        on_success_pipeline: Callable
-        | tuple[Callable, tuple | None, dict | None]
-        | None = None,
-        on_failure_pipeline: Callable
-        | tuple[Callable, tuple | None, dict | None]
-        | None = None,
-        **kwargs: Any,
-    ) -> str | UUID | None:
-        """Schedule a pipeline to run on a recurring or future basis.
-
-        If the job queue is not configured, it logs an error and returns None.
-
-        Args:
-            name (str): The name of the pipeline to run.
-            inputs (dict | None): Inputs for the pipeline run (overrides config).
-            final_vars (list[str] | None): Final variables for the pipeline run (overrides config).
-            config (dict | None): Hamilton driver config (overrides config).
-            cache (bool | dict): Cache settings (overrides config).
-            executor_cfg (str | dict | ExecutorConfig | None): Executor configuration (overrides config).
-            with_adapter_cfg (dict | WithAdapterConfig | None): Adapter configuration (overrides config).
-            pipeline_adapter_cfg (dict | PipelineAdapterConfig | None): Pipeline adapter configuration (overrides config).
-            project_adapter_cfg (dict | ProjectAdapterConfig | None): Project adapter configuration (overrides config).
-            adapter (dict[str, Any] | None): Additional Hamilton adapters (overrides config).
-            reload (bool): Whether to reload module and pipeline config. Defaults to False.
-            log_level (str | None): Log level for the run (overrides config).
-            cron (str | dict[str, str | int] | None): Cron expression or settings
-                Example string: "0 0 * * *" (daily at midnight)
-                Example dict: {"minute": "0", "hour": "*/2"} (every 2 hours)
-            interval (int | str | dict[str, str | int] | None): Time interval for recurring execution
-                Example int: 3600 (every hour in seconds)
-                Example str: "1h" (every hour)
-                Example dict: {"hours": 1, "minutes": 30} (every 90 minutes)
-            date (dt.datetime | str | None): Future date for
-                Example: datetime(2025, 4, 28, 12, 0)
-                Example str: "2025-04-28T12:00:00" (ISO format)
-            overwrite (bool): Whether to overwrite existing schedule with the same ID
-            schedule_id (str | None): Unique identifier for the schedule
-            max_retries (int): Maximum number of retries for execution
-            retry_delay (float): Delay between retries in seconds
-            jitter_factor (float): Random jitter factor to add to retry delay
-            retry_exceptions (tuple): Exceptions that trigger a retry
-            on_success (Callable | tuple[Callable, tuple | None, dict | None] | None): Callback to run on successful schedule creation.
-            on_failure (Callable | tuple[Callable, tuple | None, dict | None] | None): Callback to run on schedule creation failure.
-            on_success_pipeline (Callable | tuple[Callable, tuple | None, dict | None] | None): Callback to run on successful pipeline execution.
-            on_failure_pipeline (Callable | tuple[Callable, tuple | None, dict | None] | None): Callback to run on pipeline execution failure.
-            **kwargs: JobQueue-specific scheduling options
-                For RQ:
-                    - result_ttl: Result lifetime (int seconds)
-                    - ttl: Job lifetime (int seconds)
-                    - timeout: Job execution timeout (int seconds)
-                    - queue_name: Queue to use (str)
-                    - repeat: Repeat count (int or dict)
-                    - rq_on_failure: Callback function on failure (callable)
-                    - rq_on_success: Callback function on success (callable)
-                    - rq_on_stopped: Callback function on stop (callable)
-                For APScheduler:
-                    - misfire_grace_time: Late execution window
-                    - coalesce: Combine missed executions (bool)
-                    - max_running_jobs: Concurrent instances limit (int)
-
-        Returns:
-            str | UUID | None: Unique identifier for the created schedule, or None if scheduling fails.
-
-        Raises:
-            ValueError: If schedule parameters are invalid
-            RuntimeError: If scheduling fails
-
-        Example:
-            >>> from flowerpower.pipeline import PipelineManager
-            >>> from datetime import datetime, timedelta
-            >>>
-            >>> manager = PipelineManager()
-            >>>
-            >>> # Daily schedule with cron
-            >>> schedule_id = manager.schedule(
-            ...     name="daily_metrics",
-            ...     cron="0 0 * * *",
-            ...     inputs={"date": "{{ execution_date }}"}
-            ... )
-            >>>
-            >>> # Interval-based schedule
-            >>> schedule_id = manager.schedule(
-            ...     name="monitoring",
-            ...     interval={"minutes": 15},
-            ...     with_adapter_cfg={"enable_alerts": True}
-            ... )
-            >>>
-            >>> # Future one-time execution
-            >>> future_date = datetime.now() + timedelta(days=1)
-            >>> schedule_id = manager.schedule(
-            ...     name="batch_process",
-            ...     date=future_date,
-            ...     executor_cfg={"type": "async"}
-            ... )
-        """
-        if self.jqm is None:
-            logger.error(
-                "This PipelineManager instance does not have a job queue configured. Skipping job execution."
-            )
-            return None
-
-        kwargs["on_success"] = kwargs.get("rq_on_success", None)
-        kwargs["on_failure"] = kwargs.get("rq_on_failure", None)
-        kwargs["on_stopped"] = kwargs.get("rq_on_stopped", None)
-
-        # pipeline_cfg = self._load_pipeline_cfg(name=name, reload=reload)
-        run_func = self._get_run_func(
-            name=name,
-            reload=reload,
-            on_success=on_success_pipeline,
-            on_failure=on_failure_pipeline,
-        )
-        interval = (
-            duration_parser.parse(interval) if isinstance(interval, str) else interval
-        )
-        date = dt.datetime.fromisoformat(date) if isinstance(date, str) else date
-
-        schedule = run_with_callback(on_success=on_success, on_failure=on_failure)(
-            self.jqm.schedule
-        )
-        return schedule(
-            run_func=run_func,
-            pipeline_cfg=self._pipeline_cfg,
-            inputs=inputs,
-            final_vars=final_vars,
-            config=config,
-            cache=cache,
-            executor_cfg=executor_cfg,
-            with_adapter_cfg=with_adapter_cfg,
-            pipeline_adapter_cfg=pipeline_adapter_cfg,
-            project_adapter_cfg=project_adapter_cfg,
-            adapter=adapter,
-            reload=reload,
-            log_level=log_level,
-            cron=cron,
-            interval=interval,
-            date=date,
-            overwrite=overwrite,
-            schedule_id=schedule_id,
-            max_retries=max_retries,
-            retry_delay=retry_delay,
-            jitter_factor=jitter_factor,
-            retry_exceptions=retry_exceptions,
-            **kwargs,
-        )
-
-    def schedule_all(self, **kwargs: Any) -> None:
-        """Schedule all pipelines that are enabled in their configuration.
-
-        For each enabled pipeline, applies its configured schedule settings
-        and any provided overrides.
-
-        Args:
-            **kwargs: Overrides for schedule settings that apply to all pipelines.
-                See schedule() method for supported arguments.
-
-        Example:
-            >>> from flowerpower.pipeline import PipelineManager
-            >>>
-            >>> manager = PipelineManager()
-            >>>
-            >>> # Schedule all with default settings
-            >>> manager.schedule_all()
-            >>>
-            >>> # Schedule all with common overrides
-            >>> manager.schedule_all(
-            ...     max_running_jobs=2,
-            ...     coalesce=True,
-            ...     misfire_grace_time=300
-            ... )
-        """
-        scheduled_ids = []
-        errors = []
-        pipeline_names = self.list_pipelines()
-        if not pipeline_names:
-            logger.warning("No pipelines found to schedule.")
-            return
-
-        logger.info(f"Attempting to schedule {len(pipeline_names)} pipelines...")
-        for name in pipeline_names:
-            try:
-                pipeline_cfg = self.load_pipeline(name=name, reload=True)
-
-                if not pipeline_cfg.schedule.enabled:
-                    logger.info(
-                        f"Skipping scheduling for '{name}': Not enabled in config."
-                    )
-                    continue
-
-                logger.info(f"Scheduling [cyan]{name}[/cyan]...")
-                schedule_id = self.schedule(name=name, reload=False, **kwargs)
-                if schedule_id is None:
-                    logger.info(
-                        f"🟡 Skipping adding schedule for [cyan]{name}[/cyan]: Job queue backend not available or scheduling failed."
-                    )
-                    continue
-                scheduled_ids.append(schedule_id)
-            except Exception as e:
-                logger.error(f"Failed to schedule pipeline '{name}': {e}")
-                errors.append(name)
-
-        if errors:
-            logger.error(f"Finished scheduling with errors for: {', '.join(errors)}")
-        else:
-            logger.info(f"Successfully scheduled {len(scheduled_ids)} pipelines.")
-
-    @property
-    def schedules(self) -> list[Any]:
-        """Get list of current pipeline schedules.
-
-        Retrieves all active schedules from the worker system.
-
-        Returns:
-            list[Any]: List of schedule objects. Exact type depends on worker:
-                - RQ: List[rq.job.Job]
-                - APScheduler: List[apscheduler.schedulers.base.Schedule]
-
-        Example:
-            >>> from flowerpower.pipeline import PipelineManager
-            >>>
-            >>> manager = PipelineManager()
-            >>> for schedule in manager.schedules:
-            ...     print(f"{schedule.id}: Next run at {schedule.next_run_time}")
-        """
-        if self.jqm is None:
-            logger.error(
-                "This PipelineManager instance does not have a job queue configured. Skipping schedule retrieval."
-            )
-            return []
-        try:
-            return self.jqm._get_schedules()
-        except Exception as e:
-            logger.error(f"Failed to retrieve schedules: {e}")
-            return []