FlowerPower 0.20.0__py3-none-any.whl → 0.30.0__py3-none-any.whl

flowerpower/pipeline/base.py

@@ -7,6 +7,7 @@ from fsspec_utils import AbstractFileSystem, BaseStorageOptions, filesystem
 from loguru import logger
 from munch import Munch
 
+from ..settings import CONFIG_DIR, PIPELINES_DIR
 from ..cfg import PipelineConfig, ProjectConfig
 from ..utils.logging import setup_logging
 
@@ -40,26 +41,35 @@ class BasePipeline:
         base_dir: str | None = None,
         storage_options: dict | Munch | BaseStorageOptions = {},
         fs: AbstractFileSystem | None = None,
-        cfg_dir: str = "conf",
-        pipelines_dir: str = "pipelines",
-        job_queue_type: str | None = None,  # New parameter for worker backend
+        
     ):
         self._base_dir = base_dir
         self._storage_options = storage_options
         if fs is None:
             fs = filesystem(self._base_dir, **self._storage_options)
         self._fs = fs
-        self._cfg_dir = cfg_dir
-        self._pipelines_dir = pipelines_dir
-        self._job_queue_type = job_queue_type
+        
 
+        self._setup_paths()
+        self._setup_directories()
+        self._add_modules_path()
+
+    def _setup_paths(self) -> None:
+        """Set up configuration and pipeline directory paths."""
+        self._cfg_dir = CONFIG_DIR
+        self._pipelines_dir = PIPELINES_DIR
+
+    def _setup_directories(self) -> None:
+        """Set up required directories with proper error handling."""
         try:
             self._fs.makedirs(f"{self._cfg_dir}/pipelines", exist_ok=True)
             self._fs.makedirs(self._pipelines_dir, exist_ok=True)
-        except Exception as e:
+        except (OSError, PermissionError) as e:
             logger.error(f"Error creating directories: {e}")
-
-        self._add_modules_path()
+            raise
+        except Exception as e:
+            logger.error(f"Unexpected error creating directories: {e}")
+            raise
 
     def __enter__(self) -> "BasePipeline":
         return self
@@ -98,7 +108,6 @@ class BasePipeline:
         """
         return ProjectConfig.load(
             base_dir=self._base_dir,
-            job_queue_type=self._job_queue_type,
             fs=self._fs,
             storage_options=self._storage_options,
         )

flowerpower/pipeline/io.py

@@ -43,6 +43,41 @@ class PipelineIOManager:
         self._cfg_dir = registry._cfg_dir
         self._pipelines_dir = registry._pipelines_dir
 
+    def _get_pipeline_files(self, name: str) -> list[str]:
+        """Get the list of files for a single pipeline."""
+        return [
+            "conf/project.yml",
+            f"conf/pipelines/{name}.yml",
+            f"pipelines/{name}.py",
+        ]
+
+    def _get_many_pipeline_files(self, names: list[str]) -> list[str]:
+        """Get the list of files for multiple pipelines."""
+        files = ["conf/project.yml"]
+        for name in names:
+            files.extend([
+                f"conf/pipelines/{name}.yml",
+                f"pipelines/{name}.py",
+            ])
+        return files
+
+    def _get_all_pipeline_files(self) -> list[str] | None:
+        """Get all pipeline files (returns None to let _sync_filesystem auto-discover)."""
+        return None
+
+    def _print_import_success(self, names: list[str], src_base_dir: str) -> None:
+        """Print success message for import operations."""
+        console.print(
+            f"✅ Imported pipelines [bold blue]{', '.join(names)}[/bold blue] from [green]{src_base_dir}[/green] to [bold blue]{self.project_cfg.name}[/bold blue]"
+        )
+
+    def _print_export_success(self, names: list[str] | None, dest_base_dir: str) -> None:
+        """Print success message for export operations."""
+        if names:
+            self._print_export_success(names, dest_base_dir)
+        else:
+            self._print_export_success(None, dest_base_dir)
+
     def _sync_filesystem(
         self,
         src_base_dir: str,
@@ -156,12 +191,7 @@ class PipelineIOManager:
             pm.import_pipeline("my_pipeline", "/path/to/pipeline")
             ```
         """
-        files = [
-            "conf/project.yml",
-            f"conf/pipelines/{name}.yml",
-            f"pipelines/{name}.py",
-        ]
-
+        files = self._get_pipeline_files(name)
         self._sync_filesystem(
             src_base_dir=src_base_dir,
             src_fs=src_fs,
@@ -173,10 +203,7 @@ class PipelineIOManager:
             overwrite=overwrite,
         )
 
-        # Use project_cfg.name directly
-        console.print(
-            f"✅ Imported pipeline [bold blue]{name}[/bold blue] from [green]{src_base_dir}[/green] to [bold blue]{self.project_cfg.name}[/bold blue]"
-        )
+        self._print_import_success([name], src_base_dir)
 
     def import_many(
         self,
@@ -210,15 +237,8 @@ class PipelineIOManager:
             ```
         """
 
-        files = ["conf/project.yml"]
+        files = self._get_many_pipeline_files(names)
 
-        for name in names:
-            files.extend([
-                f"conf/pipelines/{name}.yml",
-                f"pipelines/{name}.py",
-            ])
-
-        # Sync the filesystem
         self._sync_filesystem(
             src_base_dir=src_base_dir,
             src_fs=src_fs,
@@ -229,9 +249,7 @@ class PipelineIOManager:
             files=files,
             overwrite=overwrite,
         )
-        console.print(
-            f"✅ Imported pipelines [bold blue]{', '.join(names)}[/bold blue] from [green]{src_base_dir}[/green] to [bold blue]{self.project_cfg.name}[/bold blue]"
-        )
+        self._print_import_success(names, src_base_dir)
 
     def import_all(
         self,
@@ -260,6 +278,8 @@ class PipelineIOManager:
             # pm.import_all("s3://my-bucket/pipelines_backup", storage_options={"key": "...", "secret": "..."}, overwrite=False)
             ```
         """
+        files = self._get_all_pipeline_files()
+
         self._sync_filesystem(
             src_base_dir=src_base_dir,
             src_fs=src_fs,
@@ -267,12 +287,10 @@ class PipelineIOManager:
             dest_base_dir=".",
             dest_fs=self._fs,
             dest_storage_options=None,
-            files=None,
+            files=files,
             overwrite=overwrite,
         )
-        console.print(
-            f"✅ Imported all pipelines from [green]{src_base_dir}[/green] to [bold blue]{self.project_cfg.name}[/bold blue]"
-        )
+        self._print_import_success([], src_base_dir)
 
     def export_pipeline(
         self,
@@ -306,13 +324,8 @@ class PipelineIOManager:
             # pm.export("my_pipeline", "s3://my-bucket/exports", storage_options={"key": "...", "secret": "..."})
             ```
         """
-        files = [
-            "conf/project.yml",
-            f"conf/pipelines/{name}.yml",
-            f"pipelines/{name}.py",
-        ]
+        files = self._get_pipeline_files(name)
 
-        # Sync the filesystem
         self._sync_filesystem(
             src_base_dir=".",
             src_fs=self._fs,
@@ -324,9 +337,7 @@ class PipelineIOManager:
             overwrite=overwrite,
         )
 
-        console.print(
-            f"✅ Exported pipeline [bold blue]{self.project_cfg.name}.{name}[/bold blue] to [green]{dest_base_dir}[/green]"
-        )
+        self._print_export_success([name], dest_base_dir)
 
     def export_many(
         self,
@@ -356,21 +367,15 @@ class PipelineIOManager:
             pm.export_many(pipelines_to_export, "/path/to/export_dir", overwrite=True)
             ```
         """
-        files = [
-            "conf/project.yml",
-        ]
+        # Check if pipelines exist in the registry
         for name in names:
-            # Check if the pipeline exists in the registry
             if not self.registry.has_pipeline(name):
                 raise ValueError(
                     f"Pipeline {name} does not exist in the registry. Please check the name."
                 )
-            # Add pipeline files to the list
-            files.extend([
-                f"conf/pipelines/{name}.yml",
-                f"pipelines/{name}.py",
-            ])
-        # Sync the filesystem
+
+        files = self._get_many_pipeline_files(names)
+
         self._sync_filesystem(
             src_base_dir=".",
             src_fs=self._fs,
@@ -412,7 +417,8 @@ class PipelineIOManager:
             # pm.export_all("s3://my-bucket/pipelines_backup", storage_options={"key": "...", "secret": "..."}, overwrite=False)
             ```
         """
-        # sync the filesystem
+        files = self._get_all_pipeline_files()
+
         self._sync_filesystem(
             src_base_dir=".",
             src_fs=self._fs,
@@ -420,7 +426,7 @@ class PipelineIOManager:
             dest_base_dir=dest_base_dir,
             dest_fs=dest_fs,
             dest_storage_options=dest_storage_options,
-            files=None,
+            files=files,
             overwrite=overwrite,
         )
         console.print(

flowerpower/pipeline/manager.py

@@ -18,17 +18,17 @@ except ImportError:
 
 from fsspec_utils import AbstractFileSystem, BaseStorageOptions, filesystem
 
-from .. import settings
+from ..settings import CONFIG_DIR, PIPELINES_DIR, CACHE_DIR
 from ..cfg import PipelineConfig, ProjectConfig
 from ..cfg.pipeline.adapter import AdapterConfig as PipelineAdapterConfig
-from ..cfg.pipeline.run import ExecutorConfig, WithAdapterConfig
+from ..cfg.pipeline.run import ExecutorConfig, RunConfig, WithAdapterConfig
 from ..cfg.project.adapter import AdapterConfig as ProjectAdapterConfig
 from ..utils.logging import setup_logging
 from .io import PipelineIOManager
 from .registry import HookType, PipelineRegistry
 from .visualizer import PipelineVisualizer
 
-setup_logging(level=settings.LOG_LEVEL)
+setup_logging()
 
 GraphType = TypeVar("GraphType")  # Type variable for graphviz.Digraph
 
@@ -40,13 +40,11 @@ class PipelineManager:
     - Configuration management and loading
     - Pipeline creation, deletion, and discovery
     - Pipeline execution via PipelineRunner
-    - Job scheduling via PipelineScheduler
     - Visualization via PipelineVisualizer
     - Import/export operations via PipelineIOManager
 
     Attributes:
         registry (PipelineRegistry): Handles pipeline registration and discovery
-        scheduler (PipelineScheduler): Manages job scheduling and execution
         visualizer (PipelineVisualizer): Handles pipeline visualization
         io (PipelineIOManager): Manages pipeline import/export operations
         project_cfg (ProjectConfig): Current project configuration
@@ -64,7 +62,6 @@ class PipelineManager:
         >>> # Create manager with custom settings
         >>> manager = PipelineManager(
         ...     base_dir="/path/to/project",
-        ...     job_queue_type="rq",
         ...     log_level="DEBUG"
         ... )
     """
@@ -74,9 +71,9 @@ class PipelineManager:
         base_dir: str | None = None,
         storage_options: dict | Munch | BaseStorageOptions | None = None,
         fs: AbstractFileSystem | None = None,
-        cfg_dir: str | None = settings.CONFIG_DIR,
-        pipelines_dir: str | None = settings.PIPELINES_DIR,
-        job_queue_type: str | None = None,
+        cfg_dir: str | None = CONFIG_DIR,
+        pipelines_dir: str | None = PIPELINES_DIR,
+        
         log_level: str | None = None,
     ) -> None:
         """Initialize the PipelineManager.
@@ -95,8 +92,7 @@ class PipelineManager:
                 Example: "config" or "settings".
             pipelines_dir: Override default pipelines directory name ('pipelines').
                 Example: "flows" or "dags".
-            job_queue_type: Override worker type from project config/settings.
-                Valid values: "rq".
+            
             log_level: Set logging level for the manager.
                 Valid values: "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"
 
@@ -116,7 +112,7 @@ class PipelineManager:
             ...         "key": "ACCESS_KEY",
             ...         "secret": "SECRET_KEY"
             ...     },
-            ...     job_queue_type="rq",
+            
             ...     log_level="DEBUG"
             ... )
         """
@@ -128,7 +124,7 @@ class PipelineManager:
         if storage_options is not None:
             cached = True
             cache_storage = posixpath.join(
-                posixpath.expanduser(settings.CACHE_DIR),
+                posixpath.expanduser(CACHE_DIR),
                 self._base_dir.split("://")[-1],
             )
             os.makedirs(cache_storage, exist_ok=True)
@@ -154,17 +150,20 @@ class PipelineManager:
         self._pipelines_dir = pipelines_dir
 
         self._load_project_cfg(
-            reload=True, job_queue_type=job_queue_type
+            reload=True
         )  # Load project config
-        self._job_queue_type = job_queue_type or self.project_cfg.job_queue.type
+        
 
         # Ensure essential directories exist (using paths from loaded project_cfg)
         try:
             self._fs.makedirs(self._cfg_dir, exist_ok=True)
             self._fs.makedirs(self._pipelines_dir, exist_ok=True)
-        except Exception as e:
+        except (OSError, PermissionError) as e:
             logger.error(f"Error creating essential directories: {e}")
-            # Consider raising an error here depending on desired behavior
+            raise RuntimeError(f"Failed to create essential directories: {e}") from e
+        except Exception as e:
+            logger.error(f"Unexpected error creating essential directories: {e}")
+            raise RuntimeError(f"Unexpected filesystem error: {e}") from e
 
         # Ensure pipeline modules can be imported
         self._add_modules_path()
@@ -263,7 +262,7 @@ class PipelineManager:
             sys.path.insert(0, modules_path)
 
     def _load_project_cfg(
-        self, reload: bool = False, job_queue_type: str | None = None
+        self, reload: bool = False
     ) -> ProjectConfig:
         """Load or reload the project configuration.
 
@@ -287,8 +286,8 @@ class PipelineManager:
             >>> # Internal usage
             >>> manager = PipelineManager()
             >>> project_cfg = manager._load_project_cfg(reload=True)
-            >>> print(project_cfg.worker.type)
-            'rq'
+            >>> print(project_cfg.name)
+            'my_project'
         """
         if hasattr(self, "_project_cfg") and not reload:
             return self._project_cfg
@@ -296,7 +295,6 @@ class PipelineManager:
         # Pass overrides to ProjectConfig.load
         self._project_cfg = ProjectConfig.load(
             base_dir=self._base_dir,
-            job_queue_type=job_queue_type,
             fs=self._fs,  # Pass pre-configured fs if provided
             storage_options=self._storage_options,
         )
@@ -371,8 +369,8 @@ class PipelineManager:
         Example:
             >>> manager = PipelineManager()
             >>> cfg = manager.project_cfg
-            >>> print(cfg.worker.type)
-            'rq'
+            >>> print(cfg.name)
+            'my_project'
         """
         if not hasattr(self, "_project_cfg"):
             self._load_project_cfg()
@@ -402,26 +400,82 @@ class PipelineManager:
 
     # --- Core Execution Method ---
 
+    def _merge_run_config_with_kwargs(self, run_config: RunConfig, kwargs: dict) -> RunConfig:
+        """Merge kwargs into a RunConfig object.
+        
+        This helper method updates the RunConfig object with values from kwargs,
+        handling different types of attributes appropriately.
+        
+        Args:
+            run_config: The RunConfig object to update
+            kwargs: Dictionary of additional parameters to merge
+            
+        Returns:
+            RunConfig: Updated RunConfig object
+        """
+        # Handle dictionary-like attributes with update or deep merge
+        if 'inputs' in kwargs and kwargs['inputs'] is not None:
+            if run_config.inputs is None:
+                run_config.inputs = kwargs['inputs']
+            else:
+                run_config.inputs.update(kwargs['inputs'])
+                
+        if 'config' in kwargs and kwargs['config'] is not None:
+            if run_config.config is None:
+                run_config.config = kwargs['config']
+            else:
+                run_config.config.update(kwargs['config'])
+                
+        if 'cache' in kwargs and kwargs['cache'] is not None:
+            run_config.cache = kwargs['cache']
+            
+        if 'adapter' in kwargs and kwargs['adapter'] is not None:
+            if run_config.adapter is None:
+                run_config.adapter = kwargs['adapter']
+            else:
+                run_config.adapter.update(kwargs['adapter'])
+        
+        # Handle executor_cfg - convert string/dict to ExecutorConfig if needed
+        if 'executor_cfg' in kwargs and kwargs['executor_cfg'] is not None:
+            executor_cfg = kwargs['executor_cfg']
+            if isinstance(executor_cfg, str):
+                run_config.executor = ExecutorConfig(type=executor_cfg)
+            elif isinstance(executor_cfg, dict):
+                run_config.executor = ExecutorConfig.from_dict(executor_cfg)
+            elif isinstance(executor_cfg, ExecutorConfig):
+                run_config.executor = executor_cfg
+        
+        # Handle adapter configurations
+        if 'with_adapter_cfg' in kwargs and kwargs['with_adapter_cfg'] is not None:
+            with_adapter_cfg = kwargs['with_adapter_cfg']
+            if isinstance(with_adapter_cfg, dict):
+                run_config.with_adapter = WithAdapterConfig.from_dict(with_adapter_cfg)
+            elif isinstance(with_adapter_cfg, WithAdapterConfig):
+                run_config.with_adapter = with_adapter_cfg
+                
+        if 'pipeline_adapter_cfg' in kwargs and kwargs['pipeline_adapter_cfg'] is not None:
+            run_config.pipeline_adapter_cfg = kwargs['pipeline_adapter_cfg']
+            
+        if 'project_adapter_cfg' in kwargs and kwargs['project_adapter_cfg'] is not None:
+            run_config.project_adapter_cfg = kwargs['project_adapter_cfg']
+        
+        # Handle simple attributes
+        simple_attrs = [
+            'final_vars', 'reload', 'log_level', 'max_retries', 'retry_delay',
+            'jitter_factor', 'retry_exceptions', 'on_success', 'on_failure'
+        ]
+        
+        for attr in simple_attrs:
+            if attr in kwargs and kwargs[attr] is not None:
+                setattr(run_config, attr, kwargs[attr])
+        
+        return run_config
+
     def run(
         self,
         name: str,
-        inputs: dict | None = None,
-        final_vars: list[str] | None = None,
-        config: dict | None = None,
-        cache: dict | None = None,
-        executor_cfg: str | dict | ExecutorConfig | None = None,
-        with_adapter_cfg: dict | WithAdapterConfig | None = None,
-        pipeline_adapter_cfg: dict | PipelineAdapterConfig | None = None,
-        project_adapter_cfg: dict | ProjectAdapterConfig | None = None,
-        adapter: dict[str, Any] | None = None,
-        reload: bool = False,
-        log_level: str | None = None,
-        max_retries: int | None = None,
-        retry_delay: float | None = None,
-        jitter_factor: float | None = None,
-        retry_exceptions: tuple | list | None = None,
-        on_success: Callable | tuple[Callable, tuple | None, dict | None] | None = None,
-        on_failure: Callable | tuple[Callable, tuple | None, dict | None] | None = None,
+        run_config: RunConfig | None = None,
+        **kwargs
     ) -> dict[str, Any]:
         """Execute a pipeline synchronously and return its results.
 
@@ -430,33 +484,36 @@ class PipelineManager:
 
         Args:
             name (str): Name of the pipeline to run. Must be a valid identifier.
-            inputs (dict | None): Override pipeline input values. Example: {"data_date": "2025-04-28"}
-            final_vars (list[str] | None): Specify which output variables to return.
-                Example: ["model", "metrics"]
-            config (dict | None): Configuration for Hamilton pipeline executor.
-                Example: {"model": "LogisticRegression"}
-            cache (dict | None): Cache configuration for results. Example: {"recompute": ["node1", "final_node"]}
-            executor_cfg (str | dict | ExecutorConfig | None): Execution configuration, can be:
-                - str: Executor name, e.g. "threadpool", "local"
-                - dict: Raw config, e.g. {"type": "threadpool", "max_workers": 4}
-                - ExecutorConfig: Structured config object
-            with_adapter_cfg (dict | WithAdapterConfig | None): Adapter settings for pipeline execution.
-                Example: {"opentelemetry": True, "tracker": False}
-             pipeline_adapter_cfg (dict | PipelineAdapterConfig | None): Pipeline-specific adapter settings.
-                Example: {"tracker": {"project_id": "123", "tags": {"env": "prod"}}}
-            project_adapter_cfg (dict | ProjectAdapterConfig | None): Project-level adapter settings.
-                Example: {"opentelemetry": {"host": "http://localhost:4317"}}
-            adapter (dict[str, Any] | None): Custom adapter instance for pipeline
-                Example: {"ray_graph_adapter": RayGraphAdapter()}
-            reload (bool): Force reload of pipeline configuration.
-            log_level (str | None): Logging level for the execution. Default None uses project config.
-                Valid values: "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"
-            max_retries (int): Maximum number of retries for execution.
-            retry_delay (float): Delay between retries in seconds.
-            jitter_factor (float): Random jitter factor to add to retry delay
-            retry_exceptions (tuple): Exceptions that trigger a retry.
-            on_success (Callable | tuple[Callable, tuple | None, dict | None] | None): Callback to run on successful pipeline execution.
-            on_failure (Callable | tuple[Callable, tuple | None, dict | None] | None): Callback to run on pipeline execution failure.
+            run_config (RunConfig | None): Run configuration object containing all execution parameters.
+                If None, the default configuration from the pipeline will be used.
+            **kwargs: Additional parameters to override the run_config. Supported parameters include:
+                inputs (dict | None): Override pipeline input values. Example: {"data_date": "2025-04-28"}
+                final_vars (list[str] | None): Specify which output variables to return.
+                    Example: ["model", "metrics"]
+                config (dict | None): Configuration for Hamilton pipeline executor.
+                    Example: {"model": "LogisticRegression"}
+                cache (dict | None): Cache configuration for results. Example: {"recompute": ["node1", "final_node"]}
+                executor_cfg (str | dict | ExecutorConfig | None): Execution configuration, can be:
+                    - str: Executor name, e.g. "threadpool", "local"
+                    - dict: Raw config, e.g. {"type": "threadpool", "max_workers": 4}
+                    - ExecutorConfig: Structured config object
+                with_adapter_cfg (dict | WithAdapterConfig | None): Adapter settings for pipeline execution.
+                    Example: {"opentelemetry": True, "tracker": False}
+                pipeline_adapter_cfg (dict | PipelineAdapterConfig | None): Pipeline-specific adapter settings.
+                    Example: {"tracker": {"project_id": "123", "tags": {"env": "prod"}}}
+                project_adapter_cfg (dict | ProjectAdapterConfig | None): Project-level adapter settings.
+                    Example: {"opentelemetry": {"host": "http://localhost:4317"}}
+                adapter (dict[str, Any] | None): Custom adapter instance for pipeline
+                    Example: {"ray_graph_adapter": RayGraphAdapter()}
+                reload (bool): Force reload of pipeline configuration.
+                log_level (str | None): Logging level for the execution. Default None uses project config.
+                    Valid values: "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"
+                max_retries (int): Maximum number of retries for execution.
+                retry_delay (float): Delay between retries in seconds.
+                jitter_factor (float): Random jitter factor to add to retry delay
+                retry_exceptions (tuple): Exceptions that trigger a retry.
+                on_success (Callable | tuple[Callable, tuple | None, dict | None] | None): Callback to run on successful pipeline execution.
+                on_failure (Callable | tuple[Callable, tuple | None, dict | None] | None): Callback to run on pipeline execution failure.
 
         Returns:
             dict[str, Any]: Pipeline execution results, mapping output variable names
@@ -475,46 +532,47 @@ class PipelineManager:
             >>> # Basic pipeline run
             >>> results = manager.run("data_pipeline")
             >>>
-            >>> # Complex run with overrides
+            >>> # Run with custom RunConfig
+            >>> from flowerpower.cfg.pipeline.run import RunConfig
+            >>> config = RunConfig(inputs={"date": "2025-04-28"}, final_vars=["result"])
+            >>> results = manager.run("ml_pipeline", run_config=config)
+            >>>
+            >>> # Complex run with kwargs overrides
             >>> results = manager.run(
-            ...     name="ml_pipeline",
-            ...     inputs={
-            ...         "training_date": "2025-04-28",
-            ...         "model_params": {"n_estimators": 100}
-            ...     },
+            ...     "ml_pipeline",
+            ...     inputs={"training_date": "2025-04-28"},
             ...     final_vars=["model", "metrics"],
             ...     executor_cfg={"type": "threadpool", "max_workers": 4},
             ...     with_adapter_cfg={"tracker": True},
             ...     reload=True
             ... )
         """
+        # Initialize run_config - use provided config or load pipeline default
+        if run_config is None:
+            run_config = self.load_pipeline(name=name).run
+        
+        # Merge kwargs into run_config
+        if kwargs:
+            run_config = self._merge_run_config_with_kwargs(run_config, kwargs)
+
+        # Set up logging for this specific run if log_level is provided
+        if run_config.log_level is not None:
+            setup_logging(level=run_config.log_level)
+        else:
+            # Ensure logging is reset to default if no specific level is provided for this run
+            setup_logging()
+            
         # Use injected project context, fallback to self for backward compatibility
         project_context = getattr(self, "_project_context", self)
 
         # Get Pipeline instance from registry
         pipeline = self.registry.get_pipeline(
-            name=name, project_context=project_context, reload=reload
+            name=name, project_context=project_context, reload=run_config.reload
         )
 
         # Execute pipeline using its own run method
         return pipeline.run(
-            inputs=inputs,
-            final_vars=final_vars,
-            config=config,
-            cache=cache,
-            executor_cfg=executor_cfg,
-            with_adapter_cfg=with_adapter_cfg,
-            pipeline_adapter_cfg=pipeline_adapter_cfg,
-            project_adapter_cfg=project_adapter_cfg,
-            adapter=adapter,
-            reload=reload,
-            log_level=log_level,
-            max_retries=max_retries,
-            retry_delay=retry_delay,
-            jitter_factor=jitter_factor,
-            retry_exceptions=retry_exceptions,
-            on_success=on_success,
-            on_failure=on_failure,
+            run_config=run_config,
         )
 
     # --- Delegated Methods ---