runnable 0.37.0__py3-none-any.whl → 0.39.0__py3-none-any.whl

extensions/nodes/conditional.py

@@ -7,7 +7,7 @@ from pydantic import Field, field_serializer, field_validator
 from runnable import console, defaults
 from runnable.datastore import Parameter
 from runnable.graph import Graph, create_graph
-from runnable.nodes import CompositeNode, MapVariableType
+from runnable.nodes import CompositeNode, MapVariableType, NodeInD3
 
 logger = logging.getLogger(defaults.LOGGER_NAME)
 
@@ -241,3 +241,21 @@ class ConditionalNode(CompositeNode):
             step_log.status = defaults.FAIL
 
         self._context.run_log_store.add_step_log(step_log, self._context.run_id)
+
+    def to_d3_node(self) -> NodeInD3:
+        def get_display_string() -> str:
+            display = f"match {self.parameter}:\n"
+            for case in self.branches.keys():
+                display += f'    case "{case}":\n        ...\n'
+            if self.default:
+                display += "    case _:\n        ...\n"
+            return display
+
+        return NodeInD3(
+            id=self.internal_name,
+            label="conditional",
+            metadata={
+                "conditioned on": self.parameter,
+                "display": get_display_string(),
+            },
+        )

extensions/nodes/fail.py

@@ -6,7 +6,7 @@ from pydantic import Field
 from runnable import datastore, defaults
 from runnable.datastore import StepLog
 from runnable.defaults import MapVariableType
-from runnable.nodes import TerminalNode
+from runnable.nodes import NodeInD3, TerminalNode
 
 
 class FailNode(TerminalNode):
@@ -70,3 +70,9 @@ class FailNode(TerminalNode):
         step_log.attempts.append(attempt_log)
 
         return step_log
+
+    def to_d3_node(self) -> NodeInD3:
+        return NodeInD3(
+            id=self.internal_name,
+            label="fail",
+        )

extensions/nodes/map.py

@@ -18,7 +18,7 @@ from runnable.datastore import (
 )
 from runnable.defaults import MapVariableType
 from runnable.graph import Graph, create_graph
-from runnable.nodes import CompositeNode
+from runnable.nodes import CompositeNode, NodeInD3
 
 logger = logging.getLogger(defaults.LOGGER_NAME)
 
@@ -348,3 +348,18 @@ class MapNode(CompositeNode):
         self._context.run_log_store.set_parameters(
             parameters=params, run_id=self._context.run_id
         )
+
+    def to_d3_node(self) -> NodeInD3:
+        return NodeInD3(
+            id=self.internal_name,
+            label="map",
+            metadata={
+                "node_type": "map",
+                "iterate_on": self.iterate_on,  # Parameter name containing the iterable
+                "iterate_as": self.iterate_as,  # Name used for each iteration
+                "map_branch_id": self.internal_name
+                + "."
+                + defaults.MAP_PLACEHOLDER,  # The branch identifier pattern
+                "is_composite": True,  # Flag indicating this is a composite node
+            },
+        )

extensions/nodes/parallel.py

@@ -6,7 +6,7 @@ from pydantic import Field, field_serializer
 from runnable import defaults
 from runnable.defaults import MapVariableType
 from runnable.graph import Graph, create_graph
-from runnable.nodes import CompositeNode
+from runnable.nodes import CompositeNode, NodeInD3
 
 
 class ParallelNode(CompositeNode):
@@ -157,3 +157,9 @@ class ParallelNode(CompositeNode):
             step_log.status = defaults.FAIL
 
         self._context.run_log_store.add_step_log(step_log, self._context.run_id)
+
+    def to_d3_node(self) -> NodeInD3:
+        return NodeInD3(
+            id=self.internal_name,
+            label="parallel",
+        )

extensions/nodes/stub.py

@@ -7,7 +7,7 @@ from pydantic import ConfigDict, Field
 from runnable import datastore, defaults
 from runnable.datastore import StepLog
 from runnable.defaults import MapVariableType
-from runnable.nodes import ExecutableNode
+from runnable.nodes import ExecutableNode, NodeInD3
 
 logger = logging.getLogger(defaults.LOGGER_NAME)
 
@@ -87,3 +87,9 @@ class StubNode(ExecutableNode):
         step_log.attempts.append(attempt_log)
 
         return step_log
+
+    def to_d3_node(self) -> NodeInD3:
+        return NodeInD3(
+            id=self.internal_name,
+            label="stub",
+        )

extensions/nodes/success.py

@@ -6,7 +6,7 @@ from pydantic import Field
 from runnable import datastore, defaults
 from runnable.datastore import StepLog
 from runnable.defaults import MapVariableType
-from runnable.nodes import TerminalNode
+from runnable.nodes import NodeInD3, TerminalNode
 
 
 class SuccessNode(TerminalNode):
@@ -70,3 +70,9 @@ class SuccessNode(TerminalNode):
         step_log.attempts.append(attempt_log)
 
         return step_log
+
+    def to_d3_node(self) -> NodeInD3:
+        return NodeInD3(
+            id=self.internal_name,
+            label="success",
+        )

extensions/nodes/task.py

@@ -7,7 +7,7 @@ from pydantic import ConfigDict, Field
 from runnable import datastore, defaults
 from runnable.datastore import StepLog
 from runnable.defaults import MapVariableType
-from runnable.nodes import ExecutableNode
+from runnable.nodes import ExecutableNode, NodeInD3
 from runnable.tasks import BaseTaskType, create_task
 
 logger = logging.getLogger(defaults.LOGGER_NAME)
@@ -90,3 +90,17 @@ class TaskNode(ExecutableNode):
         step_log.attempts.append(attempt_log)
 
         return step_log
+
+    def to_d3_node(self) -> NodeInD3:
+        """
+        Convert the task node to a D3 node representation.
+
+        Returns:
+            NodeInD3: The D3 node representation of the task node.
+        """
+        return NodeInD3(
+            id=self.internal_name,
+            label="task",
+            task_type=self.executable.task_type,
+            metadata=self.executable.get_d3_metadata(),
+        )

runnable/cli.py

@@ -274,5 +274,57 @@ def execute_job(
     )
 
 
+@app.command()
+def ui(
+    host: Annotated[
+        str,
+        typer.Option(
+            "--host",
+            "-h",
+            help="The host to bind the server to",
+        ),
+    ] = "127.0.0.1",
+    port: Annotated[
+        int,
+        typer.Option(
+            "--port",
+            "-p",
+            help="The port to bind the server to",
+        ),
+    ] = 8000,
+    reload: Annotated[
+        bool,
+        typer.Option(
+            "--reload",
+            help="Enable auto-reload for development",
+        ),
+    ] = False,
+):
+    """
+    Start the web UI for pipeline visualization.
+
+    This command starts a FastAPI web server that provides a user interface
+    for visualizing and exploring runnable pipelines.
+    """
+    try:
+        import uvicorn
+
+        from visualization.main import app as web_app
+    except ImportError:
+        typer.echo(
+            "UI dependencies not installed. Install with: pip install runnable[ui]",
+            err=True,
+        )
+        raise typer.Exit(1)
+
+    typer.echo(f"Starting web UI at http://{host}:{port}")
+    uvicorn.run(
+        web_app,
+        host=host,
+        port=port,
+        reload=reload,
+    )
+
+
 if __name__ == "__main__":
     app()

runnable/graph.py

@@ -329,7 +329,7 @@ def create_graph(dag_config: Dict[str, Any], internal_branch_name: str = "") ->
     Returns:
         Graph: The created graph object
     """
-    description: str = dag_config.get("description", None)
+    description: str | None = dag_config.get("description", None)
     start_at: str = cast(
         str, dag_config.get("start_at")
     )  # Let the start_at be relative to the graph
@@ -499,3 +499,274 @@ def search_branch_by_internal_name(dag: Graph, internal_name: str):
         return current_branch
 
     raise exceptions.BranchNotFoundError(internal_name)
+
+
+def get_visualization_data(graph: Graph) -> Dict[str, Any]:
+    """
+    Convert the graph into a D3 visualization friendly format with nodes and links.
+    Handles composite nodes (parallel, map, conditional) by recursively processing their embedded graphs.
+
+    Args:
+        graph: The Graph object to convert
+
+    Returns:
+        Dict with two keys:
+        - nodes: List of node objects with id, type, name, and alias
+        - links: List of edge objects with source and target node ids
+    """
+    import rich.console
+
+    from extensions.nodes.conditional import ConditionalNode
+    from extensions.nodes.map import MapNode
+    from extensions.nodes.parallel import ParallelNode
+    from runnable.nodes import ExecutableNode
+
+    rich_print = rich.console.Console().print
+
+    rich_print(graph)
+
+    nodes = []
+    links = []
+    processed_nodes = set()
+
+    def process_node(
+        node: BaseNode,
+        parent_id: Optional[str] = None,
+        current_graph: Graph = graph,
+        map_node_id: Optional[str] = None,
+        conditional_node_id: Optional[str] = None,
+    ) -> str:
+        node_id = f"{node.internal_name}"
+        node_alias = node.name  # Alias based on the node's name
+
+        if node_id not in processed_nodes:
+            node_data = node.to_d3_node().model_dump(exclude_none=True)
+            node_data["alias"] = node_alias  # Add alias to the node data
+            node_data["display_name"] = node_alias  # Use alias as the display name
+
+            # Add map or parallel related metadata if this node is part of a map branch or parallel branch
+            if map_node_id:
+                if "metadata" not in node_data:
+                    node_data["metadata"] = {}
+
+                # Mark this node as being part of a map branch
+                node_data["metadata"]["belongs_to_node"] = map_node_id
+
+                # If this is the map node itself, add a special attribute
+                if node_id == map_node_id:
+                    node_data["metadata"]["is_map_root"] = True
+
+            # Add conditional related metadata if this node is part of a conditional branch
+            if conditional_node_id:
+                if "metadata" not in node_data:
+                    node_data["metadata"] = {}
+
+                # Mark this node as being part of a conditional branch
+                node_data["metadata"]["belongs_to_node"] = conditional_node_id
+
+                # If this is the conditional node itself, add a special attribute
+                if node_id == conditional_node_id:
+                    node_data["metadata"]["is_conditional_root"] = True
+
+            # Mark parallel nodes with special metadata
+            if isinstance(node, ParallelNode):
+                if "metadata" not in node_data:
+                    node_data["metadata"] = {}
+
+                # Add parallel node type to metadata
+                node_data["metadata"]["node_type"] = "parallel"
+                node_data["metadata"]["parallel_branch_id"] = node_id
+
+            # Mark conditional nodes with special metadata
+            if isinstance(node, ConditionalNode):
+                if "metadata" not in node_data:
+                    node_data["metadata"] = {}
+
+                # Add conditional node type to metadata
+                node_data["metadata"]["node_type"] = "conditional"
+                node_data["metadata"]["conditional_branch_id"] = node_id
+
+            nodes.append(node_data)
+            processed_nodes.add(node_id)
+
+            # Add link from parent if it exists
+            if parent_id:
+                links.append({"source": parent_id, "target": node_id})
+
+            # Handle composite nodes with embedded graphs
+            if isinstance(node, (ParallelNode, MapNode, ConditionalNode)):
+                if isinstance(node, ParallelNode):
+                    # Process each parallel branch
+                    for _, branch in node.branches.items():
+                        branch_start = branch.get_node_by_name(branch.start_at)
+                        process_node(
+                            branch_start,
+                            node_id,
+                            branch,
+                            map_node_id=node_id,
+                            conditional_node_id=conditional_node_id,
+                        )
+
+                    # Handle next node connection after parallel branches complete
+                    if hasattr(node, "next_node") and node.next_node:
+                        try:
+                            next_node = current_graph.get_node_by_name(node.next_node)
+                            next_id = process_node(
+                                next_node,
+                                None,
+                                current_graph=current_graph,
+                                map_node_id=map_node_id,
+                                conditional_node_id=conditional_node_id,
+                            )
+                            links.append(
+                                {
+                                    "source": node_id,
+                                    "target": next_id,
+                                    "type": "success",
+                                }
+                            )
+                        except exceptions.NodeNotFoundError as e:
+                            rich_print(
+                                f"Warning: Next node '{node.next_node}' not found for parallel node '{node.name}': {e}"
+                            )
+
+                elif isinstance(node, MapNode):
+                    # Process map branch
+                    branch_start = node.branch.get_node_by_name(node.branch.start_at)
+                    # Process the branch with additional context about the map node
+                    process_node(
+                        branch_start,
+                        node_id,
+                        node.branch,
+                        map_node_id=node_id,
+                        conditional_node_id=conditional_node_id,
+                    )
+
+                elif isinstance(node, ConditionalNode):
+                    # Process each conditional branch
+                    for _, branch in node.branches.items():
+                        branch_start = branch.get_node_by_name(branch.start_at)
+                        process_node(
+                            branch_start,
+                            node_id,
+                            branch,
+                            map_node_id=map_node_id,
+                            conditional_node_id=node_id,
+                        )
+                    if node.default:
+                        default_start = node.default.get_node_by_name(
+                            node.default.start_at
+                        )
+                        process_node(
+                            default_start,
+                            node_id,
+                            node.default,
+                            map_node_id=map_node_id,
+                            conditional_node_id=node_id,
+                        )
+
+                    # Handle next node connection after conditional branches complete
+                    if hasattr(node, "next_node") and node.next_node:
+                        try:
+                            next_node = current_graph.get_node_by_name(node.next_node)
+                            next_id = process_node(
+                                next_node,
+                                None,
+                                current_graph=current_graph,
+                                map_node_id=map_node_id,
+                                conditional_node_id=conditional_node_id,
+                            )
+                            links.append(
+                                {
+                                    "source": node_id,
+                                    "target": next_id,
+                                    "type": "success",
+                                }
+                            )
+                        except exceptions.NodeNotFoundError as e:
+                            rich_print(
+                                f"Warning: Next node '{node.next_node}' not found for conditional node '{node.name}': {e}"
+                            )
+
+            # Add links to next and on_failure nodes if they exist
+            if isinstance(node, ExecutableNode):
+                # Handle normal "next" links (success path)
+                if hasattr(node, "next_node") and node.next_node:
+                    try:
+                        next_node = current_graph.get_node_by_name(node.next_node)
+                        next_id = process_node(
+                            next_node,
+                            None,
+                            current_graph=current_graph,
+                            map_node_id=map_node_id,
+                            conditional_node_id=conditional_node_id,
+                        )
+                        links.append(
+                            {"source": node_id, "target": next_id, "type": "success"}
+                        )
+                    except exceptions.NodeNotFoundError as e:
+                        rich_print(
+                            f"Warning: Next node '{node.next_node}' not found for node '{node.name}': {e}"
+                        )
+
+                # Handle on_failure links (failure path)
+                if hasattr(node, "on_failure") and node.on_failure:
+                    try:
+                        failure_node = current_graph.get_node_by_name(node.on_failure)
+                        failure_id = process_node(
+                            failure_node,
+                            None,
+                            current_graph=current_graph,
+                            map_node_id=map_node_id,
+                            conditional_node_id=conditional_node_id,
+                        )
+                        links.append(
+                            {"source": node_id, "target": failure_id, "type": "failure"}
+                        )
+                    except exceptions.NodeNotFoundError as e:
+                        rich_print(
+                            f"Warning: On-failure node '{node.on_failure}' not found for node '{node.name}': {e}"
+                        )
+
+                # For backward compatibility, also process all neighbors
+                # This handles cases where node might have other connection types
+                next_nodes = node._get_neighbors()
+                for next_node_name in next_nodes:
+                    # Skip nodes we've already handled explicitly
+                    if (
+                        hasattr(node, "next_node") and node.next_node == next_node_name
+                    ) or (
+                        hasattr(node, "on_failure")
+                        and node.on_failure == next_node_name
+                    ):
+                        continue
+
+                    try:
+                        next_node = current_graph.get_node_by_name(next_node_name)
+                        next_id = process_node(
+                            next_node,
+                            None,
+                            current_graph=current_graph,
+                            map_node_id=map_node_id,
+                            conditional_node_id=conditional_node_id,
+                        )
+                        links.append(
+                            {"source": node_id, "target": next_id, "type": "default"}
+                        )
+                    except exceptions.NodeNotFoundError as e:
+                        rich_print(
+                            f"Warning: Neighbor node '{next_node_name}' not found for node '{node.name}': {e}"
+                        )
+
+        return node_id
+
+    # Start processing from the start node
+    start_node = graph.get_node_by_name(graph.start_at)
+    try:
+        process_node(
+            start_node, None, graph, map_node_id=None, conditional_node_id=None
+        )
+    except (exceptions.NodeNotFoundError, AttributeError, KeyError) as e:
+        rich_print(f"Error processing node {start_node}: {e}")
+
+    return {"nodes": nodes, "links": links}

runnable/nodes.py

@@ -15,6 +15,13 @@ logger = logging.getLogger(defaults.LOGGER_NAME)
 # --8<-- [start:docs]
 
 
+class NodeInD3(BaseModel):
+    id: str
+    label: str
+    task_type: Optional[str] = None
+    metadata: Optional[Dict[str, Any]] = None
+
+
 class BaseNode(ABC, BaseModel):
     """
     Base class with common functionality provided for a Node of a graph.
@@ -369,6 +376,15 @@ class BaseNode(ABC, BaseModel):
             Dict[str, Any]: _description_
         """
 
+    @abstractmethod
+    def to_d3_node(self) -> NodeInD3:
+        """
+        Convert the node to a D3 node representation.
+
+        Returns:
+            NodeInD3: The D3 node representation of the current node.
+        """
+
 
 # --8<-- [end:docs]
 class TraversalNode(BaseNode):

runnable/parameters.py

@@ -1,10 +1,10 @@
+import argparse
 import inspect
 import json
 import logging
 import os
 from typing import Any, Dict, Type
 
-import pydantic
 from pydantic import BaseModel, ConfigDict
 from typing_extensions import Callable
 
@@ -48,6 +48,25 @@ def get_user_set_parameters(remove: bool = False) -> Dict[str, JsonParameter]:
     return parameters
 
 
+def return_json_parameters(params: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Returns the parameters as a JSON serializable dictionary.
+
+    Args:
+        params (dict): The parameters to serialize.
+
+    Returns:
+        dict: The JSON serializable dictionary.
+    """
+    return_params = {}
+    for key, value in params.items():
+        if isinstance(value, ObjectParameter):
+            continue
+
+        return_params[key] = value.get_value()
+    return return_params
+
+
 def filter_arguments_for_func(
     func: Callable[..., Any],
     params: Dict[str, Any],
@@ -55,8 +74,14 @@ def filter_arguments_for_func(
 ) -> Dict[str, Any]:
     """
     Inspects the function to be called as part of the pipeline to find the arguments of the function.
-    Matches the function arguments to the parameters available either by command line or by up stream steps.
+    Matches the function arguments to the parameters available either by static parameters or by up stream steps.
 
+    The function "func" signature could be:
+    - def my_function(arg1: int, arg2: str, arg3: float):
+    - def my_function(arg1: int, arg2: str, arg3: float, **kwargs):
+        in this case, we would need to send in remaining keyword arguments as a dictionary.
+    - def my_function(arg1: int, arg2: str, arg3: float, args: argparse.Namespace):
+        In this case, we need to send the rest of the parameters as attributes of the args object.
 
     Args:
         func (Callable): The function to inspect
@@ -72,63 +97,109 @@ def filter_arguments_for_func(
         params[key] = JsonParameter(kind="json", value=v)
 
     bound_args = {}
-    unassigned_params = set(params.keys())
-    # Check if VAR_KEYWORD is used, it is we send back everything
-    # If **kwargs is present in the function signature, we send back everything
-    for name, value in function_args.items():
-        if value.kind != inspect.Parameter.VAR_KEYWORD:
-            continue
-        # Found VAR_KEYWORD, we send back everything as found
-        for key, value in params.items():
-            bound_args[key] = params[key].get_value()
+    missing_required_args: list[str] = []
+    var_keyword_param = None
+    namespace_param = None
 
-        return bound_args
-
-    # Lets return what is asked for then!!
+    # First pass: Handle regular parameters and identify special parameters
     for name, value in function_args.items():
         # Ignore any *args
         if value.kind == inspect.Parameter.VAR_POSITIONAL:
             logger.warning(f"Ignoring parameter {name} as it is VAR_POSITIONAL")
             continue
 
-        if name not in params:
-            # No parameter of this name was provided
-            if value.default == inspect.Parameter.empty:
-                # No default value is given in the function signature. error as parameter is required.
-                raise ValueError(
-                    f"Parameter {name} is required for {func.__name__} but not provided"
-                )
-            # default value is given in the function signature, nothing further to do.
+        # Check for **kwargs parameter
+        if value.kind == inspect.Parameter.VAR_KEYWORD:
+            var_keyword_param = name
             continue
 
-        param_value = params[name]
-
-        if type(value.annotation) in [
-            BaseModel,
-            pydantic._internal._model_construction.ModelMetaclass,
-        ] and not isinstance(param_value, ObjectParameter):
-            # Even if the annotation is a pydantic model, it can be passed as an object parameter
-            # We try to cast it as a pydantic model if asked
-            named_param = params[name].get_value()
-
-            if not isinstance(named_param, dict):
-                # A case where the parameter is a one attribute model
-                named_param = {name: named_param}
-
-            bound_model = bind_args_for_pydantic_model(named_param, value.annotation)
-            bound_args[name] = bound_model
+        # Check for argparse.Namespace parameter
+        if value.annotation == argparse.Namespace:
+            namespace_param = name
+            continue
 
-        elif value.annotation in [str, int, float, bool]:
-            # Cast it if its a primitive type. Ensure the type matches the annotation.
-            bound_args[name] = value.annotation(params[name].get_value())
+        # Handle regular parameters
+        if name not in params:
+            if value.default != inspect.Parameter.empty:
+                # Default value is given in the function signature, we can use it
+                bound_args[name] = value.default
+            else:
+                # This is a required parameter that's missing
+                missing_required_args.append(name)
         else:
-            bound_args[name] = params[name].get_value()
-
-        unassigned_params.remove(name)
-
-        params = {
-            key: params[key] for key in unassigned_params
-        }  # remove keys from params if they are assigned
+            # We have a parameter of this name, lets bind it
+            param_value = params[name]
+
+            if (
+                inspect.isclass(value.annotation)
+                and issubclass(value.annotation, BaseModel)
+            ) and not isinstance(param_value, ObjectParameter):
+                # Even if the annotation is a pydantic model, it can be passed as an object parameter
+                # We try to cast it as a pydantic model if asked
+                named_param = params[name].get_value()
+
+                if not isinstance(named_param, dict):
+                    # A case where the parameter is a one attribute model
+                    named_param = {name: named_param}
+
+                bound_model = bind_args_for_pydantic_model(
+                    named_param, value.annotation
+                )
+                bound_args[name] = bound_model
+
+            elif value.annotation in [str, int, float, bool] and callable(
+                value.annotation
+            ):
+                # Cast it if its a primitive type. Ensure the type matches the annotation.
+                try:
+                    bound_args[name] = value.annotation(params[name].get_value())
+                except (ValueError, TypeError) as e:
+                    raise ValueError(
+                        f"Cannot cast parameter '{name}' to {value.annotation.__name__}: {e}"
+                    )
+            else:
+                # We do not know type of parameter, we send the value as found
+                bound_args[name] = params[name].get_value()
+
+    # Find extra parameters (parameters in params but not consumed by regular function parameters)
+    consumed_param_names = set(bound_args.keys()) | set(missing_required_args)
+    extra_params = {k: v for k, v in params.items() if k not in consumed_param_names}
+
+    # Second pass: Handle **kwargs and argparse.Namespace parameters
+    if var_keyword_param is not None:
+        # Function accepts **kwargs - add all extra parameters directly to bound_args
+        for param_name, param_value in extra_params.items():
+            bound_args[param_name] = param_value.get_value()
+    elif namespace_param is not None:
+        # Function accepts argparse.Namespace - create namespace with extra parameters
+        args_namespace = argparse.Namespace()
+        for param_name, param_value in extra_params.items():
+            setattr(args_namespace, param_name, param_value.get_value())
+        bound_args[namespace_param] = args_namespace
+    elif extra_params:
+        # Function doesn't accept **kwargs or namespace, but we have extra parameters
+        # This should only be an error if we also have missing required parameters
+        # or if the function truly can't handle the extra parameters
+        if missing_required_args:
+            # We have both missing required and extra parameters - this is an error
+            raise ValueError(
+                f"Function {func.__name__} has parameters {missing_required_args} that are not present in the parameters"
+            )
+        # If we only have extra parameters and no missing required ones, we just ignore the extras
+        # This allows for more flexible parameter passing
+
+    # Check for missing required parameters
+    if missing_required_args:
+        if var_keyword_param is None and namespace_param is None:
+            # No way to handle missing parameters
+            raise ValueError(
+                f"Function {func.__name__} has parameters {missing_required_args} that are not present in the parameters"
+            )
+        # If we have **kwargs or namespace, missing parameters might be handled there
+        # But if they're truly required (no default), we should still error
+        raise ValueError(
+            f"Function {func.__name__} has parameters {missing_required_args} that are not present in the parameters"
+        )
 
     return bound_args
 

runnable/tasks.py

@@ -209,6 +209,9 @@ class BaseTaskType(BaseModel):
                 parameters=diff_parameters, run_id=self._context.run_id
             )
 
+    def get_d3_metadata(self) -> dict[str, str]:
+        raise NotImplementedError
+
 
 def task_return_to_parameter(task_return: TaskReturns, value: Any) -> Parameter:
     # implicit support for pydantic models
@@ -287,6 +290,72 @@ class PythonTaskType(BaseTaskType):  # pylint: disable=too-few-public-methods
     task_type: str = Field(default="python", serialization_alias="command_type")
     command: str
 
+    def get_d3_metadata(self) -> Dict[str, str]:
+        module, func = utils.get_module_and_attr_names(self.command)
+
+        # Import inspect module to get function signature
+        import inspect
+
+        def format_type_annotation(annotation):
+            """Format type annotation in a more readable way"""
+            if annotation == inspect._empty:
+                return "Any"
+            elif hasattr(annotation, "__name__"):
+                return annotation.__name__
+            elif hasattr(annotation, "__origin__"):
+                # Handle typing types like List, Dict, etc.
+                origin = (
+                    annotation.__origin__.__name__
+                    if hasattr(annotation.__origin__, "__name__")
+                    else str(annotation.__origin__)
+                )
+                args = ", ".join(
+                    format_type_annotation(arg) for arg in annotation.__args__
+                )
+                return f"{origin}[{args}]"
+            else:
+                # Fall back to string representation without 'typing.'
+                return str(annotation).replace("typing.", "")
+
+        # Import the module and get the function
+        sys.path.insert(0, os.getcwd())
+        try:
+            imported_module = importlib.import_module(module)
+            f = getattr(imported_module, func)
+
+            # Get function signature
+            sig = inspect.signature(f)
+
+            # Format parameters with type annotations
+            params_list = []
+            for param_name, param in sig.parameters.items():
+                type_annotation = format_type_annotation(param.annotation)
+                params_list.append(f"{param_name}: {type_annotation}")
+
+            params_str = ", ".join(params_list)
+
+            # Format returns based on self.returns or use "None" if no returns specified
+            if self.returns:
+                returns_str = ", ".join([f"{r.name}({r.kind})" for r in self.returns])
+            else:
+                returns_str = "None"
+
+            # Format function signature
+            signature = f"def {func}({params_str}) -> {returns_str}:"
+
+            return {
+                "module": module,
+                "function": func,
+                "signature": signature,
+            }
+        except Exception as e:
+            logger.warning(f"Could not inspect function {self.command}: {str(e)}")
+            return {
+                "module": module,
+                "function": func,
+                "signature": f"def {func}(...) -> ...",
+            }
+
     def execute_command(
         self,
         map_variable: MapVariableType = None,
@@ -440,6 +509,11 @@ class NotebookTaskType(BaseTaskType):
     command: str
     optional_ploomber_args: dict = {}
 
+    def get_d3_metadata(self) -> Dict[str, str]:
+        return {
+            "notebook": self.command,
+        }
+
     @field_validator("command")
     @classmethod
     def notebook_should_end_with_ipynb(cls, command: str) -> str:
@@ -640,6 +714,11 @@ class ShellTaskType(BaseTaskType):
     task_type: str = Field(default="shell", serialization_alias="command_type")
     command: str
 
+    def get_d3_metadata(self) -> Dict[str, str]:
+        return {
+            "command": self.command[:50],
+        }
+
     @field_validator("returns")
     @classmethod
     def returns_should_be_json(cls, returns: List[TaskReturns]):

{runnable-0.37.0.dist-info → runnable-0.39.0.dist-info}/METADATA

@@ -1,12 +1,10 @@
 Metadata-Version: 2.4
 Name: runnable
-Version: 0.37.0
+Version: 0.39.0
 Summary: Add your description here
 Author-email: "Vammi, Vijay" <vijay.vammi@astrazeneca.com>
 License-File: LICENSE
 Requires-Python: >=3.10
-Requires-Dist: click-plugins>=1.1.1
-Requires-Dist: click<=8.1.3
 Requires-Dist: cloudpathlib>=0.20.0
 Requires-Dist: dill>=0.3.9
 Requires-Dist: pydantic>=2.10.3
@@ -15,17 +13,24 @@ Requires-Dist: rich>=13.9.4
 Requires-Dist: ruamel-yaml>=0.18.6
 Requires-Dist: setuptools>=75.6.0
 Requires-Dist: stevedore>=5.4.0
-Requires-Dist: typer>=0.15.1
+Requires-Dist: typer>=0.17.3
 Provides-Extra: docker
 Requires-Dist: docker>=7.1.0; extra == 'docker'
 Provides-Extra: examples
 Requires-Dist: pandas>=2.2.3; extra == 'examples'
+Provides-Extra: examples-torch
+Requires-Dist: torch>=2.7.1; extra == 'examples-torch'
 Provides-Extra: k8s
 Requires-Dist: kubernetes>=31.0.0; extra == 'k8s'
 Provides-Extra: notebook
 Requires-Dist: ploomber-engine>=0.0.33; extra == 'notebook'
 Provides-Extra: s3
 Requires-Dist: cloudpathlib[s3]; extra == 's3'
+Provides-Extra: ui
+Requires-Dist: fastapi>=0.95.0; extra == 'ui'
+Requires-Dist: jinja2>=3.1.2; extra == 'ui'
+Requires-Dist: python-multipart>=0.0.5; extra == 'ui'
+Requires-Dist: uvicorn>=0.22.0; extra == 'ui'
 Description-Content-Type: text/markdown
 
 

{runnable-0.37.0.dist-info → runnable-0.39.0.dist-info}/RECORD

@@ -16,14 +16,14 @@ extensions/job_executor/local_container.py,sha256=uoRbsyR5QiVytWJQtF7nXUA1h8wHAa
 extensions/job_executor/pyproject.toml,sha256=UIEgiCYHTXcRWSByNMFuKJFKgxTBpQqTqyUecIsb_Vc,286
 extensions/nodes/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 extensions/nodes/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-extensions/nodes/conditional.py,sha256=lsXLerIDWJk0fHo4EEngl8rYw6dN40EgKIgdBuvt8ro,8641
-extensions/nodes/fail.py,sha256=P_wryruE2INiAmu4Vf7q4fTEkfbIMnhT71Y9IrOqRkM,2170
-extensions/nodes/map.py,sha256=s0l07uAc6j8MO7Fqch0DzOE3_CsV6pBNHj5E3nf1S60,13659
-extensions/nodes/parallel.py,sha256=cuxpR1e0KwuxvoZaSwiiEn9YmYskplbjLBWBzzU-Ti8,5952
+extensions/nodes/conditional.py,sha256=5wDMrnkXu-cSxaELpHTHzFCebJ39pkVkq3Kzd5MrSWI,9245
+extensions/nodes/fail.py,sha256=wkJW1PUgWvs6PP1J84sk-ed-DxLpL7XS8ltzAazgi3o,2315
+extensions/nodes/map.py,sha256=_hLjkxVkf9dEz6t5k9DgAt6V7-rZwYCyM0sVX4V4l7o,14278
+extensions/nodes/parallel.py,sha256=IMbjMFZcRshIXdVrX1K9QttPBUKTssxLvZDgCSU5-Mo,6101
 extensions/nodes/pyproject.toml,sha256=YTu-ETN3JNFSkMzzWeOwn4m-O2nbRH-PmiPBALDCUw4,278
-extensions/nodes/stub.py,sha256=o9DjBekNa9O4b0VQOiNOA9eNjJ3C2a9Sn9d2fX7KaWg,2715
-extensions/nodes/success.py,sha256=yT4WkUI-1YN6qy4Ji6zSoZFXI9jOl6Ond4WxZRq-33k,2179
-extensions/nodes/task.py,sha256=m5_s5u7lQDoTmeO6CG94btrG5If9AKm2-RvOzvm0fDc,3147
+extensions/nodes/stub.py,sha256=TQPeqHkAAIQBI0hj_UdsloebyYDNrH1PadezExGgXEg,2860
+extensions/nodes/success.py,sha256=XA5UOLmlhNronuhcS0Ct5bGxJjcgBjPTZ4sdako7FwI,2327
+extensions/nodes/task.py,sha256=rfOGdsmgchBKQQy7tpaf7ul9H4r1WeY1zYi6MCSB_Ys,3565
 extensions/pipeline_executor/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 extensions/pipeline_executor/__init__.py,sha256=JqJFGgLOUaDINUi4_xC3zoDSZxDrUYLNuHQVzr8-ByE,24620
 extensions/pipeline_executor/argo.py,sha256=5nEItAv-gNVi2sylZA4spKLWfcKuJ3zyQx3RekoLb28,38240
@@ -49,24 +49,24 @@ extensions/secrets/dotenv.py,sha256=nADHXI6KJ_LUYOIe5EbtYH-21OBebSNVr0Pjb1GlZ7w,
 extensions/secrets/pyproject.toml,sha256=mLJNImNcBlbLKHh-0ugVWT9V83R4RibyyYDtBCSqVF4,282
 runnable/__init__.py,sha256=MN9x2jmQb2eOr-rap1DXLzNSC926U-aad_YwENzG52w,509
 runnable/catalog.py,sha256=6l0tT0jwHi40aE6fhQMgYtYe_-2As-bRKztAKiFvy3o,3842
-runnable/cli.py,sha256=CziCKBoL-dklSEo_x-YO1AysrG2eaf2LMQZbcNTeCbM,7283
+runnable/cli.py,sha256=TlSjQ1wVlVZjyGqc-99ibAARjr1_y1FQcjyRrJXFmNc,8459
 runnable/context.py,sha256=mLpq5rtMsPawjnaN9Woq7HWZ1FAppeudZtYMT5vf6Fo,17594
 runnable/datastore.py,sha256=2pYg4i1JRMzw_CUUIsPOWt7wYPiGBamfo-CPVAkEH54,32375
 runnable/defaults.py,sha256=4UYuShnjEyWP529UlFnubvkBpOcczKIdE4jEOhPBwl4,3076
 runnable/entrypoints.py,sha256=46prgr3_FYtBMlRbUXIDSpgZUBgaxcdJAekXhgEIj7M,6578
 runnable/exceptions.py,sha256=t5tSlYqe_EjU5liXu32yLLh_yrnXeFL93BuXfmQzV98,3268
 runnable/executor.py,sha256=CwzHkeGVpocACZLzfFS94TzKeiaPLv4NtXtvT3eoocY,15222
-runnable/graph.py,sha256=poQz5zcvq89ju_u5sYlunQLPbHnXTaUmjcvstPwvT4U,16536
+runnable/graph.py,sha256=KfT5nXSCRwVyKytG88h52xeu6kEEM3uytFftG_16Vt4,28666
 runnable/names.py,sha256=A9ldUyULXuWjJ1MoXihHqlg-xeTVX-oWYTO5Ah0trmo,8128
-runnable/nodes.py,sha256=JHBxJib7SSQXY51bLHBXUvb0DlNSLNvyqz3JNEDLt8c,16926
-runnable/parameters.py,sha256=zEehAliVvCOLOnNZ4ExJvSDJM_2PWY0URZ0bmZUgCQA,5289
+runnable/nodes.py,sha256=-IEJKd9RP6SswMjiTaRh2SXqGsGWa_0NeGoQICUx6bs,17289
+runnable/parameters.py,sha256=HZW0bhAYxgMyvRZzUlwp29MVxmzoFU5ZoVJMJHnOcX8,8734
 runnable/pickler.py,sha256=ydJ_eti_U1F4l-YacFp7BWm6g5vTn04UXye25S1HVok,2684
 runnable/sdk.py,sha256=blLBWzXV2x7jxKQXWpjmeJ9k22jt5CKBQBqQpnt4agk,32587
 runnable/secrets.py,sha256=4L_dBFxTgr8r_hHUD6RlZEtqaOHDRsFG5PXO5wlvMI0,2324
-runnable/tasks.py,sha256=7yuoeG4ZqfxFUmN4mPS4i6kbQmzEpAwbPQweAUWY-ic,31366
+runnable/tasks.py,sha256=dKUFE3aJWcBg-aH9xdyyT4DaDBVa1IyBtEQK6LTcoRE,34234
 runnable/utils.py,sha256=amHW3KR_NGTDysGHcSafhh5WJUX7GPBSxqdPyzAIhao,11350
-runnable-0.37.0.dist-info/METADATA,sha256=nD9ezWOwkWw3Yi5_NBE_xqjRGKxqf_FnSsYb8P9oqxo,10047
-runnable-0.37.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-runnable-0.37.0.dist-info/entry_points.txt,sha256=KkxihZ0LLEiwvFl7RquyqZ0tp2fJDIs7DgzHYDlmc3U,2018
-runnable-0.37.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-runnable-0.37.0.dist-info/RECORD,,
+runnable-0.39.0.dist-info/METADATA,sha256=l9vd673bcqL7Dgqz99MzXeBOOsi7_ilqNRMnSE3sMEU,10278
+runnable-0.39.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+runnable-0.39.0.dist-info/entry_points.txt,sha256=KkxihZ0LLEiwvFl7RquyqZ0tp2fJDIs7DgzHYDlmc3U,2018
+runnable-0.39.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+runnable-0.39.0.dist-info/RECORD,,