runnable 0.38.0__tar.gz → 0.39.0__tar.gz

{runnable-0.38.0 → runnable-0.39.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: runnable
-Version: 0.38.0
+Version: 0.39.0
 Summary: Add your description here
 Author-email: "Vammi, Vijay" <vijay.vammi@astrazeneca.com>
 License-File: LICENSE
@@ -18,12 +18,19 @@ Provides-Extra: docker
 Requires-Dist: docker>=7.1.0; extra == 'docker'
 Provides-Extra: examples
 Requires-Dist: pandas>=2.2.3; extra == 'examples'
+Provides-Extra: examples-torch
+Requires-Dist: torch>=2.7.1; extra == 'examples-torch'
 Provides-Extra: k8s
 Requires-Dist: kubernetes>=31.0.0; extra == 'k8s'
 Provides-Extra: notebook
 Requires-Dist: ploomber-engine>=0.0.33; extra == 'notebook'
 Provides-Extra: s3
 Requires-Dist: cloudpathlib[s3]; extra == 's3'
+Provides-Extra: ui
+Requires-Dist: fastapi>=0.95.0; extra == 'ui'
+Requires-Dist: jinja2>=3.1.2; extra == 'ui'
+Requires-Dist: python-multipart>=0.0.5; extra == 'ui'
+Requires-Dist: uvicorn>=0.22.0; extra == 'ui'
 Description-Content-Type: text/markdown
 
 

{runnable-0.38.0 → runnable-0.39.0}/extensions/nodes/conditional.py

@@ -7,7 +7,7 @@ from pydantic import Field, field_serializer, field_validator
 from runnable import console, defaults
 from runnable.datastore import Parameter
 from runnable.graph import Graph, create_graph
-from runnable.nodes import CompositeNode, MapVariableType
+from runnable.nodes import CompositeNode, MapVariableType, NodeInD3
 
 logger = logging.getLogger(defaults.LOGGER_NAME)
 
@@ -241,3 +241,21 @@ class ConditionalNode(CompositeNode):
             step_log.status = defaults.FAIL
 
         self._context.run_log_store.add_step_log(step_log, self._context.run_id)
+
+    def to_d3_node(self) -> NodeInD3:
+        def get_display_string() -> str:
+            display = f"match {self.parameter}:\n"
+            for case in self.branches.keys():
+                display += f'    case "{case}":\n        ...\n'
+            if self.default:
+                display += "    case _:\n        ...\n"
+            return display
+
+        return NodeInD3(
+            id=self.internal_name,
+            label="conditional",
+            metadata={
+                "conditioned on": self.parameter,
+                "display": get_display_string(),
+            },
+        )

{runnable-0.38.0 → runnable-0.39.0}/extensions/nodes/fail.py

@@ -6,7 +6,7 @@ from pydantic import Field
 from runnable import datastore, defaults
 from runnable.datastore import StepLog
 from runnable.defaults import MapVariableType
-from runnable.nodes import TerminalNode
+from runnable.nodes import NodeInD3, TerminalNode
 
 
 class FailNode(TerminalNode):
@@ -70,3 +70,9 @@ class FailNode(TerminalNode):
         step_log.attempts.append(attempt_log)
 
         return step_log
+
+    def to_d3_node(self) -> NodeInD3:
+        return NodeInD3(
+            id=self.internal_name,
+            label="fail",
+        )

{runnable-0.38.0 → runnable-0.39.0}/extensions/nodes/map.py

@@ -18,7 +18,7 @@ from runnable.datastore import (
 )
 from runnable.defaults import MapVariableType
 from runnable.graph import Graph, create_graph
-from runnable.nodes import CompositeNode
+from runnable.nodes import CompositeNode, NodeInD3
 
 logger = logging.getLogger(defaults.LOGGER_NAME)
 
@@ -348,3 +348,18 @@ class MapNode(CompositeNode):
         self._context.run_log_store.set_parameters(
             parameters=params, run_id=self._context.run_id
         )
+
+    def to_d3_node(self) -> NodeInD3:
+        return NodeInD3(
+            id=self.internal_name,
+            label="map",
+            metadata={
+                "node_type": "map",
+                "iterate_on": self.iterate_on,  # Parameter name containing the iterable
+                "iterate_as": self.iterate_as,  # Name used for each iteration
+                "map_branch_id": self.internal_name
+                + "."
+                + defaults.MAP_PLACEHOLDER,  # The branch identifier pattern
+                "is_composite": True,  # Flag indicating this is a composite node
+            },
+        )

{runnable-0.38.0 → runnable-0.39.0}/extensions/nodes/parallel.py

@@ -6,7 +6,7 @@ from pydantic import Field, field_serializer
 from runnable import defaults
 from runnable.defaults import MapVariableType
 from runnable.graph import Graph, create_graph
-from runnable.nodes import CompositeNode
+from runnable.nodes import CompositeNode, NodeInD3
 
 
 class ParallelNode(CompositeNode):
@@ -157,3 +157,9 @@ class ParallelNode(CompositeNode):
             step_log.status = defaults.FAIL
 
         self._context.run_log_store.add_step_log(step_log, self._context.run_id)
+
+    def to_d3_node(self) -> NodeInD3:
+        return NodeInD3(
+            id=self.internal_name,
+            label="parallel",
+        )

{runnable-0.38.0 → runnable-0.39.0}/extensions/nodes/stub.py

@@ -7,7 +7,7 @@ from pydantic import ConfigDict, Field
 from runnable import datastore, defaults
 from runnable.datastore import StepLog
 from runnable.defaults import MapVariableType
-from runnable.nodes import ExecutableNode
+from runnable.nodes import ExecutableNode, NodeInD3
 
 logger = logging.getLogger(defaults.LOGGER_NAME)
 
@@ -87,3 +87,9 @@ class StubNode(ExecutableNode):
         step_log.attempts.append(attempt_log)
 
         return step_log
+
+    def to_d3_node(self) -> NodeInD3:
+        return NodeInD3(
+            id=self.internal_name,
+            label="stub",
+        )

{runnable-0.38.0 → runnable-0.39.0}/extensions/nodes/success.py

@@ -6,7 +6,7 @@ from pydantic import Field
 from runnable import datastore, defaults
 from runnable.datastore import StepLog
 from runnable.defaults import MapVariableType
-from runnable.nodes import TerminalNode
+from runnable.nodes import NodeInD3, TerminalNode
 
 
 class SuccessNode(TerminalNode):
@@ -70,3 +70,9 @@ class SuccessNode(TerminalNode):
         step_log.attempts.append(attempt_log)
 
         return step_log
+
+    def to_d3_node(self) -> NodeInD3:
+        return NodeInD3(
+            id=self.internal_name,
+            label="success",
+        )

{runnable-0.38.0 → runnable-0.39.0}/extensions/nodes/task.py

@@ -7,7 +7,7 @@ from pydantic import ConfigDict, Field
 from runnable import datastore, defaults
 from runnable.datastore import StepLog
 from runnable.defaults import MapVariableType
-from runnable.nodes import ExecutableNode
+from runnable.nodes import ExecutableNode, NodeInD3
 from runnable.tasks import BaseTaskType, create_task
 
 logger = logging.getLogger(defaults.LOGGER_NAME)
@@ -90,3 +90,17 @@ class TaskNode(ExecutableNode):
         step_log.attempts.append(attempt_log)
 
         return step_log
+
+    def to_d3_node(self) -> NodeInD3:
+        """
+        Convert the task node to a D3 node representation.
+
+        Returns:
+            NodeInD3: The D3 node representation of the task node.
+        """
+        return NodeInD3(
+            id=self.internal_name,
+            label="task",
+            task_type=self.executable.task_type,
+            metadata=self.executable.get_d3_metadata(),
+        )

{runnable-0.38.0 → runnable-0.39.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "runnable"
-version = "0.38.0"
+version = "0.39.0"
 description = "Add your description here"
 readme = "README.md"
 authors = [
@@ -35,6 +35,15 @@ k8s = [
 s3 = [
     "cloudpathlib[s3]"
 ]
+examples-torch = [
+    "torch>=2.7.1",
+]
+ui = [
+    "fastapi>=0.95.0",
+    "jinja2>=3.1.2",
+    "uvicorn>=0.22.0",
+    "python-multipart>=0.0.5",
+]
 
 
 [dependency-groups]
@@ -56,9 +65,7 @@ docs = [
 release = [
     "python-semantic-release>=9.15.2",
 ]
-examples-torch = [
-    "torch>=2.7.1",
-]
+
 
 [tool.uv.workspace]
 members = ["extensions/catalog",
@@ -67,6 +74,7 @@ members = ["extensions/catalog",
         "extensions/pipeline_executor",
         "extensions/run_log_store",
         "extensions/secrets",
+        "visualization"
 ]
 
 [tool.uv.sources]
@@ -76,6 +84,7 @@ catalog = {workspace = true}
 run_log_store = {workspace = true}
 pipeline_executor = {workspace = true}
 job_executor = {workspace = true}
+visualization = {workspace = true}
 
 
 [project.scripts]

{runnable-0.38.0 → runnable-0.39.0}/runnable/cli.py

@@ -274,5 +274,57 @@ def execute_job(
     )
 
 
+@app.command()
+def ui(
+    host: Annotated[
+        str,
+        typer.Option(
+            "--host",
+            "-h",
+            help="The host to bind the server to",
+        ),
+    ] = "127.0.0.1",
+    port: Annotated[
+        int,
+        typer.Option(
+            "--port",
+            "-p",
+            help="The port to bind the server to",
+        ),
+    ] = 8000,
+    reload: Annotated[
+        bool,
+        typer.Option(
+            "--reload",
+            help="Enable auto-reload for development",
+        ),
+    ] = False,
+):
+    """
+    Start the web UI for pipeline visualization.
+
+    This command starts a FastAPI web server that provides a user interface
+    for visualizing and exploring runnable pipelines.
+    """
+    try:
+        import uvicorn
+
+        from visualization.main import app as web_app
+    except ImportError:
+        typer.echo(
+            "UI dependencies not installed. Install with: pip install runnable[ui]",
+            err=True,
+        )
+        raise typer.Exit(1)
+
+    typer.echo(f"Starting web UI at http://{host}:{port}")
+    uvicorn.run(
+        web_app,
+        host=host,
+        port=port,
+        reload=reload,
+    )
+
+
 if __name__ == "__main__":
     app()

{runnable-0.38.0 → runnable-0.39.0}/runnable/graph.py

@@ -499,3 +499,274 @@ def search_branch_by_internal_name(dag: Graph, internal_name: str):
         return current_branch
 
     raise exceptions.BranchNotFoundError(internal_name)
+
+
+def get_visualization_data(graph: Graph) -> Dict[str, Any]:
+    """
+    Convert the graph into a D3 visualization friendly format with nodes and links.
+    Handles composite nodes (parallel, map, conditional) by recursively processing their embedded graphs.
+
+    Args:
+        graph: The Graph object to convert
+
+    Returns:
+        Dict with two keys:
+        - nodes: List of node objects with id, type, name, and alias
+        - links: List of edge objects with source and target node ids
+    """
+    import rich.console
+
+    from extensions.nodes.conditional import ConditionalNode
+    from extensions.nodes.map import MapNode
+    from extensions.nodes.parallel import ParallelNode
+    from runnable.nodes import ExecutableNode
+
+    rich_print = rich.console.Console().print
+
+    rich_print(graph)
+
+    nodes = []
+    links = []
+    processed_nodes = set()
+
+    def process_node(
+        node: BaseNode,
+        parent_id: Optional[str] = None,
+        current_graph: Graph = graph,
+        map_node_id: Optional[str] = None,
+        conditional_node_id: Optional[str] = None,
+    ) -> str:
+        node_id = f"{node.internal_name}"
+        node_alias = node.name  # Alias based on the node's name
+
+        if node_id not in processed_nodes:
+            node_data = node.to_d3_node().model_dump(exclude_none=True)
+            node_data["alias"] = node_alias  # Add alias to the node data
+            node_data["display_name"] = node_alias  # Use alias as the display name
+
+            # Add map or parallel related metadata if this node is part of a map branch or parallel branch
+            if map_node_id:
+                if "metadata" not in node_data:
+                    node_data["metadata"] = {}
+
+                # Mark this node as being part of a map branch
+                node_data["metadata"]["belongs_to_node"] = map_node_id
+
+                # If this is the map node itself, add a special attribute
+                if node_id == map_node_id:
+                    node_data["metadata"]["is_map_root"] = True
+
+            # Add conditional related metadata if this node is part of a conditional branch
+            if conditional_node_id:
+                if "metadata" not in node_data:
+                    node_data["metadata"] = {}
+
+                # Mark this node as being part of a conditional branch
+                node_data["metadata"]["belongs_to_node"] = conditional_node_id
+
+                # If this is the conditional node itself, add a special attribute
+                if node_id == conditional_node_id:
+                    node_data["metadata"]["is_conditional_root"] = True
+
+            # Mark parallel nodes with special metadata
+            if isinstance(node, ParallelNode):
+                if "metadata" not in node_data:
+                    node_data["metadata"] = {}
+
+                # Add parallel node type to metadata
+                node_data["metadata"]["node_type"] = "parallel"
+                node_data["metadata"]["parallel_branch_id"] = node_id
+
+            # Mark conditional nodes with special metadata
+            if isinstance(node, ConditionalNode):
+                if "metadata" not in node_data:
+                    node_data["metadata"] = {}
+
+                # Add conditional node type to metadata
+                node_data["metadata"]["node_type"] = "conditional"
+                node_data["metadata"]["conditional_branch_id"] = node_id
+
+            nodes.append(node_data)
+            processed_nodes.add(node_id)
+
+            # Add link from parent if it exists
+            if parent_id:
+                links.append({"source": parent_id, "target": node_id})
+
+            # Handle composite nodes with embedded graphs
+            if isinstance(node, (ParallelNode, MapNode, ConditionalNode)):
+                if isinstance(node, ParallelNode):
+                    # Process each parallel branch
+                    for _, branch in node.branches.items():
+                        branch_start = branch.get_node_by_name(branch.start_at)
+                        process_node(
+                            branch_start,
+                            node_id,
+                            branch,
+                            map_node_id=node_id,
+                            conditional_node_id=conditional_node_id,
+                        )
+
+                    # Handle next node connection after parallel branches complete
+                    if hasattr(node, "next_node") and node.next_node:
+                        try:
+                            next_node = current_graph.get_node_by_name(node.next_node)
+                            next_id = process_node(
+                                next_node,
+                                None,
+                                current_graph=current_graph,
+                                map_node_id=map_node_id,
+                                conditional_node_id=conditional_node_id,
+                            )
+                            links.append(
+                                {
+                                    "source": node_id,
+                                    "target": next_id,
+                                    "type": "success",
+                                }
+                            )
+                        except exceptions.NodeNotFoundError as e:
+                            rich_print(
+                                f"Warning: Next node '{node.next_node}' not found for parallel node '{node.name}': {e}"
+                            )
+
+                elif isinstance(node, MapNode):
+                    # Process map branch
+                    branch_start = node.branch.get_node_by_name(node.branch.start_at)
+                    # Process the branch with additional context about the map node
+                    process_node(
+                        branch_start,
+                        node_id,
+                        node.branch,
+                        map_node_id=node_id,
+                        conditional_node_id=conditional_node_id,
+                    )
+
+                elif isinstance(node, ConditionalNode):
+                    # Process each conditional branch
+                    for _, branch in node.branches.items():
+                        branch_start = branch.get_node_by_name(branch.start_at)
+                        process_node(
+                            branch_start,
+                            node_id,
+                            branch,
+                            map_node_id=map_node_id,
+                            conditional_node_id=node_id,
+                        )
+                    if node.default:
+                        default_start = node.default.get_node_by_name(
+                            node.default.start_at
+                        )
+                        process_node(
+                            default_start,
+                            node_id,
+                            node.default,
+                            map_node_id=map_node_id,
+                            conditional_node_id=node_id,
+                        )
+
+                    # Handle next node connection after conditional branches complete
+                    if hasattr(node, "next_node") and node.next_node:
+                        try:
+                            next_node = current_graph.get_node_by_name(node.next_node)
+                            next_id = process_node(
+                                next_node,
+                                None,
+                                current_graph=current_graph,
+                                map_node_id=map_node_id,
+                                conditional_node_id=conditional_node_id,
+                            )
+                            links.append(
+                                {
+                                    "source": node_id,
+                                    "target": next_id,
+                                    "type": "success",
+                                }
+                            )
+                        except exceptions.NodeNotFoundError as e:
+                            rich_print(
+                                f"Warning: Next node '{node.next_node}' not found for conditional node '{node.name}': {e}"
+                            )
+
+            # Add links to next and on_failure nodes if they exist
+            if isinstance(node, ExecutableNode):
+                # Handle normal "next" links (success path)
+                if hasattr(node, "next_node") and node.next_node:
+                    try:
+                        next_node = current_graph.get_node_by_name(node.next_node)
+                        next_id = process_node(
+                            next_node,
+                            None,
+                            current_graph=current_graph,
+                            map_node_id=map_node_id,
+                            conditional_node_id=conditional_node_id,
+                        )
+                        links.append(
+                            {"source": node_id, "target": next_id, "type": "success"}
+                        )
+                    except exceptions.NodeNotFoundError as e:
+                        rich_print(
+                            f"Warning: Next node '{node.next_node}' not found for node '{node.name}': {e}"
+                        )
+
+                # Handle on_failure links (failure path)
+                if hasattr(node, "on_failure") and node.on_failure:
+                    try:
+                        failure_node = current_graph.get_node_by_name(node.on_failure)
+                        failure_id = process_node(
+                            failure_node,
+                            None,
+                            current_graph=current_graph,
+                            map_node_id=map_node_id,
+                            conditional_node_id=conditional_node_id,
+                        )
+                        links.append(
+                            {"source": node_id, "target": failure_id, "type": "failure"}
+                        )
+                    except exceptions.NodeNotFoundError as e:
+                        rich_print(
+                            f"Warning: On-failure node '{node.on_failure}' not found for node '{node.name}': {e}"
+                        )
+
+                # For backward compatibility, also process all neighbors
+                # This handles cases where node might have other connection types
+                next_nodes = node._get_neighbors()
+                for next_node_name in next_nodes:
+                    # Skip nodes we've already handled explicitly
+                    if (
+                        hasattr(node, "next_node") and node.next_node == next_node_name
+                    ) or (
+                        hasattr(node, "on_failure")
+                        and node.on_failure == next_node_name
+                    ):
+                        continue
+
+                    try:
+                        next_node = current_graph.get_node_by_name(next_node_name)
+                        next_id = process_node(
+                            next_node,
+                            None,
+                            current_graph=current_graph,
+                            map_node_id=map_node_id,
+                            conditional_node_id=conditional_node_id,
+                        )
+                        links.append(
+                            {"source": node_id, "target": next_id, "type": "default"}
+                        )
+                    except exceptions.NodeNotFoundError as e:
+                        rich_print(
+                            f"Warning: Neighbor node '{next_node_name}' not found for node '{node.name}': {e}"
+                        )
+
+        return node_id
+
+    # Start processing from the start node
+    start_node = graph.get_node_by_name(graph.start_at)
+    try:
+        process_node(
+            start_node, None, graph, map_node_id=None, conditional_node_id=None
+        )
+    except (exceptions.NodeNotFoundError, AttributeError, KeyError) as e:
+        rich_print(f"Error processing node {start_node}: {e}")
+
+    return {"nodes": nodes, "links": links}

{runnable-0.38.0 → runnable-0.39.0}/runnable/nodes.py

@@ -15,6 +15,13 @@ logger = logging.getLogger(defaults.LOGGER_NAME)
 # --8<-- [start:docs]
 
 
+class NodeInD3(BaseModel):
+    id: str
+    label: str
+    task_type: Optional[str] = None
+    metadata: Optional[Dict[str, Any]] = None
+
+
 class BaseNode(ABC, BaseModel):
     """
     Base class with common functionality provided for a Node of a graph.
@@ -369,6 +376,15 @@ class BaseNode(ABC, BaseModel):
             Dict[str, Any]: _description_
         """
 
+    @abstractmethod
+    def to_d3_node(self) -> NodeInD3:
+        """
+        Convert the node to a D3 node representation.
+
+        Returns:
+            NodeInD3: The D3 node representation of the current node.
+        """
+
 
 # --8<-- [end:docs]
 class TraversalNode(BaseNode):

{runnable-0.38.0 → runnable-0.39.0}/runnable/tasks.py

@@ -209,6 +209,9 @@ class BaseTaskType(BaseModel):
                 parameters=diff_parameters, run_id=self._context.run_id
             )
 
+    def get_d3_metadata(self) -> dict[str, str]:
+        raise NotImplementedError
+
 
 def task_return_to_parameter(task_return: TaskReturns, value: Any) -> Parameter:
     # implicit support for pydantic models
@@ -287,6 +290,72 @@ class PythonTaskType(BaseTaskType):  # pylint: disable=too-few-public-methods
     task_type: str = Field(default="python", serialization_alias="command_type")
     command: str
 
+    def get_d3_metadata(self) -> Dict[str, str]:
+        module, func = utils.get_module_and_attr_names(self.command)
+
+        # Import inspect module to get function signature
+        import inspect
+
+        def format_type_annotation(annotation):
+            """Format type annotation in a more readable way"""
+            if annotation == inspect._empty:
+                return "Any"
+            elif hasattr(annotation, "__name__"):
+                return annotation.__name__
+            elif hasattr(annotation, "__origin__"):
+                # Handle typing types like List, Dict, etc.
+                origin = (
+                    annotation.__origin__.__name__
+                    if hasattr(annotation.__origin__, "__name__")
+                    else str(annotation.__origin__)
+                )
+                args = ", ".join(
+                    format_type_annotation(arg) for arg in annotation.__args__
+                )
+                return f"{origin}[{args}]"
+            else:
+                # Fall back to string representation without 'typing.'
+                return str(annotation).replace("typing.", "")
+
+        # Import the module and get the function
+        sys.path.insert(0, os.getcwd())
+        try:
+            imported_module = importlib.import_module(module)
+            f = getattr(imported_module, func)
+
+            # Get function signature
+            sig = inspect.signature(f)
+
+            # Format parameters with type annotations
+            params_list = []
+            for param_name, param in sig.parameters.items():
+                type_annotation = format_type_annotation(param.annotation)
+                params_list.append(f"{param_name}: {type_annotation}")
+
+            params_str = ", ".join(params_list)
+
+            # Format returns based on self.returns or use "None" if no returns specified
+            if self.returns:
+                returns_str = ", ".join([f"{r.name}({r.kind})" for r in self.returns])
+            else:
+                returns_str = "None"
+
+            # Format function signature
+            signature = f"def {func}({params_str}) -> {returns_str}:"
+
+            return {
+                "module": module,
+                "function": func,
+                "signature": signature,
+            }
+        except Exception as e:
+            logger.warning(f"Could not inspect function {self.command}: {str(e)}")
+            return {
+                "module": module,
+                "function": func,
+                "signature": f"def {func}(...) -> ...",
+            }
+
     def execute_command(
         self,
         map_variable: MapVariableType = None,
@@ -440,6 +509,11 @@ class NotebookTaskType(BaseTaskType):
     command: str
     optional_ploomber_args: dict = {}
 
+    def get_d3_metadata(self) -> Dict[str, str]:
+        return {
+            "notebook": self.command,
+        }
+
     @field_validator("command")
     @classmethod
     def notebook_should_end_with_ipynb(cls, command: str) -> str:
@@ -640,6 +714,11 @@ class ShellTaskType(BaseTaskType):
     task_type: str = Field(default="shell", serialization_alias="command_type")
     command: str
 
+    def get_d3_metadata(self) -> Dict[str, str]:
+        return {
+            "command": self.command[:50],
+        }
+
     @field_validator("returns")
     @classmethod
     def returns_should_be_json(cls, returns: List[TaskReturns]):