runnable 0.13.0__py3-none-any.whl → 0.16.0__py3-none-any.whl

runnable/exceptions.py

@@ -10,6 +10,18 @@ class RunLogExistsError(Exception):  # pragma: no cover
         self.message = f"Run id for {run_id} is already found in the datastore"
 
 
+class JobLogNotFoundError(Exception):
+    """
+    Exception class
+    Args:
+        Exception ([type]): [description]
+    """
+
+    def __init__(self, run_id):
+        super().__init__()
+        self.message = f"Job for {run_id} is not found in the datastore"
+
+
 class RunLogNotFoundError(Exception):  # pragma: no cover
     """
     Exception class
@@ -31,7 +43,9 @@ class StepLogNotFoundError(Exception):  # pragma: no cover
 
     def __init__(self, run_id, name):
         super().__init__()
-        self.message = f"Step log for {name} is not found in the datastore for Run id: {run_id}"
+        self.message = (
+            f"Step log for {name} is not found in the datastore for Run id: {run_id}"
+        )
 
 
 class BranchLogNotFoundError(Exception):  # pragma: no cover
@@ -43,7 +57,9 @@ class BranchLogNotFoundError(Exception):  # pragma: no cover
 
     def __init__(self, run_id, name):
         super().__init__()
-        self.message = f"Branch log for {name} is not found in the datastore for Run id: {run_id}"
+        self.message = (
+            f"Branch log for {name} is not found in the datastore for Run id: {run_id}"
+        )
 
 
 class NodeNotFoundError(Exception):  # pragma: no cover
@@ -70,6 +86,16 @@ class BranchNotFoundError(Exception):  # pragma: no cover
         self.message = f"Branch of name {name} is not found the graph"
 
 
+class NodeMethodCallError(Exception):
+    """
+    Exception class
+    """
+
+    def __init__(self, message):
+        super().__init__()
+        self.message = message
+
+
 class TerminalNodeError(Exception):  # pragma: no cover
     def __init__(self):
         super().__init__()

runnable/executor.py

@@ -5,17 +5,17 @@ import os
 from abc import ABC, abstractmethod
 from typing import TYPE_CHECKING, Any, Dict, List, Optional
 
-from pydantic import BaseModel, ConfigDict
+from pydantic import BaseModel, ConfigDict, PrivateAttr
 
 import runnable.context as context
 from runnable import defaults
-from runnable.datastore import DataCatalog, StepLog
+from runnable.datastore import DataCatalog, JobLog, StepLog
 from runnable.defaults import TypeMapVariable
 from runnable.graph import Graph
 
 if TYPE_CHECKING:  # pragma: no cover
-    from runnable.extensions.nodes import TaskNode
     from runnable.nodes import BaseNode
+    from runnable.tasks import BaseTaskType
 
 logger = logging.getLogger(defaults.LOGGER_NAME)
 
@@ -34,11 +34,10 @@ class BaseExecutor(ABC, BaseModel):
     service_name: str = ""
     service_type: str = "executor"
 
-    overrides: dict = {}
-
-    _local: bool = False  # This is a flag to indicate whether the executor is local or not.
+    _is_local: bool = (
+        False  # This is a flag to indicate whether the executor is local or not.
+    )
 
-    _context_node = None  # type: BaseNode
     model_config = ConfigDict(extra="forbid")
 
     @property
@@ -65,32 +64,63 @@ class BaseExecutor(ABC, BaseModel):
         """
         ...
 
+    # TODO: Make this attempt number
+    @property
+    def step_attempt_number(self) -> int:
+        """
+        The attempt number of the current step.
+        Orchestrators should use this step to submit multiple attempts of the job.
+
+        Returns:
+            int: The attempt number of the current step. Defaults to 1.
+        """
+        return int(os.environ.get(defaults.ATTEMPT_NUMBER, 1))
+
     @abstractmethod
-    def prepare_for_graph_execution(self):
+    def send_return_code(self, stage="traversal"):
+        """
+        Convenience function used by pipeline to send return code to the caller of the cli
+
+        Raises:
+            Exception: If the pipeline execution failed
         """
-        This method should be called prior to calling execute_graph.
-        Perform any steps required before doing the graph execution.
+        ...
+
 
-        The most common implementation is to prepare a run log for the run if the run uses local interactive compute.
+class BaseJobExecutor(BaseExecutor):
+    service_type: str = "job_executor"
 
-        But in cases of actual rendering the job specs (eg: AWS step functions, K8's) we check if the services are OK.
-        We do not set up a run log as its not relevant.
+    @abstractmethod
+    def submit_job(self, job: BaseTaskType, catalog_settings: Optional[List[str]]):
+        """
+        Local executors should
+        - create the run log
+        - and call an execute_job
+
+        Non local executors should
+        - transpile the job to the platform specific job spec
+        - submit the job to call execute_job
         """
         ...
 
     @abstractmethod
-    def prepare_for_node_execution(self):
+    def add_code_identities(self, job_log: JobLog, **kwargs):
         """
-        Perform any modifications to the services prior to execution of the node.
+        Add code identities specific to the implementation.
+
+        The Base class has an implementation of adding git code identities.
 
         Args:
-            node (Node): [description]
-            map_variable (dict, optional): [description]. Defaults to None.
+            step_log (object): The step log object
+            node (BaseNode): The node we are adding the step log for
         """
         ...
 
     @abstractmethod
-    def _sync_catalog(self, stage: str, synced_catalogs=None) -> Optional[List[DataCatalog]]:
+    def _sync_catalog(
+        self,
+        catalog_settings: Optional[List[str]],
+    ) -> Optional[List[DataCatalog]]:
         """
         1). Identify the catalog settings by over-riding node settings with the global settings.
         2). For stage = get:
@@ -112,6 +142,34 @@ class BaseExecutor(ABC, BaseModel):
         """
         ...
 
+    @abstractmethod
+    def execute_job(self, job: BaseTaskType, catalog_settings: Optional[List[str]]):
+        """
+        Focusses only on execution of the job.
+        """
+        ...
+
+
+# TODO: Consolidate execute_node, trigger_node_execution, _execute_node
+class BasePipelineExecutor(BaseExecutor):
+    service_type: str = "pipeline_executor"
+    overrides: dict = {}
+
+    _context_node: Optional[BaseNode] = PrivateAttr(default=None)
+
+    @abstractmethod
+    def add_code_identities(self, node: BaseNode, step_log: StepLog, **kwargs):
+        """
+        Add code identities specific to the implementation.
+
+        The Base class has an implementation of adding git code identities.
+
+        Args:
+            step_log (object): The step log object
+            node (BaseNode): The node we are adding the step log for
+        """
+        ...
+
     @abstractmethod
     def get_effective_compute_data_folder(self) -> Optional[str]:
         """
@@ -129,19 +187,39 @@ class BaseExecutor(ABC, BaseModel):
         """
         ...
 
-    @property
-    def step_attempt_number(self) -> int:
+    @abstractmethod
+    def _sync_catalog(
+        self, stage: str, synced_catalogs=None
+    ) -> Optional[List[DataCatalog]]:
         """
-        The attempt number of the current step.
-        Orchestrators should use this step to submit multiple attempts of the job.
+        1). Identify the catalog settings by over-riding node settings with the global settings.
+        2). For stage = get:
+                Identify the catalog items that are being asked to get from the catalog
+                And copy them to the local compute data folder
+        3). For stage = put:
+                Identify the catalog items that are being asked to put into the catalog
+                Copy the items from local compute folder to the catalog
+        4). Add the items onto the step log according to the stage
+
+        Args:
+            node (Node): The current node being processed
+            step_log (StepLog): The step log corresponding to that node
+            stage (str): One of get or put
+
+        Raises:
+            Exception: If the stage is not in one of get/put
 
-        Returns:
-            int: The attempt number of the current step. Defaults to 1.
         """
-        return int(os.environ.get(defaults.ATTEMPT_NUMBER, 1))
+        ...
 
     @abstractmethod
-    def _execute_node(self, node: BaseNode, map_variable: TypeMapVariable = None, mock: bool = False, **kwargs):
+    def _execute_node(
+        self,
+        node: BaseNode,
+        map_variable: TypeMapVariable = None,
+        mock: bool = False,
+        **kwargs,
+    ):
         """
         This is the entry point when we do the actual execution of the function.
 
@@ -163,7 +241,9 @@ class BaseExecutor(ABC, BaseModel):
         ...
 
     @abstractmethod
-    def execute_node(self, node: BaseNode, map_variable: TypeMapVariable = None, **kwargs):
+    def execute_node(
+        self, node: BaseNode, map_variable: TypeMapVariable = None, **kwargs
+    ):
         """
         The entry point for all executors apart from local.
         We have already prepared for node execution.
@@ -178,20 +258,9 @@ class BaseExecutor(ABC, BaseModel):
         ...
 
     @abstractmethod
-    def add_code_identities(self, node: BaseNode, step_log: StepLog, **kwargs):
-        """
-        Add code identities specific to the implementation.
-
-        The Base class has an implementation of adding git code identities.
-
-        Args:
-            step_log (object): The step log object
-            node (BaseNode): The node we are adding the step log for
-        """
-        ...
-
-    @abstractmethod
-    def execute_from_graph(self, node: BaseNode, map_variable: TypeMapVariable = None, **kwargs):
+    def execute_from_graph(
+        self, node: BaseNode, map_variable: TypeMapVariable = None, **kwargs
+    ):
         """
         This is the entry point to from the graph execution.
 
@@ -219,24 +288,9 @@ class BaseExecutor(ABC, BaseModel):
         ...
 
     @abstractmethod
-    def trigger_job(self, node: BaseNode, map_variable: TypeMapVariable = None, **kwargs):
-        """
-        Executor specific way of triggering jobs when runnable does both traversal and execution
-
-        Transpilers will NEVER use this method and will NEVER call them.
-        Only interactive executors who need execute_from_graph will ever implement it.
-
-        Args:
-            node (BaseNode): The node to execute
-            map_variable (str, optional): If the node if of a map state, this corresponds to the value of iterable.
-                    Defaults to ''.
-
-        NOTE: We do not raise an exception as this method is not required by many extensions
-        """
-        ...
-
-    @abstractmethod
-    def _get_status_and_next_node_name(self, current_node: BaseNode, dag: Graph, map_variable: TypeMapVariable = None):
+    def _get_status_and_next_node_name(
+        self, current_node: BaseNode, dag: Graph, map_variable: TypeMapVariable = None
+    ) -> tuple[str, str]:
         """
         Given the current node and the graph, returns the name of the next node to execute.
 
@@ -275,17 +329,7 @@ class BaseExecutor(ABC, BaseModel):
         ...
 
     @abstractmethod
-    def send_return_code(self, stage="traversal"):
-        """
-        Convenience function used by pipeline to send return code to the caller of the cli
-
-        Raises:
-            Exception: If the pipeline execution failed
-        """
-        ...
-
-    @abstractmethod
-    def _resolve_executor_config(self, node: BaseNode):
+    def _resolve_executor_config(self, node: BaseNode) -> Dict[str, Any]:
         """
         The overrides section can contain specific over-rides to an global executor config.
         To avoid too much clutter in the dag definition, we allow the configuration file to have overrides block.
@@ -318,22 +362,6 @@ class BaseExecutor(ABC, BaseModel):
         """
         ...
 
-    @abstractmethod
-    def execute_job(self, node: TaskNode):
-        """
-        Executor specific way of executing a job (python function or a notebook).
-
-        Interactive executors should execute the job.
-        Transpilers should write the instructions.
-
-        Args:
-            node (BaseNode): The job node to execute
-
-        Raises:
-            NotImplementedError: Executors should choose to extend this functionality or not.
-        """
-        ...
-
     @abstractmethod
     def fan_out(self, node: BaseNode, map_variable: TypeMapVariable = None):
         """
@@ -378,3 +406,22 @@ class BaseExecutor(ABC, BaseModel):
 
         """
         ...
+
+    @abstractmethod
+    def trigger_node_execution(
+        self, node: BaseNode, map_variable: TypeMapVariable = None, **kwargs
+    ):
+        """
+        Executor specific way of triggering jobs when runnable does both traversal and execution
+
+        Transpilers will NEVER use this method and will NEVER call them.
+        Only interactive executors who need execute_from_graph will ever implement it.
+
+        Args:
+            node (BaseNode): The node to execute
+            map_variable (str, optional): If the node if of a map state, this corresponds to the value of iterable.
+                    Defaults to ''.
+
+        NOTE: We do not raise an exception as this method is not required by many extensions
+        """
+        ...

runnable/graph.py

@@ -24,7 +24,9 @@ class Graph(BaseModel):
     name: str = ""
     description: Optional[str] = ""
     internal_branch_name: str = Field(default="", exclude=True)
-    nodes: SerializeAsAny[Dict[str, "BaseNode"]] = Field(default_factory=dict, serialization_alias="steps")
+    nodes: SerializeAsAny[Dict[str, "BaseNode"]] = Field(
+        default_factory=dict, serialization_alias="steps"
+    )
 
     def get_summary(self) -> Dict[str, Any]:
         """
@@ -229,7 +231,9 @@ class Graph(BaseModel):
                     return False
         return True
 
-    def is_cyclic_util(self, node: "BaseNode", visited: Dict[str, bool], recstack: Dict[str, bool]) -> bool:
+    def is_cyclic_util(
+        self, node: "BaseNode", visited: Dict[str, bool], recstack: Dict[str, bool]
+    ) -> bool:
         """
         Recursive utility that determines if a node and neighbors has a cycle. Is used in is_dag method.
 
@@ -327,7 +331,9 @@ def create_graph(dag_config: Dict[str, Any], internal_branch_name: str = "") ->
         Graph: The created graph object
     """
     description: str = dag_config.get("description", None)
-    start_at: str = cast(str, dag_config.get("start_at"))  # Let the start_at be relative to the graph
+    start_at: str = cast(
+        str, dag_config.get("start_at")
+    )  # Let the start_at be relative to the graph
 
     graph = Graph(
         start_at=start_at,
@@ -339,7 +345,9 @@ def create_graph(dag_config: Dict[str, Any], internal_branch_name: str = "") ->
     for name, step_config in dag_config.get("steps", {}).items():
         logger.info(f"Adding node {name} with :{step_config}")
 
-        node = create_node(name, step_config=step_config, internal_branch_name=internal_branch_name)
+        node = create_node(
+            name, step_config=step_config, internal_branch_name=internal_branch_name
+        )
         graph.add_node(node)
 
     graph.add_terminal_nodes(internal_branch_name=internal_branch_name)
@@ -369,8 +377,12 @@ def create_node(name: str, step_config: dict, internal_branch_name: Optional[str
         internal_name = internal_branch_name + "." + name
 
     try:
-        node_type = step_config.pop("type")  # Remove the type as it is not used in node creation.
-        node_mgr: BaseNode = driver.DriverManager(namespace="nodes", name=node_type).driver
+        node_type = step_config.pop(
+            "type"
+        )  # Remove the type as it is not used in node creation.
+        node_mgr: BaseNode = driver.DriverManager(
+            namespace="nodes", name=node_type
+        ).driver
 
         next_node = step_config.pop("next", None)
 
@@ -386,7 +398,6 @@ def create_node(name: str, step_config: dict, internal_branch_name: Optional[str
         node = node_mgr.parse_from_config(config=invoke_kwds)
         return node
     except KeyError:
-        # type is missing!!
         msg = "The node configuration does not contain the required key 'type'."
         logger.exception(step_config)
         raise Exception(msg)
@@ -424,11 +435,18 @@ def search_node_by_internal_name(dag: Graph, internal_name: str):
     for i in range(len(dot_path)):
         if i % 2:
             # Its odd, so we are in brach name
-            current_branch = current_node._get_branch_by_name(".".join(dot_path[: i + 1]))  # type: ignore
-            logger.debug(f"Finding step for {internal_name} in branch: {current_branch}")
+
+            current_branch = current_node._get_branch_by_name(  # type: ignore
+                ".".join(dot_path[: i + 1])
+            )
+            logger.debug(
+                f"Finding step for {internal_name} in branch: {current_branch}"
+            )
         else:
             # Its even, so we are in Step, we start here!
-            current_node = current_branch.get_node_by_internal_name(".".join(dot_path[: i + 1]))
+            current_node = current_branch.get_node_by_internal_name(
+                ".".join(dot_path[: i + 1])
+            )
             logger.debug(f"Finding {internal_name} in node: {current_node}")
 
     logger.debug(f"current branch : {current_branch}, current step {current_node}")
@@ -463,12 +481,18 @@ def search_branch_by_internal_name(dag: Graph, internal_name: str):
     for i in range(len(dot_path)):
         if i % 2:
             # Its odd, so we are in brach name
-            current_branch = current_node._get_branch_by_name(".".join(dot_path[: i + 1]))  # type: ignore
-            logger.debug(f"Finding step for {internal_name} in branch: {current_branch}")
+            current_branch = current_node._get_branch_by_name(  # type: ignore
+                ".".join(dot_path[: i + 1])
+            )
+            logger.debug(
+                f"Finding step for {internal_name} in branch: {current_branch}"
+            )
 
         else:
             # Its even, so we are in Step, we start here!
-            current_node = current_branch.get_node_by_internal_name(".".join(dot_path[: i + 1]))
+            current_node = current_branch.get_node_by_internal_name(
+                ".".join(dot_path[: i + 1])
+            )
             logger.debug(f"Finding {internal_name} in node: {current_node}")
 
     logger.debug(f"current branch : {current_branch}, current step {current_node}")

runnable/nodes.py

@@ -51,7 +51,9 @@ class BaseNode(ABC, BaseModel):
             raise ValueError("Node names cannot have . or '%' in them")
         return name
 
-    def _command_friendly_name(self, replace_with=defaults.COMMAND_FRIENDLY_CHARACTER) -> str:
+    def _command_friendly_name(
+        self, replace_with=defaults.COMMAND_FRIENDLY_CHARACTER
+    ) -> str:
         """
         Replace spaces with special character for spaces.
         Spaces in the naming of the node is convenient for the user but causes issues when used programmatically.
@@ -76,7 +78,9 @@ class BaseNode(ABC, BaseModel):
         return command_name.replace(defaults.COMMAND_FRIENDLY_CHARACTER, " ")
 
     @classmethod
-    def _resolve_map_placeholders(cls, name: str, map_variable: TypeMapVariable = None) -> str:
+    def _resolve_map_placeholders(
+        cls, name: str, map_variable: TypeMapVariable = None
+    ) -> str:
         """
         If there is no map step used, then we just return the name as we find it.
 
@@ -141,7 +145,9 @@ class BaseNode(ABC, BaseModel):
         Returns:
             str: The dot path name of the step log name
         """
-        return self._resolve_map_placeholders(self.internal_name, map_variable=map_variable)
+        return self._resolve_map_placeholders(
+            self.internal_name, map_variable=map_variable
+        )
 
     def _get_branch_log_name(self, map_variable: TypeMapVariable = None) -> str:
         """
@@ -158,7 +164,9 @@ class BaseNode(ABC, BaseModel):
         Returns:
             str: The dot path name of the branch log
         """
-        return self._resolve_map_placeholders(self.internal_branch_name, map_variable=map_variable)
+        return self._resolve_map_placeholders(
+            self.internal_branch_name, map_variable=map_variable
+        )
 
     def __str__(self) -> str:  # pragma: no cover
         """
@@ -180,7 +188,6 @@ class BaseNode(ABC, BaseModel):
             str: The on_failure node defined by the dag or ''
         This is a base implementation which the BaseNode does not satisfy
         """
-        ...
 
     @abstractmethod
     def _get_next_node(self) -> str:
@@ -190,7 +197,6 @@ class BaseNode(ABC, BaseModel):
         Returns:
             str: The node name, relative to the dag, as defined by the config
         """
-        ...
 
     @abstractmethod
     def _is_terminal_node(self) -> bool:
@@ -200,7 +206,6 @@ class BaseNode(ABC, BaseModel):
         Returns:
             bool: True or False of whether there is next node.
         """
-        ...
 
     @abstractmethod
     def _get_catalog_settings(self) -> Dict[str, Any]:
@@ -210,7 +215,6 @@ class BaseNode(ABC, BaseModel):
         Returns:
             dict: catalog settings defined as per the node or None
         """
-        ...
 
     @abstractmethod
     def _get_branch_by_name(self, branch_name: str):
@@ -225,7 +229,6 @@ class BaseNode(ABC, BaseModel):
         Raises:
             Exception: [description]
         """
-        ...
 
     def _get_neighbors(self) -> List[str]:
         """
@@ -261,7 +264,6 @@ class BaseNode(ABC, BaseModel):
         Returns:
             dict: The executor config, if defined or an empty dict
         """
-        ...
 
     @abstractmethod
     def _get_max_attempts(self) -> int:
@@ -271,7 +273,6 @@ class BaseNode(ABC, BaseModel):
         Returns:
             int: The number of maximum retries as defined by the config or 1.
         """
-        ...
 
     @abstractmethod
     def execute(
@@ -296,7 +297,6 @@ class BaseNode(ABC, BaseModel):
         Raises:
             NotImplementedError: Base class, hence not implemented.
         """
-        ...
 
     @abstractmethod
     def execute_as_graph(self, map_variable: TypeMapVariable = None, **kwargs):
@@ -312,7 +312,6 @@ class BaseNode(ABC, BaseModel):
         Raises:
             NotImplementedError: Base class, hence not implemented.
         """
-        ...
 
     @abstractmethod
     def fan_out(self, map_variable: TypeMapVariable = None, **kwargs):
@@ -329,7 +328,6 @@ class BaseNode(ABC, BaseModel):
         Raises:
             Exception: If the node is not a composite node.
         """
-        ...
 
     @abstractmethod
     def fan_in(self, map_variable: TypeMapVariable = None, **kwargs):
@@ -346,7 +344,6 @@ class BaseNode(ABC, BaseModel):
         Raises:
             Exception: If the node is not a composite node.
         """
-        ...
 
     @classmethod
     @abstractmethod
@@ -360,7 +357,6 @@ class BaseNode(ABC, BaseModel):
         Returns:
             BaseNode: The corresponding node.
         """
-        ...
 
     @abstractmethod
     def get_summary(self) -> Dict[str, Any]:
@@ -439,16 +435,34 @@ class ExecutableNode(TraversalNode):
         return self.max_attempts
 
     def _get_branch_by_name(self, branch_name: str):
-        raise Exception("This is an executable node and does not have branches")
+        raise exceptions.NodeMethodCallError(
+            "This is an executable node and does not have branches"
+        )
 
     def execute_as_graph(self, map_variable: TypeMapVariable = None, **kwargs):
-        raise Exception("This is an executable node and does not have a graph")
+        raise exceptions.NodeMethodCallError(
+            "This is an executable node and does not have a graph"
+        )
 
     def fan_in(self, map_variable: TypeMapVariable = None, **kwargs):
-        raise Exception("This is an executable node and does not have a fan in")
+        raise exceptions.NodeMethodCallError(
+            "This is an executable node and does not have a fan in"
+        )
 
     def fan_out(self, map_variable: TypeMapVariable = None, **kwargs):
-        raise Exception("This is an executable node and does not have a fan out")
+        raise exceptions.NodeMethodCallError(
+            "This is an executable node and does not have a fan out"
+        )
+
+    def prepare_for_job_execution(self):
+        raise exceptions.NodeMethodCallError(
+            "This is an executable node and does not have a prepare_for_job_execution"
+        )
+
+    def tear_down_after_job_execution(self):
+        raise exceptions.NodeMethodCallError(
+            "This is an executable node and does not have a tear_down_after_job_execution",
+        )
 
 
 class CompositeNode(TraversalNode):
@@ -459,7 +473,9 @@ class CompositeNode(TraversalNode):
         Returns:
             dict: catalog settings defined as per the node or None
         """
-        raise Exception("This is a composite node and does not have a catalog settings")
+        raise exceptions.NodeMethodCallError(
+            "This is a composite node and does not have a catalog settings"
+        )
 
     def _get_max_attempts(self) -> int:
         raise Exception("This is a composite node and does not have a max_attempts")
@@ -471,7 +487,19 @@ class CompositeNode(TraversalNode):
         attempt_number: int = 1,
         **kwargs,
     ) -> StepLog:
-        raise Exception("This is a composite node and does not have an execute function")
+        raise exceptions.NodeMethodCallError(
+            "This is a composite node and does not have an execute function"
+        )
+
+    def prepare_for_job_execution(self):
+        raise exceptions.NodeMethodCallError(
+            "This is an executable node and does not have a prepare_for_job_execution"
+        )
+
+    def tear_down_after_job_execution(self):
+        raise exceptions.NodeMethodCallError(
+            "This is an executable node and does not have a tear_down_after_job_execution"
+        )
 
 
 class TerminalNode(BaseNode):