runnable 0.34.0a1__py3-none-any.whl → 1.0.0__py3-none-any.whl

runnable/exceptions.py

@@ -6,8 +6,9 @@ class RunLogExistsError(Exception):  # pragma: no cover
     """
 
     def __init__(self, run_id):
-        super().__init__()
-        self.message = f"Run id for {run_id} is already found in the datastore"
+        self.run_id = run_id
+        message = f"Run id for {run_id} is already found in the datastore"
+        super().__init__(message)
 
 
 class JobLogNotFoundError(Exception):
@@ -18,8 +19,9 @@ class JobLogNotFoundError(Exception):
     """
 
     def __init__(self, run_id):
-        super().__init__()
-        self.message = f"Job for {run_id} is not found in the datastore"
+        self.run_id = run_id
+        message = f"Job for {run_id} is not found in the datastore"
+        super().__init__(message)
 
 
 class RunLogNotFoundError(Exception):  # pragma: no cover
@@ -30,8 +32,9 @@ class RunLogNotFoundError(Exception):  # pragma: no cover
     """
 
     def __init__(self, run_id):
-        super().__init__()
-        self.message = f"Run id for {run_id} is not found in the datastore"
+        self.run_id = run_id
+        message = f"Run id for {run_id} is not found in the datastore"
+        super().__init__(message)
 
 
 class StepLogNotFoundError(Exception):  # pragma: no cover
@@ -41,11 +44,11 @@ class StepLogNotFoundError(Exception):  # pragma: no cover
         Exception ([type]): [description]
     """
 
-    def __init__(self, run_id, name):
-        super().__init__()
-        self.message = (
-            f"Step log for {name} is not found in the datastore for Run id: {run_id}"
-        )
+    def __init__(self, run_id, step_name):
+        self.run_id = run_id
+        self.step_name = step_name
+        message = f"Step log for {step_name} is not found in the datastore for Run id: {run_id}"
+        super().__init__(message)
 
 
 class BranchLogNotFoundError(Exception):  # pragma: no cover
@@ -55,11 +58,11 @@ class BranchLogNotFoundError(Exception):  # pragma: no cover
         Exception ([type]): [description]
     """
 
-    def __init__(self, run_id, name):
-        super().__init__()
-        self.message = (
-            f"Branch log for {name} is not found in the datastore for Run id: {run_id}"
-        )
+    def __init__(self, run_id, branch_name):
+        self.run_id = run_id
+        self.branch_name = branch_name
+        message = f"Branch log for {branch_name} is not found in the datastore for Run id: {run_id}"
+        super().__init__(message)
 
 
 class NodeNotFoundError(Exception):  # pragma: no cover
@@ -70,8 +73,9 @@ class NodeNotFoundError(Exception):  # pragma: no cover
     """
 
     def __init__(self, name):
-        super().__init__()
-        self.message = f"Node of name {name} is not found the graph"
+        self.name = name
+        message = f"Node of name {name} is not found the graph"
+        super().__init__(message)
 
 
 class BranchNotFoundError(Exception):  # pragma: no cover
@@ -82,8 +86,9 @@ class BranchNotFoundError(Exception):  # pragma: no cover
     """
 
     def __init__(self, name):
-        super().__init__()
-        self.message = f"Branch of name {name} is not found the graph"
+        self.name = name
+        message = f"Branch of name {name} is not found the graph"
+        super().__init__(message)
 
 
 class NodeMethodCallError(Exception):
@@ -92,32 +97,28 @@ class NodeMethodCallError(Exception):
     """
 
     def __init__(self, message):
-        super().__init__()
-        self.message = message
+        super().__init__(message)
 
 
 class TerminalNodeError(Exception):  # pragma: no cover
     def __init__(self):
-        super().__init__()
-        self.message = "Terminal Nodes do not have next node"
+        message = "Terminal Nodes do not have next node"
+        super().__init__(message)
 
 
 class SecretNotFoundError(Exception):  # pragma: no cover
-    """
-    Exception class
-    Args:
-        Exception ([type]): [description]
-    """
-
     def __init__(self, secret_name, secret_setting):
-        super().__init__()
-        self.message = f"No secret found by name:{secret_name} in {secret_setting}"
+        self.secret_name = secret_name
+        self.secret_setting = secret_setting
+        message = f"No secret found by name:{secret_name} in {secret_setting}"
+        super().__init__(message)
 
 
 class ExecutionFailedError(Exception):  # pragma: no cover
     def __init__(self, run_id: str):
-        super().__init__()
-        self.message = f"Execution failed for run id: {run_id}"
+        self.run_id = run_id
+        message = f"Execution failed for run id: {run_id}"
+        super().__init__(message)
 
 
 class CommandCallError(Exception):  # pragma: no cover

runnable/executor.py

@@ -10,7 +10,7 @@ from pydantic import BaseModel, ConfigDict, PrivateAttr
 import runnable.context as context
 from runnable import defaults
 from runnable.datastore import DataCatalog, JobLog, StepLog
-from runnable.defaults import TypeMapVariable
+from runnable.defaults import MapVariableType
 
 if TYPE_CHECKING:  # pragma: no cover
     from runnable.graph import Graph
@@ -34,7 +34,9 @@ class BaseExecutor(ABC, BaseModel):
     service_name: str = ""
     service_type: str = "executor"
 
-    _is_local: bool = PrivateAttr(default=False)
+    # Should have _should_setup_run_log_at_traversal, local, local_container, emulator is true
+    # False for everything else
+    _should_setup_run_log_at_traversal: bool = PrivateAttr(default=True)
 
     model_config = ConfigDict(extra="forbid")
 
@@ -86,7 +88,7 @@ class BaseExecutor(ABC, BaseModel):
 
     @abstractmethod
     def add_task_log_to_catalog(
-        self, name: str, map_variable: Optional[TypeMapVariable] = None
+        self, name: str, map_variable: Optional[MapVariableType] = None
     ): ...
 
 
@@ -153,16 +155,7 @@ class BaseJobExecutor(BaseExecutor):
         """
         ...
 
-    # @abstractmethod
-    # def scale_up(self, job: BaseTaskType):
-    #     """
-    #     Scale up the job to run on max_nodes
-    #     This has to also call the entry point
-    #     """
-    #     ...
 
-
-# TODO: Consolidate execute_node, trigger_node_execution, _execute_node
 class BasePipelineExecutor(BaseExecutor):
     service_type: str = "pipeline_executor"
     overrides: dict[str, Any] = {}
@@ -214,7 +207,7 @@ class BasePipelineExecutor(BaseExecutor):
     def _execute_node(
         self,
         node: BaseNode,
-        map_variable: TypeMapVariable = None,
+        map_variable: MapVariableType = None,
         mock: bool = False,
     ):
         """
@@ -238,7 +231,7 @@ class BasePipelineExecutor(BaseExecutor):
         ...
 
     @abstractmethod
-    def execute_node(self, node: BaseNode, map_variable: TypeMapVariable = None):
+    def execute_node(self, node: BaseNode, map_variable: MapVariableType = None):
         """
         The entry point for all executors apart from local.
         We have already prepared for node execution.
@@ -253,7 +246,7 @@ class BasePipelineExecutor(BaseExecutor):
         ...
 
     @abstractmethod
-    def execute_from_graph(self, node: BaseNode, map_variable: TypeMapVariable = None):
+    def execute_from_graph(self, node: BaseNode, map_variable: MapVariableType = None):
         """
         This is the entry point to from the graph execution.
 
@@ -282,7 +275,7 @@ class BasePipelineExecutor(BaseExecutor):
 
     @abstractmethod
     def _get_status_and_next_node_name(
-        self, current_node: BaseNode, dag: Graph, map_variable: TypeMapVariable = None
+        self, current_node: BaseNode, dag: Graph, map_variable: MapVariableType = None
     ) -> tuple[str, str]:
         """
         Given the current node and the graph, returns the name of the next node to execute.
@@ -301,7 +294,7 @@ class BasePipelineExecutor(BaseExecutor):
         ...
 
     @abstractmethod
-    def execute_graph(self, dag: Graph, map_variable: TypeMapVariable = None):
+    def execute_graph(self, dag: Graph, map_variable: MapVariableType = None):
         """
         The parallelization is controlled by the nodes and not by this function.
 
@@ -356,7 +349,7 @@ class BasePipelineExecutor(BaseExecutor):
         ...
 
     @abstractmethod
-    def fan_out(self, node: BaseNode, map_variable: TypeMapVariable = None):
+    def fan_out(self, node: BaseNode, map_variable: MapVariableType = None):
         """
         This method is used to appropriately fan-out the execution of a composite node.
         This is only useful when we want to execute a composite node during 3rd party orchestrators.
@@ -379,7 +372,7 @@ class BasePipelineExecutor(BaseExecutor):
         ...
 
     @abstractmethod
-    def fan_in(self, node: BaseNode, map_variable: TypeMapVariable = None):
+    def fan_in(self, node: BaseNode, map_variable: MapVariableType = None):
         """
         This method is used to appropriately fan-in after the execution of a composite node.
         This is only useful when we want to execute a composite node during 3rd party orchestrators.
@@ -402,7 +395,7 @@ class BasePipelineExecutor(BaseExecutor):
 
     @abstractmethod
     def trigger_node_execution(
-        self, node: BaseNode, map_variable: TypeMapVariable = None
+        self, node: BaseNode, map_variable: MapVariableType = None
     ):
         """
         Executor specific way of triggering jobs when runnable does both traversal and execution