runnable 0.34.0a3__py3-none-any.whl → 0.36.0__py3-none-any.whl

runnable/exceptions.py

@@ -6,8 +6,9 @@ class RunLogExistsError(Exception):  # pragma: no cover
     """
 
     def __init__(self, run_id):
-        super().__init__()
-        self.message = f"Run id for {run_id} is already found in the datastore"
+        self.run_id = run_id
+        message = f"Run id for {run_id} is already found in the datastore"
+        super().__init__(message)
 
 
 class JobLogNotFoundError(Exception):
@@ -18,8 +19,9 @@ class JobLogNotFoundError(Exception):
     """
 
     def __init__(self, run_id):
-        super().__init__()
-        self.message = f"Job for {run_id} is not found in the datastore"
+        self.run_id = run_id
+        message = f"Job for {run_id} is not found in the datastore"
+        super().__init__(message)
 
 
 class RunLogNotFoundError(Exception):  # pragma: no cover
@@ -30,8 +32,9 @@ class RunLogNotFoundError(Exception):  # pragma: no cover
     """
 
     def __init__(self, run_id):
-        super().__init__()
-        self.message = f"Run id for {run_id} is not found in the datastore"
+        self.run_id = run_id
+        message = f"Run id for {run_id} is not found in the datastore"
+        super().__init__(message)
 
 
 class StepLogNotFoundError(Exception):  # pragma: no cover
@@ -41,11 +44,11 @@ class StepLogNotFoundError(Exception):  # pragma: no cover
         Exception ([type]): [description]
     """
 
-    def __init__(self, run_id, name):
-        super().__init__()
-        self.message = (
-            f"Step log for {name} is not found in the datastore for Run id: {run_id}"
-        )
+    def __init__(self, run_id, step_name):
+        self.run_id = run_id
+        self.step_name = step_name
+        message = f"Step log for {step_name} is not found in the datastore for Run id: {run_id}"
+        super().__init__(message)
 
 
 class BranchLogNotFoundError(Exception):  # pragma: no cover
@@ -55,11 +58,11 @@ class BranchLogNotFoundError(Exception):  # pragma: no cover
         Exception ([type]): [description]
     """
 
-    def __init__(self, run_id, name):
-        super().__init__()
-        self.message = (
-            f"Branch log for {name} is not found in the datastore for Run id: {run_id}"
-        )
+    def __init__(self, run_id, branch_name):
+        self.run_id = run_id
+        self.branch_name = branch_name
+        message = f"Branch log for {branch_name} is not found in the datastore for Run id: {run_id}"
+        super().__init__(message)
 
 
 class NodeNotFoundError(Exception):  # pragma: no cover
@@ -70,8 +73,9 @@ class NodeNotFoundError(Exception):  # pragma: no cover
     """
 
     def __init__(self, name):
-        super().__init__()
-        self.message = f"Node of name {name} is not found the graph"
+        self.name = name
+        message = f"Node of name {name} is not found the graph"
+        super().__init__(message)
 
 
 class BranchNotFoundError(Exception):  # pragma: no cover
@@ -82,8 +86,9 @@ class BranchNotFoundError(Exception):  # pragma: no cover
     """
 
     def __init__(self, name):
-        super().__init__()
-        self.message = f"Branch of name {name} is not found the graph"
+        self.name = name
+        message = f"Branch of name {name} is not found the graph"
+        super().__init__(message)
 
 
 class NodeMethodCallError(Exception):
@@ -92,32 +97,28 @@ class NodeMethodCallError(Exception):
     """
 
     def __init__(self, message):
-        super().__init__()
-        self.message = message
+        super().__init__(message)
 
 
 class TerminalNodeError(Exception):  # pragma: no cover
     def __init__(self):
-        super().__init__()
-        self.message = "Terminal Nodes do not have next node"
+        message = "Terminal Nodes do not have next node"
+        super().__init__(message)
 
 
 class SecretNotFoundError(Exception):  # pragma: no cover
-    """
-    Exception class
-    Args:
-        Exception ([type]): [description]
-    """
-
     def __init__(self, secret_name, secret_setting):
-        super().__init__()
-        self.message = f"No secret found by name:{secret_name} in {secret_setting}"
+        self.secret_name = secret_name
+        self.secret_setting = secret_setting
+        message = f"No secret found by name:{secret_name} in {secret_setting}"
+        super().__init__(message)
 
 
 class ExecutionFailedError(Exception):  # pragma: no cover
     def __init__(self, run_id: str):
-        super().__init__()
-        self.message = f"Execution failed for run id: {run_id}"
+        self.run_id = run_id
+        message = f"Execution failed for run id: {run_id}"
+        super().__init__(message)
 
 
 class CommandCallError(Exception):  # pragma: no cover

runnable/executor.py

@@ -10,7 +10,7 @@ from pydantic import BaseModel, ConfigDict, PrivateAttr
 import runnable.context as context
 from runnable import defaults
 from runnable.datastore import DataCatalog, JobLog, StepLog
-from runnable.defaults import TypeMapVariable
+from runnable.defaults import MapVariableType
 
 if TYPE_CHECKING:  # pragma: no cover
     from runnable.graph import Graph
@@ -34,7 +34,9 @@ class BaseExecutor(ABC, BaseModel):
     service_name: str = ""
     service_type: str = "executor"
 
-    _is_local: bool = PrivateAttr(default=False)
+    # Should have _should_setup_run_log_at_traversal, local, local_container, emulator is true
+    # False for everything else
+    _should_setup_run_log_at_traversal: bool = PrivateAttr(default=True)
 
     model_config = ConfigDict(extra="forbid")
 
@@ -86,7 +88,7 @@ class BaseExecutor(ABC, BaseModel):
 
     @abstractmethod
     def add_task_log_to_catalog(
-        self, name: str, map_variable: Optional[TypeMapVariable] = None
+        self, name: str, map_variable: Optional[MapVariableType] = None
     ): ...
 
 
@@ -153,16 +155,7 @@ class BaseJobExecutor(BaseExecutor):
         """
         ...
 
-    # @abstractmethod
-    # def scale_up(self, job: BaseTaskType):
-    #     """
-    #     Scale up the job to run on max_nodes
-    #     This has to also call the entry point
-    #     """
-    #     ...
 
-
-# TODO: Consolidate execute_node, trigger_node_execution, _execute_node
 class BasePipelineExecutor(BaseExecutor):
     service_type: str = "pipeline_executor"
     overrides: dict[str, Any] = {}
@@ -214,7 +207,7 @@ class BasePipelineExecutor(BaseExecutor):
     def _execute_node(
         self,
         node: BaseNode,
-        map_variable: TypeMapVariable = None,
+        map_variable: MapVariableType = None,
         mock: bool = False,
     ):
         """
@@ -238,7 +231,7 @@ class BasePipelineExecutor(BaseExecutor):
         ...
 
     @abstractmethod
-    def execute_node(self, node: BaseNode, map_variable: TypeMapVariable = None):
+    def execute_node(self, node: BaseNode, map_variable: MapVariableType = None):
         """
         The entry point for all executors apart from local.
         We have already prepared for node execution.
@@ -253,7 +246,7 @@ class BasePipelineExecutor(BaseExecutor):
         ...
 
     @abstractmethod
-    def execute_from_graph(self, node: BaseNode, map_variable: TypeMapVariable = None):
+    def execute_from_graph(self, node: BaseNode, map_variable: MapVariableType = None):
         """
         This is the entry point to from the graph execution.
 
@@ -282,7 +275,7 @@ class BasePipelineExecutor(BaseExecutor):
 
     @abstractmethod
     def _get_status_and_next_node_name(
-        self, current_node: BaseNode, dag: Graph, map_variable: TypeMapVariable = None
+        self, current_node: BaseNode, dag: Graph, map_variable: MapVariableType = None
     ) -> tuple[str, str]:
         """
         Given the current node and the graph, returns the name of the next node to execute.
@@ -301,7 +294,7 @@ class BasePipelineExecutor(BaseExecutor):
         ...
 
     @abstractmethod
-    def execute_graph(self, dag: Graph, map_variable: TypeMapVariable = None):
+    def execute_graph(self, dag: Graph, map_variable: MapVariableType = None):
         """
         The parallelization is controlled by the nodes and not by this function.
 
@@ -356,7 +349,7 @@ class BasePipelineExecutor(BaseExecutor):
         ...
 
     @abstractmethod
-    def fan_out(self, node: BaseNode, map_variable: TypeMapVariable = None):
+    def fan_out(self, node: BaseNode, map_variable: MapVariableType = None):
         """
         This method is used to appropriately fan-out the execution of a composite node.
         This is only useful when we want to execute a composite node during 3rd party orchestrators.
@@ -379,7 +372,7 @@ class BasePipelineExecutor(BaseExecutor):
         ...
 
     @abstractmethod
-    def fan_in(self, node: BaseNode, map_variable: TypeMapVariable = None):
+    def fan_in(self, node: BaseNode, map_variable: MapVariableType = None):
         """
         This method is used to appropriately fan-in after the execution of a composite node.
         This is only useful when we want to execute a composite node during 3rd party orchestrators.
@@ -402,7 +395,7 @@ class BasePipelineExecutor(BaseExecutor):
 
     @abstractmethod
     def trigger_node_execution(
-        self, node: BaseNode, map_variable: TypeMapVariable = None
+        self, node: BaseNode, map_variable: MapVariableType = None
     ):
         """
         Executor specific way of triggering jobs when runnable does both traversal and execution

runnable/names.py

@@ -306,7 +306,7 @@ left = [
     "chalky",
     "moist",
     "formal",
-    "brute force",
+    "force",
     "ancient",
     "wan",
     "square",

runnable/nodes.py

@@ -7,7 +7,8 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator
 import runnable.context as context
 from runnable import defaults, exceptions
 from runnable.datastore import StepLog
-from runnable.defaults import TypeMapVariable
+from runnable.defaults import MapVariableType
+from runnable.graph import Graph
 
 logger = logging.getLogger(defaults.LOGGER_NAME)
 
@@ -41,6 +42,7 @@ class BaseNode(ABC, BaseModel):
 
     @property
     def _context(self):
+        assert isinstance(context.run_context, context.PipelineContext)
         return context.run_context
 
     model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=False)
@@ -80,7 +82,7 @@ class BaseNode(ABC, BaseModel):
 
     @classmethod
     def _resolve_map_placeholders(
-        cls, name: str, map_variable: TypeMapVariable = None
+        cls, name: str, map_variable: MapVariableType = None
     ) -> str:
         """
         If there is no map step used, then we just return the name as we find it.
@@ -131,7 +133,7 @@ class BaseNode(ABC, BaseModel):
 
         return name
 
-    def _get_step_log_name(self, map_variable: TypeMapVariable = None) -> str:
+    def _get_step_log_name(self, map_variable: MapVariableType = None) -> str:
         """
         For every step in the dag, there is a corresponding step log name.
         This method returns the step log name in dot path convention.
@@ -150,7 +152,7 @@ class BaseNode(ABC, BaseModel):
             self.internal_name, map_variable=map_variable
         )
 
-    def _get_branch_log_name(self, map_variable: TypeMapVariable = None) -> str:
+    def _get_branch_log_name(self, map_variable: MapVariableType = None) -> str:
         """
         For nodes that are internally branches, this method returns the branch log name.
         The branch log name is in dot path convention.
@@ -218,7 +220,7 @@ class BaseNode(ABC, BaseModel):
         """
 
     @abstractmethod
-    def _get_branch_by_name(self, branch_name: str):
+    def _get_branch_by_name(self, branch_name: str) -> Graph:
         """
         Retrieve a branch by name.
 
@@ -279,7 +281,7 @@ class BaseNode(ABC, BaseModel):
     def execute(
         self,
         mock=False,
-        map_variable: TypeMapVariable = None,
+        map_variable: MapVariableType = None,
         attempt_number: int = 1,
     ) -> StepLog:
         """
@@ -299,7 +301,7 @@ class BaseNode(ABC, BaseModel):
         """
 
     @abstractmethod
-    def execute_as_graph(self, map_variable: TypeMapVariable = None):
+    def execute_as_graph(self, map_variable: MapVariableType = None):
         """
         This function would be called to set up the execution of the individual
         branches of a composite node.
@@ -314,7 +316,7 @@ class BaseNode(ABC, BaseModel):
         """
 
     @abstractmethod
-    def fan_out(self, map_variable: TypeMapVariable = None):
+    def fan_out(self, map_variable: MapVariableType = None):
         """
         This function would be called to set up the execution of the individual
         branches of a composite node.
@@ -330,7 +332,7 @@ class BaseNode(ABC, BaseModel):
         """
 
     @abstractmethod
-    def fan_in(self, map_variable: TypeMapVariable = None):
+    def fan_in(self, map_variable: MapVariableType = None):
         """
         This function would be called to tear down the execution of the individual
         branches of a composite node.
@@ -439,17 +441,17 @@ class ExecutableNode(TraversalNode):
             "This is an executable node and does not have branches"
         )
 
-    def execute_as_graph(self, map_variable: TypeMapVariable = None):
+    def execute_as_graph(self, map_variable: MapVariableType = None):
         raise exceptions.NodeMethodCallError(
             "This is an executable node and does not have a graph"
         )
 
-    def fan_in(self, map_variable: TypeMapVariable = None):
+    def fan_in(self, map_variable: MapVariableType = None):
         raise exceptions.NodeMethodCallError(
             "This is an executable node and does not have a fan in"
         )
 
-    def fan_out(self, map_variable: TypeMapVariable = None):
+    def fan_out(self, map_variable: MapVariableType = None):
         raise exceptions.NodeMethodCallError(
             "This is an executable node and does not have a fan out"
         )
@@ -475,7 +477,7 @@ class CompositeNode(TraversalNode):
     def execute(
         self,
         mock=False,
-        map_variable: TypeMapVariable = None,
+        map_variable: MapVariableType = None,
         attempt_number: int = 1,
     ) -> StepLog:
         raise exceptions.NodeMethodCallError(
@@ -505,15 +507,15 @@ class TerminalNode(BaseNode):
     def _get_max_attempts(self) -> int:
         return 1
 
-    def execute_as_graph(self, map_variable: TypeMapVariable = None):
+    def execute_as_graph(self, map_variable: MapVariableType = None):
         raise exceptions.TerminalNodeError()
 
-    def fan_in(self, map_variable: TypeMapVariable = None):
+    def fan_in(self, map_variable: MapVariableType = None):
         raise exceptions.TerminalNodeError()
 
     def fan_out(
         self,
-        map_variable: TypeMapVariable = None,
+        map_variable: MapVariableType = None,
     ):
         raise exceptions.TerminalNodeError()
 

runnable/parameters.py

@@ -10,7 +10,7 @@ from typing_extensions import Callable
 
 from runnable import defaults
 from runnable.datastore import JsonParameter, ObjectParameter
-from runnable.defaults import TypeMapVariable
+from runnable.defaults import MapVariableType
 from runnable.utils import remove_prefix
 
 logger = logging.getLogger(defaults.LOGGER_NAME)
@@ -51,7 +51,7 @@ def get_user_set_parameters(remove: bool = False) -> Dict[str, JsonParameter]:
 def filter_arguments_for_func(
     func: Callable[..., Any],
     params: Dict[str, Any],
-    map_variable: TypeMapVariable = None,
+    map_variable: MapVariableType = None,
 ) -> Dict[str, Any]:
     """
     Inspects the function to be called as part of the pipeline to find the arguments of the function.