runnable 0.1.0__py3-none-any.whl → 0.2.0__py3-none-any.whl

runnable/exceptions.py

@@ -0,0 +1,94 @@
+class RunLogExistsError(Exception):  # pragma: no cover
+    """
+    Exception class
+    Args:
+        Exception ([type]): [description]
+    """
+
+    def __init__(self, run_id):
+        super().__init__()
+        self.message = f"Run id for {run_id} is already found in the datastore"
+
+
+class RunLogNotFoundError(Exception):  # pragma: no cover
+    """
+    Exception class
+    Args:
+        Exception ([type]): [description]
+    """
+
+    def __init__(self, run_id):
+        super().__init__()
+        self.message = f"Run id for {run_id} is not found in the datastore"
+
+
+class StepLogNotFoundError(Exception):  # pragma: no cover
+    """
+    Exception class
+    Args:
+        Exception ([type]): [description]
+    """
+
+    def __init__(self, run_id, name):
+        super().__init__()
+        self.message = f"Step log for {name} is not found in the datastore for Run id: {run_id}"
+
+
+class BranchLogNotFoundError(Exception):  # pragma: no cover
+    """
+    Exception class
+    Args:
+        Exception ([type]): [description]
+    """
+
+    def __init__(self, run_id, name):
+        super().__init__()
+        self.message = f"Branch log for {name} is not found in the datastore for Run id: {run_id}"
+
+
+class NodeNotFoundError(Exception):  # pragma: no cover
+    """
+    Exception class
+    Args:
+        Exception ([type]): [description]
+    """
+
+    def __init__(self, name):
+        super().__init__()
+        self.message = f"Node of name {name} is not found the graph"
+
+
+class BranchNotFoundError(Exception):  # pragma: no cover
+    """
+    Exception class
+    Args:
+        Exception ([type]): [description]
+    """
+
+    def __init__(self, name):
+        super().__init__()
+        self.message = f"Branch of name {name} is not found the graph"
+
+
+class TerminalNodeError(Exception):  # pragma: no cover
+    def __init__(self):
+        super().__init__()
+        self.message = "Terminal Nodes do not have next node"
+
+
+class SecretNotFoundError(Exception):  # pragma: no cover
+    """
+    Exception class
+    Args:
+        Exception ([type]): [description]
+    """
+
+    def __init__(self, secret_name, secret_setting):
+        super().__init__()
+        self.message = f"No secret found by name:{secret_name} in {secret_setting}"
+
+
+class ExecutionFailedError(Exception):  # pragma: no cover
+    def __init__(self, run_id: str):
+        super().__init__()
+        self.message = f"Execution failed for run id: {run_id}"

runnable/executor.py

@@ -0,0 +1,431 @@
+from __future__ import annotations
+
+import logging
+import os
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
+
+from pydantic import BaseModel, ConfigDict
+
+import runnable.context as context
+from runnable import defaults
+from runnable.datastore import DataCatalog, RunLog, StepLog
+from runnable.defaults import TypeMapVariable
+from runnable.graph import Graph
+
+if TYPE_CHECKING:  # pragma: no cover
+    from runnable.extensions.nodes import TaskNode
+    from runnable.nodes import BaseNode
+
+logger = logging.getLogger(defaults.LOGGER_NAME)
+
+
+class BaseExecutor(ABC, BaseModel):
+    """
+    The skeleton of an executor class.
+    Any implementation of an executor should inherit this class and over-ride accordingly.
+
+    There is a extension available in magnus/extensions/executor/__init__.py
+    which implements the most common functionality which is easier to
+    extend/override in most scenarios.
+
+    """
+
+    service_name: str = ""
+    service_type: str = "executor"
+
+    enable_parallel: bool = defaults.ENABLE_PARALLEL
+    overrides: dict = {}
+
+    _previous_run_log: Optional[RunLog] = None
+    _single_step: str = ""
+
+    _context_step_log = None  # type : StepLog
+    _context_node = None  # type: BaseNode
+    model_config = ConfigDict(extra="forbid")
+
+    @property
+    def _context(self):
+        return context.run_context
+
+    def _is_parallel_execution(self) -> bool:
+        """
+        Controls the parallelization of branches in map and parallel state.
+        Defaults to False and left for the compute modes to decide.
+
+        Interactive executors like local and local-container need decisions.
+        For most transpilers it is inconsequential as its always True and supported by platforms.
+
+        Returns:
+            bool: True if the execution allows parallel execution of branches.
+        """
+        return self.enable_parallel
+
+    @abstractmethod
+    def _get_parameters(self) -> Dict[str, Any]:
+        """
+        Get the parameters for the execution.
+        The parameters can be defined in parameters file and can be overridden by environment variables.
+
+        Returns:
+            Dict[str, Any]: The parameters for the execution.
+        """
+        ...
+
+    @abstractmethod
+    def _set_up_for_re_run(self, parameters: Dict[str, Any]) -> None:
+        """
+        Set up the executor for using a previous execution.
+
+        Retrieve the older run log, error out if it does not exist.
+        Sync the catalogs from the previous run log with the current one.
+
+        Update the parameters of this execution with the previous one. The previous one take precedence.
+
+        Args:
+            parameters (Dict[str, Any]): The parameters for the current execution.
+        """
+
+    @abstractmethod
+    def _set_up_run_log(self, exists_ok=False):
+        """
+        Create a run log and put that in the run log store
+
+        If exists_ok, we allow the run log to be already present in the run log store.
+        """
+        ...
+
+    @abstractmethod
+    def prepare_for_graph_execution(self):
+        """
+        This method should be called prior to calling execute_graph.
+        Perform any steps required before doing the graph execution.
+
+        The most common implementation is to prepare a run log for the run if the run uses local interactive compute.
+
+        But in cases of actual rendering the job specs (eg: AWS step functions, K8's) we check if the services are OK.
+        We do not set up a run log as its not relevant.
+        """
+        ...
+
+    @abstractmethod
+    def prepare_for_node_execution(self):
+        """
+        Perform any modifications to the services prior to execution of the node.
+
+        Args:
+            node (Node): [description]
+            map_variable (dict, optional): [description]. Defaults to None.
+        """
+        ...
+
+    @abstractmethod
+    def _sync_catalog(self, step_log: StepLog, stage: str, synced_catalogs=None) -> Optional[List[DataCatalog]]:
+        """
+        1). Identify the catalog settings by over-riding node settings with the global settings.
+        2). For stage = get:
+                Identify the catalog items that are being asked to get from the catalog
+                And copy them to the local compute data folder
+        3). For stage = put:
+                Identify the catalog items that are being asked to put into the catalog
+                Copy the items from local compute folder to the catalog
+        4). Add the items onto the step log according to the stage
+
+        Args:
+            node (Node): The current node being processed
+            step_log (StepLog): The step log corresponding to that node
+            stage (str): One of get or put
+
+        Raises:
+            Exception: If the stage is not in one of get/put
+
+        """
+        ...
+
+    @abstractmethod
+    def get_effective_compute_data_folder(self) -> Optional[str]:
+        """
+        Get the effective compute data folder for the given stage.
+        If there is nothing to catalog, we return None.
+
+        The default is the compute data folder of the catalog but this can be over-ridden by the node.
+
+        Args:
+            stage (str): The stage we are in the process of cataloging
+
+
+        Returns:
+            Optional[str]: The compute data folder as defined by catalog handler or the node or None.
+        """
+        ...
+
+    @property
+    def step_attempt_number(self) -> int:
+        """
+        The attempt number of the current step.
+        Orchestrators should use this step to submit multiple attempts of the job.
+
+        Returns:
+            int: The attempt number of the current step. Defaults to 1.
+        """
+        return int(os.environ.get(defaults.ATTEMPT_NUMBER, 1))
+
+    @abstractmethod
+    def _execute_node(self, node: BaseNode, map_variable: TypeMapVariable = None, **kwargs):
+        """
+        This is the entry point when we do the actual execution of the function.
+
+        While in interactive execution, we just compute, in 3rd party interactive execution, we need to reach
+        this function.
+
+        In most cases,
+            * We get the corresponding step_log of the node and the parameters.
+            * We sync the catalog to GET any data sets that are in the catalog
+            * We call the execute method of the node for the actual compute and retry it as many times as asked.
+            * If the node succeeds, we get any of the user defined metrics provided by the user.
+            * We sync the catalog to PUT any data sets that are in the catalog.
+
+        Args:
+            node (Node): The node to execute
+            map_variable (dict, optional): If the node is of a map state, map_variable is the value of the iterable.
+                        Defaults to None.
+        """
+        ...
+
+    @abstractmethod
+    def execute_node(self, node: BaseNode, map_variable: TypeMapVariable = None, **kwargs):
+        """
+        The entry point for all executors apart from local.
+        We have already prepared for node execution.
+
+        Args:
+            node (BaseNode): The node to execute
+            map_variable (dict, optional): If the node is part of a map, send in the map dictionary. Defaults to None.
+
+        Raises:
+            NotImplementedError: _description_
+        """
+        ...
+
+    @abstractmethod
+    def add_code_identities(self, node: BaseNode, step_log: StepLog, **kwargs):
+        """
+        Add code identities specific to the implementation.
+
+        The Base class has an implementation of adding git code identities.
+
+        Args:
+            step_log (object): The step log object
+            node (BaseNode): The node we are adding the step log for
+        """
+        ...
+
+    @abstractmethod
+    def execute_from_graph(self, node: BaseNode, map_variable: TypeMapVariable = None, **kwargs):
+        """
+        This is the entry point to from the graph execution.
+
+        While the self.execute_graph is responsible for traversing the graph, this function is responsible for
+        actual execution of the node.
+
+        If the node type is:
+            * task : We can delegate to _execute_node after checking the eligibility for re-run in cases of a re-run
+            * success: We can delegate to _execute_node
+            * fail: We can delegate to _execute_node
+
+        For nodes that are internally graphs:
+            * parallel: Delegate the responsibility of execution to the node.execute_as_graph()
+            * dag: Delegate the responsibility of execution to the node.execute_as_graph()
+            * map: Delegate the responsibility of execution to the node.execute_as_graph()
+
+        Transpilers will NEVER use this method and will NEVER call ths method.
+        This method should only be used by interactive executors.
+
+        Args:
+            node (Node): The node to execute
+            map_variable (dict, optional): If the node if of a map state, this corresponds to the value of iterable.
+                    Defaults to None.
+        """
+        ...
+
+    @abstractmethod
+    def trigger_job(self, node: BaseNode, map_variable: TypeMapVariable = None, **kwargs):
+        """
+        Executor specific way of triggering jobs when magnus does both traversal and execution
+
+        Transpilers will NEVER use this method and will NEVER call them.
+        Only interactive executors who need execute_from_graph will ever implement it.
+
+        Args:
+            node (BaseNode): The node to execute
+            map_variable (str, optional): If the node if of a map state, this corresponds to the value of iterable.
+                    Defaults to ''.
+
+        NOTE: We do not raise an exception as this method is not required by many extensions
+        """
+        ...
+
+    @abstractmethod
+    def _get_status_and_next_node_name(self, current_node: BaseNode, dag: Graph, map_variable: TypeMapVariable = None):
+        """
+        Given the current node and the graph, returns the name of the next node to execute.
+
+        The name is always relative the graph that the node resides in.
+
+        If the current node succeeded, we return the next node as per the graph.
+        If the current node failed, we return the on failure node of the node (if provided) or the global one.
+
+        Args:
+            current_node (BaseNode): The current node.
+            dag (Graph): The dag we are traversing.
+            map_variable (dict): If the node belongs to a map branch.
+        """
+
+        ...
+
+    @abstractmethod
+    def execute_graph(self, dag: Graph, map_variable: TypeMapVariable = None, **kwargs):
+        """
+        The parallelization is controlled by the nodes and not by this function.
+
+        Transpilers should over ride this method to do the translation of dag to the platform specific way.
+        Interactive methods should use this to traverse and execute the dag.
+            - Use execute_from_graph to handle sub-graphs
+
+        Logically the method should:
+            * Start at the dag.start_at of the dag.
+            * Call the self.execute_from_graph(node)
+            * depending upon the status of the execution, either move to the success node or failure node.
+
+        Args:
+            dag (Graph): The directed acyclic graph to traverse and execute.
+            map_variable (dict, optional): If the node if of a map state, this corresponds to the value of the iterable.
+                    Defaults to None.
+        """
+        ...
+
+    @abstractmethod
+    def _is_step_eligible_for_rerun(self, node: BaseNode, map_variable: TypeMapVariable = None):
+        """
+        In case of a re-run, this method checks to see if the previous run step status to determine if a re-run is
+        necessary.
+            * True: If its not a re-run.
+            * True: If its a re-run and we failed in the last run or the corresponding logs do not exist.
+            * False: If its a re-run and we succeeded in the last run.
+
+        Most cases, this logic need not be touched
+
+        Args:
+            node (Node): The node to check against re-run
+            map_variable (dict, optional): If the node if of a map state, this corresponds to the value of iterable..
+                        Defaults to None.
+
+        Returns:
+            bool: Eligibility for re-run. True means re-run, False means skip to the next step.
+        """
+        ...
+
+    @abstractmethod
+    def send_return_code(self, stage="traversal"):
+        """
+        Convenience function used by pipeline to send return code to the caller of the cli
+
+        Raises:
+            Exception: If the pipeline execution failed
+        """
+        ...
+
+    @abstractmethod
+    def _resolve_executor_config(self, node: BaseNode):
+        """
+        The overrides section can contain specific over-rides to an global executor config.
+        To avoid too much clutter in the dag definition, we allow the configuration file to have overrides block.
+        The nodes can over-ride the global config by referring to key in the overrides.
+
+        For example:
+        # configuration.yaml
+        execution:
+          type: cloud-implementation
+          config:
+            k1: v1
+            k3: v3
+            overrides:
+              k2: v2 # Could be a mapping internally.
+
+        # in pipeline definition.yaml
+        dag:
+          steps:
+            step1:
+              overrides:
+                cloud-implementation:
+                  k1: value_specific_to_node
+                  k2:
+
+        This method should resolve the node_config to {'k1': 'value_specific_to_node', 'k2': 'v2', 'k3': 'v3'}
+
+        Args:
+            node (BaseNode): The current node being processed.
+
+        """
+        ...
+
+    @abstractmethod
+    def execute_job(self, node: TaskNode):
+        """
+        Executor specific way of executing a job (python function or a notebook).
+
+        Interactive executors should execute the job.
+        Transpilers should write the instructions.
+
+        Args:
+            node (BaseNode): The job node to execute
+
+        Raises:
+            NotImplementedError: Executors should choose to extend this functionality or not.
+        """
+        ...
+
+    @abstractmethod
+    def fan_out(self, node: BaseNode, map_variable: TypeMapVariable = None):
+        """
+        This method is used to appropriately fan-out the execution of a composite node.
+        This is only useful when we want to execute a composite node during 3rd party orchestrators.
+
+        Reason: Transpilers typically try to run the leaf nodes but do not have any capacity to do anything for the
+        step which is composite. By calling this fan-out before calling the leaf nodes, we have an opportunity to
+        do the right set up (creating the step log, exposing the parameters, etc.) for the composite step.
+
+        All 3rd party orchestrators should use this method to fan-out the execution of a composite node.
+        This ensures:
+            - The dot path notation is preserved, this method should create the step and call the node's fan out to
+            create the branch logs and let the 3rd party do the actual step execution.
+            - Gives 3rd party orchestrators an opportunity to set out the required for running a composite node.
+
+        Args:
+            node (BaseNode): The node to fan-out
+            map_variable (dict, optional): If the node if of a map state,.Defaults to None.
+
+        """
+        ...
+
+    @abstractmethod
+    def fan_in(self, node: BaseNode, map_variable: TypeMapVariable = None):
+        """
+        This method is used to appropriately fan-in after the execution of a composite node.
+        This is only useful when we want to execute a composite node during 3rd party orchestrators.
+
+        Reason: Transpilers typically try to run the leaf nodes but do not have any capacity to do anything for the
+        step which is composite. By calling this fan-in after calling the leaf nodes, we have an opportunity to
+        act depending upon the status of the individual branches.
+
+        All 3rd party orchestrators should use this method to fan-in the execution of a composite node.
+        This ensures:
+            - Gives the renderer's the control on where to go depending upon the state of the composite node.
+            - The status of the step and its underlying branches are correctly updated.
+
+        Args:
+            node (BaseNode): The node to fan-in
+            map_variable (dict, optional): If the node if of a map state,.Defaults to None.
+
+        """
+        ...

runnable/experiment_tracker.py

@@ -0,0 +1,139 @@
+import contextlib
+import json
+import logging
+import os
+from abc import ABC, abstractmethod
+from collections import defaultdict
+from typing import Any, ContextManager, Dict, Tuple, Union
+
+from pydantic import BaseModel, ConfigDict
+
+import runnable.context as context
+from runnable import defaults
+from runnable.utils import remove_prefix
+
+logger = logging.getLogger(defaults.LOGGER_NAME)
+
+
+def retrieve_step_details(key: str) -> Tuple[str, int]:
+    key = remove_prefix(key, defaults.TRACK_PREFIX)
+    data = key.split(defaults.STEP_INDICATOR)
+
+    key = data[0].lower()
+    step = 0
+
+    if len(data) > 1:
+        step = int(data[1])
+
+    return key, step
+
+
+def get_tracked_data() -> Dict[str, Any]:
+    tracked_data: Dict[str, Any] = defaultdict(dict)
+    for env_var, value in os.environ.items():
+        if env_var.startswith(defaults.TRACK_PREFIX):
+            key, step = retrieve_step_details(env_var)
+
+            # print(value, type(value))
+            try:
+                value = json.loads(value)
+            except json.decoder.JSONDecodeError:
+                logger.warning(f"Tracker {key} could not be JSON decoded, adding the literal value")
+
+            tracked_data[key][step] = value
+            del os.environ[env_var]
+
+    for key, value in tracked_data.items():
+        if len(value) == 1:
+            tracked_data[key] = value[0]
+
+    return tracked_data
+
+
+# --8<-- [start:docs]
+
+
+class BaseExperimentTracker(ABC, BaseModel):
+    """
+    Base Experiment tracker class definition.
+    """
+
+    service_name: str = ""
+    service_type: str = "experiment_tracker"
+
+    @property
+    def _context(self):
+        return context.run_context
+
+    model_config = ConfigDict(extra="forbid")
+
+    @property
+    def client_context(self) -> ContextManager:
+        """
+        Returns the client context.
+        """
+        return contextlib.nullcontext()
+
+    def publish_data(self, tracked_data: Dict[str, Any]):
+        for key, value in tracked_data.items():
+            if isinstance(value, dict):
+                for key2, value2 in value.items():
+                    self.log_metric(key, value2, step=key2)
+                continue
+            self.log_metric(key, value)
+
+    @abstractmethod
+    def log_metric(self, key: str, value: Union[int, float], step: int = 0):
+        """
+        Sets the metric in the experiment tracking.
+
+        Args:
+            key (str): The key against you want to store the value
+            value (float): The value of the metric
+            step (int): Optional step at which it was recorded
+
+        Raises:
+            NotImplementedError: Base class, hence not implemented
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def log_parameter(self, key: str, value: Any):
+        """
+        Logs a parameter in the experiment tracking.
+
+        Args:
+            key (str): The key against you want to store the value
+            value (any): The value of the metric
+
+        Raises:
+            NotImplementedError: Base class, hence not implemented
+        """
+        pass
+
+
+# --8<-- [end:docs]
+
+
+class DoNothingTracker(BaseExperimentTracker):
+    """
+    A Do nothing tracker
+    """
+
+    service_name: str = "do-nothing"
+
+    def log_metric(self, key: str, value: Union[int, float], step: int = 0):
+        """
+        Sets the metric in the experiment tracking.
+
+        Args:
+            key (str): The key against you want to store the value
+            value (float): The value of the metric
+        """
+        ...
+
+    def log_parameter(self, key: str, value: Any):
+        """
+        Since this is a Do nothing tracker, we don't need to log anything.
+        """
+        ...

runnable/extensions/catalog/__init__.py

@@ -0,0 +1,21 @@
+from typing import List, Optional
+
+from runnable.datastore import DataCatalog
+
+
+def is_catalog_out_of_sync(catalog, synced_catalogs=Optional[List[DataCatalog]]) -> bool:
+    """
+    Check if the catalog items are out of sync from already cataloged objects.
+    If they are, return False.
+    If the object does not exist or synced catalog does not exist, return True
+    """
+    if not synced_catalogs:
+        return True  # If nothing has been synced in the past
+
+    for synced_catalog in synced_catalogs:
+        if synced_catalog.catalog_relative_path == catalog.catalog_relative_path:
+            if synced_catalog.data_hash == catalog.data_hash:
+                return False
+            return True
+
+    return True  # The object does not exist, sync it