runnable 0.34.0a3__py3-none-any.whl → 0.36.0__py3-none-any.whl

extensions/job_executor/__init__.py

@@ -84,7 +84,6 @@ class GenericJobExecutor(BaseJobExecutor):
             run_id=self._context.run_id,
             tag=self._context.tag,
             status=defaults.PROCESSING,
-            dag_hash=self._context.dag_hash,
         )
         # Any interaction with run log store attributes should happen via API if available.
         self._context.run_log_store.set_parameters(
@@ -92,7 +91,7 @@ class GenericJobExecutor(BaseJobExecutor):
         )
 
         # Update run_config
-        run_config = utils.get_run_config()
+        run_config = self._context.model_dump()
         logger.debug(f"run_config as seen by executor: {run_config}")
         self._context.run_log_store.set_run_config(
             run_id=self._context.run_id, run_config=run_config
@@ -147,7 +146,7 @@ class GenericJobExecutor(BaseJobExecutor):
 
         data_catalogs = []
         for name_pattern in catalog_settings:
-            data_catalog = self._context.catalog_handler.put(
+            data_catalog = self._context.catalog.put(
                 name=name_pattern, allow_file_not_found_exc=allow_file_not_found_exc
             )
 
@@ -165,5 +164,5 @@ class GenericJobExecutor(BaseJobExecutor):
         )
         task_console.save_text(log_file_name)
         # Put the log file in the catalog
-        self._context.catalog_handler.put(name=log_file_name)
+        self._context.catalog.put(name=log_file_name)
         os.remove(log_file_name)

extensions/job_executor/emulate.py

@@ -0,0 +1,106 @@
+import logging
+import shlex
+import subprocess
+import sys
+from typing import List, Optional
+
+
+from extensions.job_executor import GenericJobExecutor
+from runnable import console, context, defaults
+from runnable.datastore import DataCatalog
+from runnable.tasks import BaseTaskType
+
+logger = logging.getLogger(defaults.LOGGER_NAME)
+
+
+class EmulatorJobExecutor(GenericJobExecutor):
+    """
+    The EmulatorJobExecutor is a job executor that emulates the job execution.
+    """
+
+    service_name: str = "emulator"
+
+    def submit_job(self, job: BaseTaskType, catalog_settings=Optional[List[str]]):
+        """
+        This method gets invoked by the CLI.
+        """
+        self._set_up_run_log()
+
+        # Call the container job
+        job_log = self._context.run_log_store.create_job_log()
+        self._context.run_log_store.add_job_log(
+            run_id=self._context.run_id, job_log=job_log
+        )
+        self.run_click_command()
+
+    def execute_job(self, job: BaseTaskType, catalog_settings=Optional[List[str]]):
+        """
+        Focusses on execution of the job.
+        """
+        logger.info("Trying to execute job")
+
+        job_log = self._context.run_log_store.get_job_log(run_id=self._context.run_id)
+        self.add_code_identities(job_log)
+
+        attempt_log = job.execute_command()
+
+        job_log.status = attempt_log.status
+        job_log.attempts.append(attempt_log)
+
+        allow_file_not_found_exc = True
+        if job_log.status == defaults.SUCCESS:
+            allow_file_not_found_exc = False
+
+        data_catalogs_put: Optional[List[DataCatalog]] = self._sync_catalog(
+            catalog_settings=catalog_settings,
+            allow_file_not_found_exc=allow_file_not_found_exc,
+        )
+
+        logger.debug(f"data_catalogs_put: {data_catalogs_put}")
+
+        job_log.add_data_catalogs(data_catalogs_put or [])
+
+        console.print("Summary of job")
+        console.print(job_log.get_summary())
+
+        self._context.run_log_store.add_job_log(
+            run_id=self._context.run_id, job_log=job_log
+        )
+
+    def run_click_command(self) -> str:
+        """
+        Execute a Click-based CLI command in the current virtual environment.
+
+        Args:
+            command: Click command to execute
+        """
+        assert isinstance(self._context, context.JobContext)
+        command = self._context.get_job_callable_command()
+
+        sub_command = [sys.executable, "-m", "runnable.cli"] + shlex.split(command)[1:]
+
+        process = subprocess.Popen(
+            sub_command,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            universal_newlines=True,
+            bufsize=1,
+        )
+
+        output = []
+        try:
+            while True:
+                line = process.stdout.readline()  # type: ignore
+                if not line and process.poll() is not None:
+                    break
+                print(line, end="")
+                output.append(line)
+        finally:
+            process.stdout.close()  # type: ignore
+
+        if process.returncode != 0:
+            raise subprocess.CalledProcessError(
+                process.returncode, command, "".join(output)
+            )
+
+        return "".join(output)

extensions/job_executor/k8s.py

@@ -9,7 +9,7 @@ from pydantic import BaseModel, ConfigDict, Field, PlainSerializer, PrivateAttr
 from pydantic.alias_generators import to_camel
 
 from extensions.job_executor import GenericJobExecutor
-from runnable import console, defaults, utils
+from runnable import console, context, defaults
 from runnable.datastore import DataCatalog, StepAttempt
 from runnable.tasks import BaseTaskType
 
@@ -173,7 +173,7 @@ class GenericK8sJobExecutor(GenericJobExecutor):
     mock: bool = False
     namespace: str = Field(default="default")
 
-    _is_local: bool = PrivateAttr(default=False)
+    _should_setup_run_log_at_traversal: bool = PrivateAttr(default=False)
     _volume_mounts: list[VolumeMount] = PrivateAttr(default_factory=lambda: [])
     _volumes: list[HostPathVolume | PVCVolume] = PrivateAttr(default_factory=lambda: [])
 
@@ -258,8 +258,8 @@ class GenericK8sJobExecutor(GenericJobExecutor):
             self._client.V1VolumeMount(**vol.model_dump())
             for vol in self._volume_mounts
         ]
-
-        command = utils.get_job_execution_command()
+        assert isinstance(self._context, context.JobContext)
+        command = self._context.get_job_callable_command()
 
         container_env = [
             self._client.V1EnvVar(**env.model_dump())
@@ -355,9 +355,9 @@ class GenericK8sJobExecutor(GenericJobExecutor):
             case "chunked-fs":
                 self._context.run_log_store.log_folder = self._container_log_location
 
-        match self._context.catalog_handler.service_name:
+        match self._context.catalog.service_name:
             case "file-system":
-                self._context.catalog_handler.catalog_location = (
+                self._context.catalog.catalog_location = (
                     self._container_catalog_location
                 )
 
@@ -415,7 +415,7 @@ class MiniK8sJobExecutor(GenericK8sJobExecutor):
                     )
                 )
 
-        match self._context.catalog_handler.service_name:
+        match self._context.catalog.service_name:
             case "file-system":
                 self._volumes.append(
                     HostPathVolume(
@@ -503,7 +503,7 @@ class K8sJobExecutor(GenericK8sJobExecutor):
                     )
                 )
 
-        match self._context.catalog_handler.service_name:
+        match self._context.catalog.service_name:
             case "file-system":
                 self._volume_mounts.append(
                     VolumeMount(

extensions/job_executor/local_container.py

@@ -2,10 +2,10 @@ import logging
 from pathlib import Path
 from typing import Dict, List, Optional
 
-from pydantic import Field
+from pydantic import Field, PrivateAttr
 
 from extensions.job_executor import GenericJobExecutor
-from runnable import console, defaults, utils
+from runnable import console, context, defaults
 from runnable.datastore import DataCatalog, StepAttempt
 from runnable.tasks import BaseTaskType
 
@@ -23,7 +23,7 @@ class LocalContainerJobExecutor(GenericJobExecutor):
     auto_remove_container: bool = True
     environment: Dict[str, str] = Field(default_factory=dict)
 
-    _is_local: bool = False
+    _should_setup_run_log_at_traversal: bool = PrivateAttr(default=True)
 
     _container_log_location = "/tmp/run_logs/"
     _container_catalog_location = "/tmp/catalog/"
@@ -100,7 +100,8 @@ class LocalContainerJobExecutor(GenericJobExecutor):
             ) from ex
 
         try:
-            command = utils.get_job_execution_command()
+            assert isinstance(self._context, context.JobContext)
+            command = self._context.get_job_callable_command()
             logger.info(f"Running the command {command}")
             print(command)
 
@@ -165,17 +166,17 @@ class LocalContainerJobExecutor(GenericJobExecutor):
                     "mode": "rw",
                 }
 
-        match self._context.catalog_handler.service_name:
+        match self._context.catalog.service_name:
             case "file-system":
-                catalog_location = self._context.catalog_handler.catalog_location
+                catalog_location = self._context.catalog.catalog_location
                 self._volumes[str(Path(catalog_location).resolve())] = {
                     "bind": f"{self._container_catalog_location}",
                     "mode": "rw",
                 }
 
-        match self._context.secrets_handler.service_name:
+        match self._context.secrets.service_name:
             case "dotenv":
-                secrets_location = self._context.secrets_handler.location
+                secrets_location = self._context.secrets.location
                 self._volumes[str(Path(secrets_location).resolve())] = {
                     "bind": f"{self._container_secrets_location}",
                     "mode": "ro",
@@ -188,14 +189,12 @@ class LocalContainerJobExecutor(GenericJobExecutor):
             case "chunked-fs":
                 self._context.run_log_store.log_folder = self._container_log_location
 
-        match self._context.catalog_handler.service_name:
+        match self._context.catalog.service_name:
             case "file-system":
-                self._context.catalog_handler.catalog_location = (
+                self._context.catalog.catalog_location = (
                     self._container_catalog_location
                 )
 
-        match self._context.secrets_handler.service_name:
+        match self._context.secrets.service_name:
             case "dotenv":
-                self._context.secrets_handler.location = (
-                    self._container_secrets_location
-                )
+                self._context.secrets.location = self._container_secrets_location

extensions/nodes/conditional.py

@@ -0,0 +1,243 @@
+import logging
+from copy import deepcopy
+from typing import Any, cast
+
+from pydantic import Field, field_serializer, field_validator
+
+from runnable import console, defaults
+from runnable.datastore import Parameter
+from runnable.graph import Graph, create_graph
+from runnable.nodes import CompositeNode, MapVariableType
+
+logger = logging.getLogger(defaults.LOGGER_NAME)
+
+
+class ConditionalNode(CompositeNode):
+    """
+    parameter: name -> the parameter which is used for evaluation
+    default: Optional[branch] = branch to execute if nothing is matched.
+    branches: {
+        "case1" : branch1,
+        "case2: branch2,
+    }
+
+    Conceptually this is equal to:
+    match parameter:
+        case "case1":
+            branch1
+        case "case2":
+            branch2
+        case _:
+            default
+
+    """
+
+    node_type: str = Field(default="conditional", serialization_alias="type")
+
+    parameter: str  # the name of the parameter should be isalnum
+    default: Graph | None = Field(default=None)  # TODO: Think about the design of this
+    branches: dict[str, Graph]
+    # The keys of the branches should be isalnum()
+
+    @field_validator("parameter", mode="after")
+    @classmethod
+    def check_parameter(cls, parameter: str) -> str:
+        """
+        Validate that the parameter name is alphanumeric.
+
+        Args:
+            parameter (str): The parameter name to validate.
+
+        Raises:
+            ValueError: If the parameter name is not alphanumeric.
+
+        Returns:
+            str: The validated parameter name.
+        """
+        if not parameter.isalnum():
+            raise ValueError(f"Parameter '{parameter}' must be alphanumeric.")
+        return parameter
+
+    def get_parameter_value(self) -> str | int | bool | float:
+        """
+        Get the parameter value from the context.
+
+        Returns:
+            Any: The value of the parameter.
+        """
+        parameters: dict[str, Parameter] = self._context.run_log_store.get_parameters(
+            run_id=self._context.run_id
+        )
+
+        if self.parameter not in parameters:
+            raise Exception(f"Parameter {self.parameter} not found in parameters")
+
+        chosen_parameter_value = parameters[self.parameter].get_value()
+
+        assert isinstance(chosen_parameter_value, (int, float, bool, str)), (
+            f"Parameter '{self.parameter}' must be of type int, float, bool, or str, "
+            f"but got {type(chosen_parameter_value).__name__}."
+        )
+
+        return chosen_parameter_value
+
+    def get_summary(self) -> dict[str, Any]:
+        summary = {
+            "name": self.name,
+            "type": self.node_type,
+            "branches": [branch.get_summary() for branch in self.branches.values()],
+            "parameter": self.parameter,
+            "default": self.default.get_summary() if self.default else None,
+        }
+
+        return summary
+
+    @field_serializer("branches")
+    def ser_branches(self, branches: dict[str, Graph]) -> dict[str, Graph]:
+        ret: dict[str, Graph] = {}
+
+        for branch_name, branch in branches.items():
+            ret[branch_name.split(".")[-1]] = branch
+
+        return ret
+
+    @classmethod
+    def parse_from_config(cls, config: dict[str, Any]) -> "ConditionalNode":
+        internal_name = cast(str, config.get("internal_name"))
+
+        config_branches = config.pop("branches", {})
+        branches = {}
+        for branch_name, branch_config in config_branches.items():
+            sub_graph = create_graph(
+                deepcopy(branch_config),
+                internal_branch_name=internal_name + "." + branch_name,
+            )
+            branches[internal_name + "." + branch_name] = sub_graph
+
+        if not branches:
+            raise Exception("A parallel node should have branches")
+        return cls(branches=branches, **config)
+
+    def _get_branch_by_name(self, branch_name: str) -> Graph:
+        if branch_name in self.branches:
+            return self.branches[branch_name]
+
+        raise Exception(f"Branch {branch_name} does not exist")
+
+    def fan_out(self, map_variable: MapVariableType = None):
+        """
+        This method is restricted to creating branch logs.
+        """
+        parameter_value = self.get_parameter_value()
+
+        hit_once = False
+
+        for internal_branch_name, _ in self.branches.items():
+            # the match is done on the last part of the branch name
+            result = str(parameter_value) == internal_branch_name.split(".")[-1]
+
+            if not result:
+                # Need not create a branch log for this branch
+                continue
+
+            effective_branch_name = self._resolve_map_placeholders(
+                internal_branch_name, map_variable=map_variable
+            )
+
+            hit_once = True
+            branch_log = self._context.run_log_store.create_branch_log(
+                effective_branch_name
+            )
+
+            console.print(
+                f"Branch log created for {effective_branch_name}: {branch_log}"
+            )
+            branch_log.status = defaults.PROCESSING
+            self._context.run_log_store.add_branch_log(branch_log, self._context.run_id)
+
+        if not hit_once:
+            raise Exception(
+                "None of the branches were true. Please check your evaluate statements"
+            )
+
+    def execute_as_graph(self, map_variable: MapVariableType = None):
+        """
+        This function does the actual execution of the sub-branches of the parallel node.
+
+        From a design perspective, this function should not be called if the execution is 3rd party orchestrated.
+
+        The modes that render the job specifications, do not need to interact with this node at all as they have their
+        own internal mechanisms of handing parallel states.
+        If they do not, you can find a way using as-is nodes as hack nodes.
+
+        The execution of a dag, could result in
+            * The dag being completely executed with a definite (fail, success) state in case of
+                local or local-container execution
+            * The dag being in a processing state with PROCESSING status in case of local-aws-batch
+
+        Only fail state is considered failure during this phase of execution.
+
+        Args:
+            executor (Executor): The Executor as per the use config
+            **kwargs: Optional kwargs passed around
+        """
+        self.fan_out(map_variable=map_variable)
+        parameter_value = self.get_parameter_value()
+
+        for internal_branch_name, branch in self.branches.items():
+            result = str(parameter_value) == internal_branch_name.split(".")[-1]
+
+            if result:
+                # if the condition is met, execute the graph
+                logger.debug(f"Executing graph for {branch}")
+                self._context.pipeline_executor.execute_graph(
+                    branch, map_variable=map_variable
+                )
+
+        self.fan_in(map_variable=map_variable)
+
+    def fan_in(self, map_variable: MapVariableType = None):
+        """
+        The general fan in method for a node of type Parallel.
+
+        3rd party orchestrators should use this method to find the status of the composite step.
+
+        Args:
+            executor (BaseExecutor): The executor class as defined by the config
+            map_variable (dict, optional): If the node is part of a map. Defaults to None.
+        """
+        effective_internal_name = self._resolve_map_placeholders(
+            self.internal_name, map_variable=map_variable
+        )
+
+        step_success_bool: bool = True
+        parameter_value = self.get_parameter_value()
+
+        for internal_branch_name, _ in self.branches.items():
+            result = str(parameter_value) == internal_branch_name.split(".")[-1]
+
+            if not result:
+                # The branch would not have been executed
+                continue
+
+            effective_branch_name = self._resolve_map_placeholders(
+                internal_branch_name, map_variable=map_variable
+            )
+
+            branch_log = self._context.run_log_store.get_branch_log(
+                effective_branch_name, self._context.run_id
+            )
+
+            if branch_log.status != defaults.SUCCESS:
+                step_success_bool = False
+
+        step_log = self._context.run_log_store.get_step_log(
+            effective_internal_name, self._context.run_id
+        )
+
+        if step_success_bool:  #  If none failed
+            step_log.status = defaults.SUCCESS
+        else:
+            step_log.status = defaults.FAIL
+
+        self._context.run_log_store.add_step_log(step_log, self._context.run_id)

extensions/nodes/fail.py

@@ -0,0 +1,72 @@
+from datetime import datetime
+from typing import Any, Dict, cast
+
+from pydantic import Field
+
+from runnable import datastore, defaults
+from runnable.datastore import StepLog
+from runnable.defaults import MapVariableType
+from runnable.nodes import TerminalNode
+
+
+class FailNode(TerminalNode):
+    """
+    A leaf node of the graph that represents a failure node
+    """
+
+    node_type: str = Field(default="fail", serialization_alias="type")
+
+    @classmethod
+    def parse_from_config(cls, config: Dict[str, Any]) -> "FailNode":
+        return cast("FailNode", super().parse_from_config(config))
+
+    def get_summary(self) -> Dict[str, Any]:
+        summary = {
+            "name": self.name,
+            "type": self.node_type,
+        }
+
+        return summary
+
+    def execute(
+        self,
+        mock=False,
+        map_variable: MapVariableType = None,
+        attempt_number: int = 1,
+    ) -> StepLog:
+        """
+        Execute the failure node.
+        Set the run or branch log status to failure.
+
+        Args:
+            executor (_type_): the executor class
+            mock (bool, optional): If we should just mock and not do the actual execution. Defaults to False.
+            map_variable (dict, optional): If the node belongs to internal branches. Defaults to None.
+
+        Returns:
+            StepAttempt: The step attempt object
+        """
+        step_log = self._context.run_log_store.get_step_log(
+            self._get_step_log_name(map_variable), self._context.run_id
+        )
+
+        attempt_log = datastore.StepAttempt(
+            status=defaults.SUCCESS,
+            start_time=str(datetime.now()),
+            end_time=str(datetime.now()),
+            attempt_number=attempt_number,
+        )
+
+        run_or_branch_log = self._context.run_log_store.get_branch_log(
+            self._get_branch_log_name(map_variable), self._context.run_id
+        )
+        run_or_branch_log.status = defaults.FAIL
+        self._context.run_log_store.add_branch_log(
+            run_or_branch_log, self._context.run_id
+        )
+
+        step_log.status = attempt_log.status
+
+        step_log.attempts.append(attempt_log)
+
+        return step_log