runnable 0.35.0__py3-none-any.whl → 0.36.0__py3-none-any.whl

extensions/job_executor/__init__.py

@@ -84,7 +84,6 @@ class GenericJobExecutor(BaseJobExecutor):
             run_id=self._context.run_id,
             tag=self._context.tag,
             status=defaults.PROCESSING,
-            dag_hash=self._context.dag_hash,
         )
         # Any interaction with run log store attributes should happen via API if available.
         self._context.run_log_store.set_parameters(
@@ -92,7 +91,7 @@ class GenericJobExecutor(BaseJobExecutor):
         )
 
         # Update run_config
-        run_config = utils.get_run_config()
+        run_config = self._context.model_dump()
         logger.debug(f"run_config as seen by executor: {run_config}")
         self._context.run_log_store.set_run_config(
             run_id=self._context.run_id, run_config=run_config
@@ -147,7 +146,7 @@ class GenericJobExecutor(BaseJobExecutor):
 
         data_catalogs = []
         for name_pattern in catalog_settings:
-            data_catalog = self._context.catalog_handler.put(
+            data_catalog = self._context.catalog.put(
                 name=name_pattern, allow_file_not_found_exc=allow_file_not_found_exc
             )
 
@@ -165,5 +164,5 @@ class GenericJobExecutor(BaseJobExecutor):
         )
         task_console.save_text(log_file_name)
         # Put the log file in the catalog
-        self._context.catalog_handler.put(name=log_file_name)
+        self._context.catalog.put(name=log_file_name)
         os.remove(log_file_name)

extensions/job_executor/emulate.py

@@ -0,0 +1,106 @@
+import logging
+import shlex
+import subprocess
+import sys
+from typing import List, Optional
+
+
+from extensions.job_executor import GenericJobExecutor
+from runnable import console, context, defaults
+from runnable.datastore import DataCatalog
+from runnable.tasks import BaseTaskType
+
+logger = logging.getLogger(defaults.LOGGER_NAME)
+
+
+class EmulatorJobExecutor(GenericJobExecutor):
+    """
+    The EmulatorJobExecutor is a job executor that emulates the job execution.
+    """
+
+    service_name: str = "emulator"
+
+    def submit_job(self, job: BaseTaskType, catalog_settings=Optional[List[str]]):
+        """
+        This method gets invoked by the CLI.
+        """
+        self._set_up_run_log()
+
+        # Call the container job
+        job_log = self._context.run_log_store.create_job_log()
+        self._context.run_log_store.add_job_log(
+            run_id=self._context.run_id, job_log=job_log
+        )
+        self.run_click_command()
+
+    def execute_job(self, job: BaseTaskType, catalog_settings=Optional[List[str]]):
+        """
+        Focusses on execution of the job.
+        """
+        logger.info("Trying to execute job")
+
+        job_log = self._context.run_log_store.get_job_log(run_id=self._context.run_id)
+        self.add_code_identities(job_log)
+
+        attempt_log = job.execute_command()
+
+        job_log.status = attempt_log.status
+        job_log.attempts.append(attempt_log)
+
+        allow_file_not_found_exc = True
+        if job_log.status == defaults.SUCCESS:
+            allow_file_not_found_exc = False
+
+        data_catalogs_put: Optional[List[DataCatalog]] = self._sync_catalog(
+            catalog_settings=catalog_settings,
+            allow_file_not_found_exc=allow_file_not_found_exc,
+        )
+
+        logger.debug(f"data_catalogs_put: {data_catalogs_put}")
+
+        job_log.add_data_catalogs(data_catalogs_put or [])
+
+        console.print("Summary of job")
+        console.print(job_log.get_summary())
+
+        self._context.run_log_store.add_job_log(
+            run_id=self._context.run_id, job_log=job_log
+        )
+
+    def run_click_command(self) -> str:
+        """
+        Execute a Click-based CLI command in the current virtual environment.
+
+        Args:
+            command: Click command to execute
+        """
+        assert isinstance(self._context, context.JobContext)
+        command = self._context.get_job_callable_command()
+
+        sub_command = [sys.executable, "-m", "runnable.cli"] + shlex.split(command)[1:]
+
+        process = subprocess.Popen(
+            sub_command,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            universal_newlines=True,
+            bufsize=1,
+        )
+
+        output = []
+        try:
+            while True:
+                line = process.stdout.readline()  # type: ignore
+                if not line and process.poll() is not None:
+                    break
+                print(line, end="")
+                output.append(line)
+        finally:
+            process.stdout.close()  # type: ignore
+
+        if process.returncode != 0:
+            raise subprocess.CalledProcessError(
+                process.returncode, command, "".join(output)
+            )
+
+        return "".join(output)

extensions/job_executor/k8s.py

@@ -9,7 +9,7 @@ from pydantic import BaseModel, ConfigDict, Field, PlainSerializer, PrivateAttr
 from pydantic.alias_generators import to_camel
 
 from extensions.job_executor import GenericJobExecutor
-from runnable import console, defaults, utils
+from runnable import console, context, defaults
 from runnable.datastore import DataCatalog, StepAttempt
 from runnable.tasks import BaseTaskType
 
@@ -173,7 +173,7 @@ class GenericK8sJobExecutor(GenericJobExecutor):
     mock: bool = False
     namespace: str = Field(default="default")
 
-    _is_local: bool = PrivateAttr(default=False)
+    _should_setup_run_log_at_traversal: bool = PrivateAttr(default=False)
     _volume_mounts: list[VolumeMount] = PrivateAttr(default_factory=lambda: [])
     _volumes: list[HostPathVolume | PVCVolume] = PrivateAttr(default_factory=lambda: [])
 
@@ -258,8 +258,8 @@ class GenericK8sJobExecutor(GenericJobExecutor):
             self._client.V1VolumeMount(**vol.model_dump())
             for vol in self._volume_mounts
         ]
-
-        command = utils.get_job_execution_command()
+        assert isinstance(self._context, context.JobContext)
+        command = self._context.get_job_callable_command()
 
         container_env = [
             self._client.V1EnvVar(**env.model_dump())
@@ -355,9 +355,9 @@ class GenericK8sJobExecutor(GenericJobExecutor):
             case "chunked-fs":
                 self._context.run_log_store.log_folder = self._container_log_location
 
-        match self._context.catalog_handler.service_name:
+        match self._context.catalog.service_name:
             case "file-system":
-                self._context.catalog_handler.catalog_location = (
+                self._context.catalog.catalog_location = (
                     self._container_catalog_location
                 )
 
@@ -415,7 +415,7 @@ class MiniK8sJobExecutor(GenericK8sJobExecutor):
                     )
                 )
 
-        match self._context.catalog_handler.service_name:
+        match self._context.catalog.service_name:
             case "file-system":
                 self._volumes.append(
                     HostPathVolume(
@@ -503,7 +503,7 @@ class K8sJobExecutor(GenericK8sJobExecutor):
                     )
                 )
 
-        match self._context.catalog_handler.service_name:
+        match self._context.catalog.service_name:
             case "file-system":
                 self._volume_mounts.append(
                     VolumeMount(

extensions/job_executor/local_container.py

@@ -2,10 +2,10 @@ import logging
 from pathlib import Path
 from typing import Dict, List, Optional
 
-from pydantic import Field
+from pydantic import Field, PrivateAttr
 
 from extensions.job_executor import GenericJobExecutor
-from runnable import console, defaults, utils
+from runnable import console, context, defaults
 from runnable.datastore import DataCatalog, StepAttempt
 from runnable.tasks import BaseTaskType
 
@@ -23,7 +23,7 @@ class LocalContainerJobExecutor(GenericJobExecutor):
     auto_remove_container: bool = True
     environment: Dict[str, str] = Field(default_factory=dict)
 
-    _is_local: bool = False
+    _should_setup_run_log_at_traversal: bool = PrivateAttr(default=True)
 
     _container_log_location = "/tmp/run_logs/"
     _container_catalog_location = "/tmp/catalog/"
@@ -100,7 +100,8 @@ class LocalContainerJobExecutor(GenericJobExecutor):
             ) from ex
 
         try:
-            command = utils.get_job_execution_command()
+            assert isinstance(self._context, context.JobContext)
+            command = self._context.get_job_callable_command()
             logger.info(f"Running the command {command}")
             print(command)
 
@@ -165,17 +166,17 @@ class LocalContainerJobExecutor(GenericJobExecutor):
                     "mode": "rw",
                 }
 
-        match self._context.catalog_handler.service_name:
+        match self._context.catalog.service_name:
             case "file-system":
-                catalog_location = self._context.catalog_handler.catalog_location
+                catalog_location = self._context.catalog.catalog_location
                 self._volumes[str(Path(catalog_location).resolve())] = {
                     "bind": f"{self._container_catalog_location}",
                     "mode": "rw",
                 }
 
-        match self._context.secrets_handler.service_name:
+        match self._context.secrets.service_name:
             case "dotenv":
-                secrets_location = self._context.secrets_handler.location
+                secrets_location = self._context.secrets.location
                 self._volumes[str(Path(secrets_location).resolve())] = {
                     "bind": f"{self._container_secrets_location}",
                     "mode": "ro",
@@ -188,14 +189,12 @@ class LocalContainerJobExecutor(GenericJobExecutor):
             case "chunked-fs":
                 self._context.run_log_store.log_folder = self._container_log_location
 
-        match self._context.catalog_handler.service_name:
+        match self._context.catalog.service_name:
             case "file-system":
-                self._context.catalog_handler.catalog_location = (
+                self._context.catalog.catalog_location = (
                     self._container_catalog_location
                 )
 
-        match self._context.secrets_handler.service_name:
+        match self._context.secrets.service_name:
             case "dotenv":
-                self._context.secrets_handler.location = (
-                    self._container_secrets_location
-                )
+                self._context.secrets.location = self._container_secrets_location

extensions/nodes/conditional.py

@@ -7,7 +7,7 @@ from pydantic import Field, field_serializer, field_validator
 from runnable import console, defaults
 from runnable.datastore import Parameter
 from runnable.graph import Graph, create_graph
-from runnable.nodes import CompositeNode, TypeMapVariable
+from runnable.nodes import CompositeNode, MapVariableType
 
 logger = logging.getLogger(defaults.LOGGER_NAME)
 
@@ -124,7 +124,7 @@ class ConditionalNode(CompositeNode):
 
         raise Exception(f"Branch {branch_name} does not exist")
 
-    def fan_out(self, map_variable: TypeMapVariable = None):
+    def fan_out(self, map_variable: MapVariableType = None):
         """
         This method is restricted to creating branch logs.
         """
@@ -160,7 +160,7 @@ class ConditionalNode(CompositeNode):
                 "None of the branches were true. Please check your evaluate statements"
             )
 
-    def execute_as_graph(self, map_variable: TypeMapVariable = None):
+    def execute_as_graph(self, map_variable: MapVariableType = None):
         """
         This function does the actual execution of the sub-branches of the parallel node.
 
@@ -190,11 +190,13 @@ class ConditionalNode(CompositeNode):
             if result:
                 # if the condition is met, execute the graph
                 logger.debug(f"Executing graph for {branch}")
-                self._context.executor.execute_graph(branch, map_variable=map_variable)
+                self._context.pipeline_executor.execute_graph(
+                    branch, map_variable=map_variable
+                )
 
         self.fan_in(map_variable=map_variable)
 
-    def fan_in(self, map_variable: TypeMapVariable = None):
+    def fan_in(self, map_variable: MapVariableType = None):
         """
         The general fan in method for a node of type Parallel.
 

extensions/nodes/fail.py

@@ -0,0 +1,72 @@
+from datetime import datetime
+from typing import Any, Dict, cast
+
+from pydantic import Field
+
+from runnable import datastore, defaults
+from runnable.datastore import StepLog
+from runnable.defaults import MapVariableType
+from runnable.nodes import TerminalNode
+
+
+class FailNode(TerminalNode):
+    """
+    A leaf node of the graph that represents a failure node
+    """
+
+    node_type: str = Field(default="fail", serialization_alias="type")
+
+    @classmethod
+    def parse_from_config(cls, config: Dict[str, Any]) -> "FailNode":
+        return cast("FailNode", super().parse_from_config(config))
+
+    def get_summary(self) -> Dict[str, Any]:
+        summary = {
+            "name": self.name,
+            "type": self.node_type,
+        }
+
+        return summary
+
+    def execute(
+        self,
+        mock=False,
+        map_variable: MapVariableType = None,
+        attempt_number: int = 1,
+    ) -> StepLog:
+        """
+        Execute the failure node.
+        Set the run or branch log status to failure.
+
+        Args:
+            executor (_type_): the executor class
+            mock (bool, optional): If we should just mock and not do the actual execution. Defaults to False.
+            map_variable (dict, optional): If the node belongs to internal branches. Defaults to None.
+
+        Returns:
+            StepAttempt: The step attempt object
+        """
+        step_log = self._context.run_log_store.get_step_log(
+            self._get_step_log_name(map_variable), self._context.run_id
+        )
+
+        attempt_log = datastore.StepAttempt(
+            status=defaults.SUCCESS,
+            start_time=str(datetime.now()),
+            end_time=str(datetime.now()),
+            attempt_number=attempt_number,
+        )
+
+        run_or_branch_log = self._context.run_log_store.get_branch_log(
+            self._get_branch_log_name(map_variable), self._context.run_id
+        )
+        run_or_branch_log.status = defaults.FAIL
+        self._context.run_log_store.add_branch_log(
+            run_or_branch_log, self._context.run_id
+        )
+
+        step_log.status = attempt_log.status
+
+        step_log.attempts.append(attempt_log)
+
+        return step_log