runnable 0.11.4__tar.gz → 0.11.5__tar.gz

{runnable-0.11.4 → runnable-0.11.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: runnable
-Version: 0.11.4
+Version: 0.11.5
 Summary: A Compute agnostic pipelining software
 Home-page: https://github.com/vijayvammi/runnable
 License: Apache-2.0

{runnable-0.11.4 → runnable-0.11.5}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "runnable"
-version = "0.11.4"
+version = "0.11.5"
 description = "A Compute agnostic pipelining software"
 authors = ["Vijay Vammi <mesanthu@gmail.com>"]
 license = "Apache-2.0"

{runnable-0.11.4 → runnable-0.11.5}/runnable/__init__.py

@@ -12,7 +12,7 @@ from runnable import defaults
 dictConfig(defaults.LOGGING_CONFIG)
 logger = logging.getLogger(defaults.LOGGER_NAME)
 
-console = Console()
+console = Console(record=True)
 console.print(":runner: Lets go!!")
 
 from runnable.sdk import (  # noqa
@@ -30,7 +30,8 @@ from runnable.sdk import (  # noqa
     pickled,
 )
 
-os.environ["_PLOOMBER_TELEMETRY_DEBUG"] = "false"
+# Needed to disable ploomber telemetry
+os.environ["PLOOMBER_STATS_ENABLED"] = "false"
 
 ## TODO: Summary should be a bit better for catalog.
 ## If the execution fails, hint them about the retry executor.

{runnable-0.11.4 → runnable-0.11.5}/runnable/entrypoints.py

@@ -19,18 +19,11 @@ def get_default_configs() -> RunnableConfig:
     """
     User can provide extensions as part of their code base, runnable-config.yaml provides the place to put them.
     """
-    user_configs = {}
+    user_configs: RunnableConfig = {}
     if utils.does_file_exist(defaults.USER_CONFIG_FILE):
-        user_configs = utils.load_yaml(defaults.USER_CONFIG_FILE)
+        user_configs = cast(RunnableConfig, utils.load_yaml(defaults.USER_CONFIG_FILE))
 
-    if not user_configs:
-        return {}
-
-    user_defaults = user_configs.get("defaults", {})
-    if user_defaults:
-        return user_defaults
-
-    return {}
+    return user_configs
 
 
 def prepare_configurations(
@@ -198,6 +191,7 @@ def execute(
             run_context.progress = progress
             executor.execute_graph(dag=run_context.dag)  # type: ignore
 
+            # Non local executors have no run logs
             if not executor._local:
                 executor.send_return_code(stage="traversal")
                 return
@@ -245,6 +239,8 @@ def execute_single_node(
     """
     from runnable import nodes
 
+    console.print(f"Executing the single node: {step_name} with map variable: {map_variable}")
+
     configuration_file = os.environ.get("RUNNABLE_CONFIGURATION_FILE", configuration_file)
 
     run_context = prepare_configurations(
@@ -264,19 +260,32 @@ def execute_single_node(
 
     executor.prepare_for_node_execution()
 
-    if not run_context.dag:
-        # There are a few entry points that make graph dynamically and do not have a dag defined statically.
-        run_log = run_context.run_log_store.get_run_log_by_id(run_id=run_id, full=False)
-        run_context.dag = graph.create_graph(run_log.run_config["pipeline"])
-
-    step_internal_name = nodes.BaseNode._get_internal_name_from_command_name(step_name)
+    # TODO: may be make its own entry point
+    # if not run_context.dag:
+    #     # There are a few entry points that make graph dynamically and do not have a dag defined statically.
+    #     run_log = run_context.run_log_store.get_run_log_by_id(run_id=run_id, full=False)
+    #     run_context.dag = graph.create_graph(run_log.run_config["pipeline"])
+    assert run_context.dag
 
     map_variable_dict = utils.json_to_ordered_dict(map_variable)
 
+    step_internal_name = nodes.BaseNode._get_internal_name_from_command_name(step_name)
     node_to_execute, _ = graph.search_node_by_internal_name(run_context.dag, step_internal_name)
 
     logger.info("Executing the single node of : %s", node_to_execute)
-    executor.execute_node(node=node_to_execute, map_variable=map_variable_dict)
+    ## This step is where we save the log file
+    try:
+        executor.execute_node(node=node_to_execute, map_variable=map_variable_dict)
+    except Exception:  # noqa: E722
+        log_file_name = utils.make_log_file_name(
+            node=node_to_execute,
+            map_variable=map_variable_dict,
+        )
+        console.save_text(log_file_name)
+
+        # Put the log file in the catalog
+        run_context.catalog_handler.put(name=log_file_name, run_id=run_context.run_id)
+        os.remove(log_file_name)
 
     executor.send_return_code(stage="execution")
 

{runnable-0.11.4 → runnable-0.11.5}/runnable/extensions/executor/argo/implementation.py

@@ -5,7 +5,7 @@ import shlex
 import string
 from abc import ABC, abstractmethod
 from collections import OrderedDict
-from typing import Any, Dict, List, Optional, Union, cast
+from typing import Dict, List, Optional, Union, cast
 
 from pydantic import (
     BaseModel,
@@ -19,7 +19,7 @@ from pydantic.functional_serializers import PlainSerializer
 from ruamel.yaml import YAML
 from typing_extensions import Annotated
 
-from runnable import defaults, exceptions, integration, parameters, utils
+from runnable import defaults, exceptions, integration, utils
 from runnable.defaults import TypeMapVariable
 from runnable.extensions.executor import GenericExecutor
 from runnable.extensions.nodes import DagNode, MapNode, ParallelNode
@@ -378,6 +378,7 @@ class ExecutionNode(NodeRenderer):
             self.node,
             over_write_run_id=self.executor._run_id_placeholder,
             map_variable=map_variable,
+            log_level=self.executor._log_level,
         )
 
         inputs = []
@@ -502,12 +503,16 @@ class MapNodeRender(NodeRenderer):
         self.node = cast(MapNode, self.node)
         task_template_arguments = []
         dag_inputs = []
-        if list_of_iter_values:
-            for value in list_of_iter_values:
-                task_template_arguments.append(Argument(name=value, value="{{inputs.parameters." + value + "}}"))
-                dag_inputs.append(Parameter(name=value))
+
+        if not list_of_iter_values:
+            list_of_iter_values = []
+
+        for value in list_of_iter_values:
+            task_template_arguments.append(Argument(name=value, value="{{inputs.parameters." + value + "}}"))
+            dag_inputs.append(Parameter(name=value))
 
         clean_name = self.executor.get_clean_name(self.node)
+
         fan_out_template = self.executor._create_fan_out_template(
             composite_node=self.node, list_of_iter_values=list_of_iter_values
         )
@@ -518,9 +523,6 @@ class MapNodeRender(NodeRenderer):
         )
         fan_in_template.arguments = task_template_arguments if task_template_arguments else None
 
-        if not list_of_iter_values:
-            list_of_iter_values = []
-
         list_of_iter_values.append(self.node.iterate_as)
 
         self.executor._gather_task_templates_of_dag(
@@ -580,8 +582,12 @@ class Spec(BaseModel):
     node_selector: Optional[Dict[str, str]] = Field(default_factory=dict, serialization_alias="nodeSelector")
     tolerations: Optional[List[Toleration]] = Field(default=None, serialization_alias="tolerations")
     parallelism: Optional[int] = Field(default=None, serialization_alias="parallelism")
+
     # TODO: This has to be user driven
-    pod_gc: Dict[str, str] = Field(default={"strategy": "OnPodCompletion"}, serialization_alias="podGC")
+    pod_gc: Dict[str, str] = Field(
+        default={"strategy": "OnPodSuccess", "deleteDelayDuration": "600s"},
+        serialization_alias="podGC",
+    )
 
     retry_strategy: Retry = Field(default=Retry(), serialization_alias="retryStrategy")
     service_account_name: Optional[str] = Field(default=None, serialization_alias="serviceAccountName")
@@ -674,6 +680,8 @@ class ArgoExecutor(GenericExecutor):
     service_name: str = "argo"
     _local: bool = False
 
+    # TODO: Add logging level as option.
+
     model_config = ConfigDict(extra="forbid")
 
     image: str
@@ -719,6 +727,7 @@ class ArgoExecutor(GenericExecutor):
     persistent_volumes: List[UserVolumeMounts] = Field(default_factory=list)
 
     _run_id_placeholder: str = "{{workflow.parameters.run_id}}"
+    _log_level: str = "{{workflow.parameters.log_level}}"
     _container_templates: List[ContainerTemplate] = []
     _dag_templates: List[DagTemplate] = []
     _clean_names: Dict[str, str] = {}
@@ -828,17 +837,7 @@ class ArgoExecutor(GenericExecutor):
             iterate_on = self._context.run_log_store.get_parameters(self._context.run_id)[node.iterate_on]
 
             with open("/tmp/output.txt", mode="w", encoding="utf-8") as myfile:
-                json.dump(iterate_on, myfile, indent=4)
-
-    def _get_parameters(self) -> Dict[str, Any]:
-        params = {}
-        if self._context.parameters_file:
-            # Parameters from the parameters file if defined
-            params.update(utils.load_yaml(self._context.parameters_file))
-        # parameters from environment variables supersede file based
-        params.update(parameters.get_user_set_parameters())
-
-        return params
+                json.dump(iterate_on.get_value(), myfile, indent=4)
 
     def sanitize_name(self, name):
         return name.replace(" ", "-").replace(".", "-").replace("_", "-")
@@ -886,6 +885,7 @@ class ArgoExecutor(GenericExecutor):
 
         if working_on.name == self._context.dag.start_at and self.expose_parameters_as_inputs:
             for key, value in self._get_parameters().items():
+                value = value.get_value()  # type: ignore
                 # Get the value from work flow parameters for dynamic behavior
                 if isinstance(value, int) or isinstance(value, float) or isinstance(value, str):
                     env_var = EnvVar(
@@ -943,6 +943,7 @@ class ArgoExecutor(GenericExecutor):
             node=composite_node,
             run_id=self._run_id_placeholder,
             map_variable=map_variable,
+            log_level=self._log_level,
         )
 
         outputs = []
@@ -984,6 +985,7 @@ class ArgoExecutor(GenericExecutor):
             node=composite_node,
             run_id=self._run_id_placeholder,
             map_variable=map_variable,
+            log_level=self._log_level,
         )
 
         step_config = {"command": command, "type": "task", "next": "dummy"}
@@ -1033,6 +1035,8 @@ class ArgoExecutor(GenericExecutor):
             if working_on.node_type not in ["success", "fail"] and working_on._get_on_failure_node():
                 failure_node = dag.get_node_by_name(working_on._get_on_failure_node())
 
+                # same logic, if a template exists, retrieve it
+                # if not, create a new one
                 render_obj = get_renderer(working_on)(executor=self, node=failure_node)
                 render_obj.render(list_of_iter_values=list_of_iter_values.copy())
 
@@ -1083,18 +1087,19 @@ class ArgoExecutor(GenericExecutor):
         # Expose "simple" parameters as workflow arguments for dynamic behavior
         if self.expose_parameters_as_inputs:
             for key, value in self._get_parameters().items():
+                value = value.get_value()  # type: ignore
                 if isinstance(value, dict) or isinstance(value, list):
                     continue
-                env_var = EnvVar(name=key, value=value)
+
+                env_var = EnvVar(name=key, value=value)  # type: ignore
                 arguments.append(env_var)
 
         run_id_var = EnvVar(name="run_id", value="{{workflow.uid}}")
+        log_level_var = EnvVar(name="log_level", value=defaults.LOG_LEVEL)
         arguments.append(run_id_var)
+        arguments.append(log_level_var)
 
-        # # TODO: Experimental feature
-
-        # original_run_id_var = EnvVar(name="original_run_id")
-        # arguments.append(original_run_id_var)
+        # TODO: Can we do reruns?
 
         for volume in self.spec.volumes:
             self._container_volumes.append(ContainerVolume(name=volume.name, mount_path=volume.mount_path))

{runnable-0.11.4 → runnable-0.11.5}/runnable/extensions/executor/local_container/implementation.py

@@ -202,7 +202,6 @@ class LocalContainerExecutor(GenericExecutor):
                     f"Please provide a docker_image using executor_config of the step {node.name} or at global config"
                 )
 
-            print("container", self._volumes)
             # TODO: Should consider using getpass.getuser() when running the docker container? Volume permissions
             container = client.containers.create(
                 image=docker_image,

{runnable-0.11.4 → runnable-0.11.5}/runnable/extensions/nodes.py

@@ -15,7 +15,7 @@ from pydantic import (
     field_validator,
 )
 
-from runnable import datastore, defaults, utils
+from runnable import console, datastore, defaults, utils
 from runnable.datastore import (
     JsonParameter,
     MetricParameter,
@@ -96,11 +96,10 @@ class TaskNode(ExecutableNode):
                 attempt_number=attempt_number,
             )
 
-        logger.debug(f"attempt_log: {attempt_log}")
+        logger.info(f"attempt_log: {attempt_log}")
         logger.info(f"Step {self.name} completed with status: {attempt_log.status}")
 
         step_log.status = attempt_log.status
-
         step_log.attempts.append(attempt_log)
 
         return step_log
@@ -347,6 +346,7 @@ class ParallelNode(CompositeNode):
         for internal_branch_name, _ in self.branches.items():
             effective_branch_name = self._resolve_map_placeholders(internal_branch_name, map_variable=map_variable)
             branch_log = self._context.run_log_store.get_branch_log(effective_branch_name, self._context.run_id)
+
             if branch_log.status != defaults.SUCCESS:
                 step_success_bool = False
 
@@ -498,6 +498,8 @@ class MapNode(CompositeNode):
                 self.internal_name + "." + str(iter_variable), map_variable=map_variable
             )
             branch_log = self._context.run_log_store.create_branch_log(effective_branch_name)
+
+            console.print(f"Branch log created for {effective_branch_name}: {branch_log}")
             branch_log.status = defaults.PROCESSING
             self._context.run_log_store.add_branch_log(branch_log, self._context.run_id)
 
@@ -589,6 +591,8 @@ class MapNode(CompositeNode):
                 self.internal_name + "." + str(iter_variable), map_variable=map_variable
             )
             branch_log = self._context.run_log_store.get_branch_log(effective_branch_name, self._context.run_id)
+            # console.print(f"Branch log for {effective_branch_name}: {branch_log}")
+
             if branch_log.status != defaults.SUCCESS:
                 step_success_bool = False
 

{runnable-0.11.4 → runnable-0.11.5}/runnable/tasks.py

@@ -14,13 +14,10 @@ from string import Template
 from typing import Any, Dict, List, Literal, Tuple
 
 from pydantic import BaseModel, ConfigDict, Field, field_validator
-
-# from rich import print
-from rich.console import Console
 from stevedore import driver
 
 import runnable.context as context
-from runnable import defaults, exceptions, parameters, utils
+from runnable import console, defaults, exceptions, parameters, utils
 from runnable.datastore import (
     JsonParameter,
     MetricParameter,
@@ -147,41 +144,26 @@ class BaseTaskType(BaseModel):
                 if context_param in params:
                     params[param_name].value = params[context_param].value
 
+        console.log("Parameters available for the execution:")
+        console.log(params)
+
         logger.debug(f"Resolved parameters: {params}")
 
         if not allow_complex:
             params = {key: value for key, value in params.items() if isinstance(value, JsonParameter)}
 
-        log_file_name = self._context.executor._context_node.internal_name
-        if map_variable:
-            for _, value in map_variable.items():
-                log_file_name += "_" + str(value)
-
-        log_file_name = "".join(x for x in log_file_name if x.isalnum()) + ".execution.log"
-
-        log_file = open(log_file_name, "w")
-
         parameters_in = copy.deepcopy(params)
         f = io.StringIO()
-        task_console = Console(file=io.StringIO())
         try:
             with contextlib.redirect_stdout(f):
                 # with contextlib.nullcontext():
-                yield params, task_console
-                print(task_console.file.getvalue())  # type: ignore
+                yield params
         except Exception as e:  # pylint: disable=broad-except
+            console.log(e, style=defaults.error_style)
             logger.exception(e)
         finally:
-            task_console = None  # type: ignore
             print(f.getvalue())  # print to console
-            log_file.write(f.getvalue())  # Print to file
-
             f.close()
-            log_file.close()
-
-            # Put the log file in the catalog
-            self._context.catalog_handler.put(name=log_file.name, run_id=context.run_context.run_id)
-            os.remove(log_file.name)
 
             # Update parameters
             # This should only update the parameters that are changed at the root level.
@@ -233,7 +215,7 @@ class PythonTaskType(BaseTaskType):  # pylint: disable=too-few-public-methods
         """Execute the notebook as defined by the command."""
         attempt_log = StepAttempt(status=defaults.FAIL, start_time=str(datetime.now()))
 
-        with self.execution_context(map_variable=map_variable) as (params, task_console), self.expose_secrets() as _:
+        with self.execution_context(map_variable=map_variable) as params, self.expose_secrets() as _:
             module, func = utils.get_module_and_attr_names(self.command)
             sys.path.insert(0, os.getcwd())  # Need to add the current directory to path
             imported_module = importlib.import_module(module)
@@ -243,9 +225,10 @@ class PythonTaskType(BaseTaskType):  # pylint: disable=too-few-public-methods
                 try:
                     filtered_parameters = parameters.filter_arguments_for_func(f, params.copy(), map_variable)
                     logger.info(f"Calling {func} from {module} with {filtered_parameters}")
+
                     user_set_parameters = f(**filtered_parameters)  # This is a tuple or single value
                 except Exception as e:
-                    task_console.log(e, style=defaults.error_style, markup=False)
+                    console.log(e, style=defaults.error_style, markup=False)
                     raise exceptions.CommandCallError(f"Function call: {self.command} did not succeed.\n") from e
 
                 attempt_log.input_parameters = params.copy()
@@ -289,8 +272,8 @@ class PythonTaskType(BaseTaskType):  # pylint: disable=too-few-public-methods
             except Exception as _e:
                 msg = f"Call to the function {self.command} did not succeed.\n"
                 attempt_log.message = msg
-                task_console.print_exception(show_locals=False)
-                task_console.log(_e, style=defaults.error_style)
+                console.print_exception(show_locals=False)
+                console.log(_e, style=defaults.error_style)
 
         attempt_log.end_time = str(datetime.now())
 
@@ -346,17 +329,17 @@ class NotebookTaskType(BaseTaskType):
 
             notebook_output_path = self.notebook_output_path
 
-            with self.execution_context(map_variable=map_variable, allow_complex=False) as (
-                params,
-                _,
-            ), self.expose_secrets() as _:
+            with self.execution_context(
+                map_variable=map_variable, allow_complex=False
+            ) as params, self.expose_secrets() as _:
+                copy_params = copy.deepcopy(params)
+
                 if map_variable:
                     for key, value in map_variable.items():
                         notebook_output_path += "_" + str(value)
-                        params[key] = JsonParameter(kind="json", value=value)
+                        copy_params[key] = JsonParameter(kind="json", value=value)
 
                 # Remove any {v}_unreduced parameters from the parameters
-                copy_params = copy.deepcopy(params)
                 unprocessed_params = [k for k, v in copy_params.items() if not v.reduced]
 
                 for key in list(copy_params.keys()):
@@ -397,6 +380,9 @@ class NotebookTaskType(BaseTaskType):
                         )
                 except PicklingError as e:
                     logger.exception("Notebooks cannot return objects")
+                    console.log("Notebooks cannot return objects", style=defaults.error_style)
+                    console.log(e, style=defaults.error_style)
+
                     logger.exception(e)
                     raise
 
@@ -413,6 +399,9 @@ class NotebookTaskType(BaseTaskType):
             )
             logger.exception(msg)
             logger.exception(e)
+
+            console.log(msg, style=defaults.error_style)
+
             attempt_log.status = defaults.FAIL
 
         attempt_log.end_time = str(datetime.now())
@@ -468,7 +457,7 @@ class ShellTaskType(BaseTaskType):
                 subprocess_env[key] = secret_value
 
         try:
-            with self.execution_context(map_variable=map_variable, allow_complex=False) as (params, task_console):
+            with self.execution_context(map_variable=map_variable, allow_complex=False) as params:
                 subprocess_env.update({k: v.get_value() for k, v in params.items()})
 
                 # Json dumps all runnable environment variables
@@ -499,14 +488,14 @@ class ShellTaskType(BaseTaskType):
 
                 if proc.returncode != 0:
                     msg = ",".join(result[1].split("\n"))
-                    task_console.print(msg, style=defaults.error_style)
+                    console.print(msg, style=defaults.error_style)
                     raise exceptions.CommandCallError(msg)
 
                 # for stderr
                 for line in result[1].split("\n"):
                     if line.strip() == "":
                         continue
-                    task_console.print(line, style=defaults.warning_style)
+                    console.print(line, style=defaults.warning_style)
 
                 output_parameters: Dict[str, Parameter] = {}
                 metrics: Dict[str, Parameter] = {}
@@ -517,7 +506,7 @@ class ShellTaskType(BaseTaskType):
                         continue
 
                     logger.info(line)
-                    task_console.print(line)
+                    console.print(line)
 
                     if line.strip() == collect_delimiter:
                         # The lines from now on should be captured
@@ -558,6 +547,10 @@ class ShellTaskType(BaseTaskType):
             msg = f"Call to the command {self.command} did not succeed"
             logger.exception(msg)
             logger.exception(e)
+
+            console.log(msg, style=defaults.error_style)
+            console.log(e, style=defaults.error_style)
+
             attempt_log.status = defaults.FAIL
 
         attempt_log.end_time = str(datetime.now())

{runnable-0.11.4 → runnable-0.11.5}/runnable/utils.py

@@ -4,6 +4,8 @@ import hashlib
 import json
 import logging
 import os
+import random
+import string
 import subprocess
 from collections import OrderedDict
 from datetime import datetime
@@ -394,6 +396,7 @@ def get_node_execution_command(
     node: BaseNode,
     map_variable: TypeMapVariable = None,
     over_write_run_id: str = "",
+    log_level: str = "",
 ) -> str:
     """A utility function to standardize execution call to a node via command line.
 
@@ -410,7 +413,7 @@ def get_node_execution_command(
     if over_write_run_id:
         run_id = over_write_run_id
 
-    log_level = logging.getLevelName(logger.getEffectiveLevel())
+    log_level = log_level or logging.getLevelName(logger.getEffectiveLevel())
 
     action = f"runnable execute_single_node {run_id} " f"{node._command_friendly_name()}" f" --log-level {log_level}"
 
@@ -437,6 +440,7 @@ def get_fan_command(
     node: BaseNode,
     run_id: str,
     map_variable: TypeMapVariable = None,
+    log_level: str = "",
 ) -> str:
     """
     An utility function to return the fan "in or out" command
@@ -451,7 +455,7 @@ def get_fan_command(
     Returns:
         str: The fan in or out command
     """
-    log_level = logging.getLevelName(logger.getEffectiveLevel())
+    log_level = log_level or logging.getLevelName(logger.getEffectiveLevel())
     action = (
         f"runnable fan {run_id} "
         f"{node._command_friendly_name()} "
@@ -614,3 +618,17 @@ def gather_variables() -> dict:
             variables[key] = value
 
     return variables
+
+
+def make_log_file_name(node: BaseNode, map_variable: TypeMapVariable) -> str:
+    random_tag = "".join(random.choices(string.ascii_uppercase + string.digits, k=3))
+    log_file_name = node.name
+
+    if map_variable:
+        for _, value in map_variable.items():
+            log_file_name += "_" + str(value)
+
+    log_file_name += "_" + random_tag
+    log_file_name = "".join(x for x in log_file_name if x.isalnum()) + ".execution.log"
+
+    return log_file_name