runnable 0.12.0__py3-none-any.whl → 0.12.2__py3-none-any.whl

runnable/__init__.py

@@ -15,6 +15,8 @@ logger = logging.getLogger(defaults.LOGGER_NAME)
 console = Console(record=True)
 console.print(":runner: Lets go!!")
 
+task_console = Console(record=True)
+
 from runnable.sdk import (  # noqa
     Catalog,
     Fail,

runnable/defaults.py

@@ -77,7 +77,7 @@ DEFAULT_CONTAINER_OUTPUT_PARAMETERS = "parameters.json"
 DEFAULT_EXECUTOR = ServiceConfig(type="local", config={})
 DEFAULT_RUN_LOG_STORE = ServiceConfig(type="file-system", config={})
 DEFAULT_CATALOG = ServiceConfig(type="file-system", config={})
-DEFAULT_SECRETS = ServiceConfig(type="do-nothing", config={})
+DEFAULT_SECRETS = ServiceConfig(type="env-secrets", config={})
 DEFAULT_EXPERIMENT_TRACKER = ServiceConfig(type="do-nothing", config={})
 DEFAULT_PICKLER = ServiceConfig(type="pickle", config={})
 

runnable/entrypoints.py

@@ -9,7 +9,7 @@ from rich.progress import BarColumn, Progress, TextColumn, TimeElapsedColumn
 from rich.table import Column
 
 import runnable.context as context
-from runnable import console, defaults, graph, utils
+from runnable import console, defaults, graph, task_console, utils
 from runnable.defaults import RunnableConfig, ServiceConfig
 
 logger = logging.getLogger(defaults.LOGGER_NAME)
@@ -165,6 +165,7 @@ def execute(
         tag=tag,
         parameters_file=parameters_file,
     )
+
     console.print("Working with context:")
     console.print(run_context)
     console.rule(style="[dark orange]")
@@ -239,7 +240,7 @@ def execute_single_node(
     """
     from runnable import nodes
 
-    console.print(f"Executing the single node: {step_name} with map variable: {map_variable}")
+    task_console.print(f"Executing the single node: {step_name} with map variable: {map_variable}")
 
     configuration_file = os.environ.get("RUNNABLE_CONFIGURATION_FILE", configuration_file)
 
@@ -250,9 +251,9 @@ def execute_single_node(
         tag=tag,
         parameters_file=parameters_file,
     )
-    console.print("Working with context:")
-    console.print(run_context)
-    console.rule(style="[dark orange]")
+    task_console.print("Working with context:")
+    task_console.print(run_context)
+    task_console.rule(style="[dark orange]")
 
     executor = run_context.executor
     run_context.execution_plan = defaults.EXECUTION_PLAN.CHAINED.value
@@ -276,18 +277,18 @@ def execute_single_node(
     ## This step is where we save the log file
     try:
         executor.execute_node(node=node_to_execute, map_variable=map_variable_dict)
-    except Exception:  # noqa: E722
+    finally:
         log_file_name = utils.make_log_file_name(
             node=node_to_execute,
             map_variable=map_variable_dict,
         )
-        console.save_text(log_file_name)
+        task_console.save_text(log_file_name)
 
         # Put the log file in the catalog
         run_context.catalog_handler.put(name=log_file_name, run_id=run_context.run_id)
         os.remove(log_file_name)
 
-    executor.send_return_code(stage="execution")
+    # executor.send_return_code(stage="execution")
 
 
 def execute_notebook(

runnable/extensions/executor/__init__.py

@@ -11,6 +11,7 @@ from runnable import (
     exceptions,
     integration,
     parameters,
+    task_console,
     utils,
 )
 from runnable.datastore import DataCatalog, JsonParameter, RunLog, StepLog
@@ -340,10 +341,18 @@ class GenericExecutor(BaseExecutor):
             node.execute_as_graph(map_variable=map_variable, **kwargs)
             return
 
+        task_console.export_text(clear=True)
+
         task_name = node._resolve_map_placeholders(node.internal_name, map_variable)
         console.print(f":runner: Executing the node {task_name} ... ", style="bold color(208)")
         self.trigger_job(node=node, map_variable=map_variable, **kwargs)
 
+        log_file_name = utils.make_log_file_name(node=node, map_variable=map_variable)
+        task_console.save_text(log_file_name, clear=True)
+
+        self._context.catalog_handler.put(name=log_file_name, run_id=self._context.run_id)
+        os.remove(log_file_name)
+
     def trigger_job(self, node: BaseNode, map_variable: TypeMapVariable = None, **kwargs):
         """
         Call this method only if we are responsible for traversing the graph via
@@ -493,6 +502,7 @@ class GenericExecutor(BaseExecutor):
 
         logger.info(f"Finished execution of the {branch} with status {run_log.status}")
 
+        # We are in the root dag
         if dag == self._context.dag:
             run_log = cast(RunLog, run_log)
             console.print("Completed Execution, Summary:", style="bold color(208)")

runnable/extensions/executor/argo/implementation.py

@@ -303,7 +303,7 @@ class DagTemplate(BaseModel):
     tasks: List[DagTaskTemplate] = Field(default=[], exclude=True)
     inputs: Optional[List[Parameter]] = Field(default=None, serialization_alias="inputs")
     parallelism: Optional[int] = None
-    fail_fast: bool = Field(default=True, serialization_alias="failFast")
+    fail_fast: bool = Field(default=False, serialization_alias="failFast")
 
     @field_validator("parallelism")
     @classmethod

runnable/extensions/executor/local_container/implementation.py

@@ -5,7 +5,7 @@ from typing import Dict, cast
 from pydantic import Field
 from rich import print
 
-from runnable import defaults, utils
+from runnable import console, defaults, task_console, utils
 from runnable.datastore import StepLog
 from runnable.defaults import TypeMapVariable
 from runnable.extensions.executor import GenericExecutor
@@ -96,6 +96,59 @@ class LocalContainerExecutor(GenericExecutor):
         """
         return self._execute_node(node, map_variable, **kwargs)
 
+    def execute_from_graph(self, node: BaseNode, map_variable: TypeMapVariable = None, **kwargs):
+        """
+        This is the entry point to from the graph execution.
+
+        While the self.execute_graph is responsible for traversing the graph, this function is responsible for
+        actual execution of the node.
+
+        If the node type is:
+            * task : We can delegate to _execute_node after checking the eligibility for re-run in cases of a re-run
+            * success: We can delegate to _execute_node
+            * fail: We can delegate to _execute_node
+
+        For nodes that are internally graphs:
+            * parallel: Delegate the responsibility of execution to the node.execute_as_graph()
+            * dag: Delegate the responsibility of execution to the node.execute_as_graph()
+            * map: Delegate the responsibility of execution to the node.execute_as_graph()
+
+        Transpilers will NEVER use this method and will NEVER call ths method.
+        This method should only be used by interactive executors.
+
+        Args:
+            node (Node): The node to execute
+            map_variable (dict, optional): If the node if of a map state, this corresponds to the value of iterable.
+                    Defaults to None.
+        """
+        step_log = self._context.run_log_store.create_step_log(node.name, node._get_step_log_name(map_variable))
+
+        self.add_code_identities(node=node, step_log=step_log)
+
+        step_log.step_type = node.node_type
+        step_log.status = defaults.PROCESSING
+
+        self._context.run_log_store.add_step_log(step_log, self._context.run_id)
+
+        logger.info(f"Executing node: {node.get_summary()}")
+
+        # Add the step log to the database as per the situation.
+        # If its a terminal node, complete it now
+        if node.node_type in ["success", "fail"]:
+            self._execute_node(node, map_variable=map_variable, **kwargs)
+            return
+
+        # We call an internal function to iterate the sub graphs and execute them
+        if node.is_composite:
+            node.execute_as_graph(map_variable=map_variable, **kwargs)
+            return
+
+        task_console.export_text(clear=True)
+
+        task_name = node._resolve_map_placeholders(node.internal_name, map_variable)
+        console.print(f":runner: Executing the node {task_name} ... ", style="bold color(208)")
+        self.trigger_job(node=node, map_variable=map_variable, **kwargs)
+
     def execute_job(self, node: TaskNode):
         """
         Set up the step log and call the execute node

runnable/tasks.py

@@ -17,7 +17,7 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator
 from stevedore import driver
 
 import runnable.context as context
-from runnable import console, defaults, exceptions, parameters, utils
+from runnable import console, defaults, exceptions, parameters, task_console, utils
 from runnable.datastore import (
     JsonParameter,
     MetricParameter,
@@ -144,8 +144,8 @@ class BaseTaskType(BaseModel):
                 if context_param in params:
                     params[param_name].value = params[context_param].value
 
-        console.log("Parameters available for the execution:")
-        console.log(params)
+        task_console.log("Parameters available for the execution:")
+        task_console.log(params)
 
         logger.debug(f"Resolved parameters: {params}")
 
@@ -153,18 +153,12 @@ class BaseTaskType(BaseModel):
             params = {key: value for key, value in params.items() if isinstance(value, JsonParameter)}
 
         parameters_in = copy.deepcopy(params)
-        f = io.StringIO()
         try:
-            with contextlib.redirect_stdout(f):
-                # with contextlib.nullcontext():
-                yield params
+            yield params
         except Exception as e:  # pylint: disable=broad-except
             console.log(e, style=defaults.error_style)
             logger.exception(e)
         finally:
-            print(f.getvalue())  # print to console
-            f.close()
-
             # Update parameters
             # This should only update the parameters that are changed at the root level.
             diff_parameters = self._diff_parameters(parameters_in=parameters_in, context_params=params)
@@ -226,9 +220,11 @@ class PythonTaskType(BaseTaskType):  # pylint: disable=too-few-public-methods
                     filtered_parameters = parameters.filter_arguments_for_func(f, params.copy(), map_variable)
                     logger.info(f"Calling {func} from {module} with {filtered_parameters}")
 
-                    user_set_parameters = f(**filtered_parameters)  # This is a tuple or single value
+                    out_file = io.StringIO()
+                    with contextlib.redirect_stdout(out_file):
+                        user_set_parameters = f(**filtered_parameters)  # This is a tuple or single value
+                    task_console.print(out_file.getvalue())
                 except Exception as e:
-                    console.log(e, style=defaults.error_style, markup=False)
                     raise exceptions.CommandCallError(f"Function call: {self.command} did not succeed.\n") from e
 
                 attempt_log.input_parameters = params.copy()
@@ -272,8 +268,8 @@ class PythonTaskType(BaseTaskType):  # pylint: disable=too-few-public-methods
             except Exception as _e:
                 msg = f"Call to the function {self.command} did not succeed.\n"
                 attempt_log.message = msg
-                console.print_exception(show_locals=False)
-                console.log(_e, style=defaults.error_style)
+                task_console.print_exception(show_locals=False)
+                task_console.log(_e, style=defaults.error_style)
 
         attempt_log.end_time = str(datetime.now())
 
@@ -359,7 +355,11 @@ class NotebookTaskType(BaseTaskType):
                 }
                 kwds.update(ploomber_optional_args)
 
-                pm.execute_notebook(**kwds)
+                out_file = io.StringIO()
+                with contextlib.redirect_stdout(out_file):
+                    pm.execute_notebook(**kwds)
+                task_console.print(out_file.getvalue())
+
                 context.run_context.catalog_handler.put(name=notebook_output_path, run_id=context.run_context.run_id)
 
                 client = PloomberClient.from_path(path=notebook_output_path)
@@ -380,8 +380,8 @@ class NotebookTaskType(BaseTaskType):
                         )
                 except PicklingError as e:
                     logger.exception("Notebooks cannot return objects")
-                    console.log("Notebooks cannot return objects", style=defaults.error_style)
-                    console.log(e, style=defaults.error_style)
+                    # task_console.log("Notebooks cannot return objects", style=defaults.error_style)
+                    # task_console.log(e, style=defaults.error_style)
 
                     logger.exception(e)
                     raise
@@ -400,8 +400,7 @@ class NotebookTaskType(BaseTaskType):
             logger.exception(msg)
             logger.exception(e)
 
-            console.log(msg, style=defaults.error_style)
-
+            # task_console.log(msg, style=defaults.error_style)
             attempt_log.status = defaults.FAIL
 
         attempt_log.end_time = str(datetime.now())
@@ -488,14 +487,14 @@ class ShellTaskType(BaseTaskType):
 
                 if proc.returncode != 0:
                     msg = ",".join(result[1].split("\n"))
-                    console.print(msg, style=defaults.error_style)
+                    task_console.print(msg, style=defaults.error_style)
                     raise exceptions.CommandCallError(msg)
 
                 # for stderr
                 for line in result[1].split("\n"):
                     if line.strip() == "":
                         continue
-                    console.print(line, style=defaults.warning_style)
+                    task_console.print(line, style=defaults.warning_style)
 
                 output_parameters: Dict[str, Parameter] = {}
                 metrics: Dict[str, Parameter] = {}
@@ -506,7 +505,7 @@ class ShellTaskType(BaseTaskType):
                         continue
 
                     logger.info(line)
-                    console.print(line)
+                    task_console.print(line)
 
                     if line.strip() == collect_delimiter:
                         # The lines from now on should be captured
@@ -548,8 +547,8 @@ class ShellTaskType(BaseTaskType):
             logger.exception(msg)
             logger.exception(e)
 
-            console.log(msg, style=defaults.error_style)
-            console.log(e, style=defaults.error_style)
+            task_console.log(msg, style=defaults.error_style)
+            task_console.log(e, style=defaults.error_style)
 
             attempt_log.status = defaults.FAIL
 

{runnable-0.12.0.dist-info → runnable-0.12.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: runnable
-Version: 0.12.0
+Version: 0.12.2
 Summary: A Compute agnostic pipelining software
 Home-page: https://github.com/vijayvammi/runnable
 License: Apache-2.0

{runnable-0.12.0.dist-info → runnable-0.12.2.dist-info}/RECORD

@@ -1,10 +1,10 @@
-runnable/__init__.py,sha256=zaZVsNzp9zrpngSMNLzT_aIFkkWSpp9gb_K-GAnr9Cw,1001
+runnable/__init__.py,sha256=G5EUe1eaTOEu1UihDBP9F43PKs3yxCYFfp8DOOpPvms,1038
 runnable/catalog.py,sha256=22OECi5TrpHErxYIhfx-lJ2vgBUi4-5V9CaYEVm98hE,4138
 runnable/cli.py,sha256=RILUrEfzernuKD3dNdXPBkqN_1OgE5GosYRuInj0FVs,9618
 runnable/context.py,sha256=QhiXJHRcEBfSKB1ijvL5yB9w44x0HCe7VEiwK1cUJ9U,1124
 runnable/datastore.py,sha256=8aQZ15KAMdre7a7G61bNRmcTeJFzOdnx_9O9UP4JQc8,27910
-runnable/defaults.py,sha256=MOX7I2S6yO4FphZaZREFQca94a20oO8uvzXLd6GLKQs,4703
-runnable/entrypoints.py,sha256=fLipciON3x2iAkBE9w00TrRPqwLxyLJeKu4V7dlgB-A,17611
+runnable/defaults.py,sha256=r9l3jCPlmMFNdYXOj6X1uIhruO2FyLZ8kuzus9Fd6OQ,4704
+runnable/entrypoints.py,sha256=ZdXnjEjtkOqKYyjc0AwqIdfWgxJsKFB8ilHYcUj7oZo,17630
 runnable/exceptions.py,sha256=6NIYoTAzdKyGQ9PvW1Hu7b80OS746395KiGDhM7ThH8,2526
 runnable/executor.py,sha256=xfBighQ5t_vejohip000XfxLwsgechUE1ZMIJWrZbUA,14484
 runnable/extensions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -14,9 +14,9 @@ runnable/extensions/catalog/file_system/implementation.py,sha256=mFPsAwPMNGWbHcz
 runnable/extensions/catalog/k8s_pvc/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 runnable/extensions/catalog/k8s_pvc/implementation.py,sha256=oJDDI0APT7lrtjWmzYJRDHLGn3Vhbn2MdFSRYvFBUpY,436
 runnable/extensions/catalog/k8s_pvc/integration.py,sha256=OfrHbNFN8sR-wsVa4os3ajmWJFSd5H4KOHGVAmjRZTQ,1850
-runnable/extensions/executor/__init__.py,sha256=0385OpNSpjyA0GjXlLw7gZtqJFFOHGLmYHzWAGBzU98,26247
+runnable/extensions/executor/__init__.py,sha256=-Xw8ZYVOfM-bYAieluEOhd98ncyrAyde8WSPRg1yFN0,26615
 runnable/extensions/executor/argo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-runnable/extensions/executor/argo/implementation.py,sha256=RMvHLt1VEHb4_C8dEJomistRztycThHfcX9l4CXVp_s,43910
+runnable/extensions/executor/argo/implementation.py,sha256=FE5jDtzv5nlsnOjN9k0VjtEFQQWAAFaSXVJlroi7q6I,43911
 runnable/extensions/executor/argo/specification.yaml,sha256=wXQcm2gOQYqy-IOQIhucohS32ZrHKCfGA5zZ0RraPYc,1276
 runnable/extensions/executor/k8s_job/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 runnable/extensions/executor/k8s_job/implementation_FF.py,sha256=1IfVG1GRcJcVFzQ-WhkJsmzdJuj51QMxXylY9UrWM0U,10259
@@ -24,7 +24,7 @@ runnable/extensions/executor/k8s_job/integration_FF.py,sha256=pG6HKhPMgCRIgu1PAn
 runnable/extensions/executor/local/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 runnable/extensions/executor/local/implementation.py,sha256=e8Tzv-FgQmJeUXVut96jeNERTR83JVG_zkQZMEjCVAs,2469
 runnable/extensions/executor/local_container/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-runnable/extensions/executor/local_container/implementation.py,sha256=Kley2sjdZMe7E5CHjjy4YaJ5YErNGvZmpVW33h6D0W0,14848
+runnable/extensions/executor/local_container/implementation.py,sha256=6iwt9tNCQawVEfIErzoqys2hrErWK0DHcAOkO49Ov9w,17322
 runnable/extensions/executor/mocked/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 runnable/extensions/executor/mocked/implementation.py,sha256=ChvlcLGpBxO6QwJcoqhBgKBR6NfWVnMdOWKQhMgcEjY,5762
 runnable/extensions/executor/retry/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -55,10 +55,10 @@ runnable/parameters.py,sha256=yZkMDnwnkdYXIwQ8LflBzn50Y0xRGxEvLlxwno6ovvs,5163
 runnable/pickler.py,sha256=5SDNf0miMUJ3ZauhQdzwk8_t-9jeOqaTjP5bvRnu9sU,2685
 runnable/sdk.py,sha256=pmxGO5HSrcEGYR_VmVk6ObfddFHo5slxiaDI6LuIdKM,27852
 runnable/secrets.py,sha256=dakb7WRloWVo-KpQp6Vy4rwFdGi58BTlT4OifQY106I,2324
-runnable/tasks.py,sha256=a5Fkvl58ku9JPo-qDDJahemD-6a7jJlXlBEowvmeKuc,21910
+runnable/tasks.py,sha256=7sAtFMu6ELD3PJoisNbm47rY2wPKVzz5_h4s_QMep0k,22043
 runnable/utils.py,sha256=fXOLoFZKYqh3wQgzA2V-VZOu-dSgLPGqCZIbMmsNzOw,20016
-runnable-0.12.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-runnable-0.12.0.dist-info/METADATA,sha256=1iBCYGj4Pv_Zw3wSYX8dSh3PVEnvIVFsG4lhthrO_V4,17020
-runnable-0.12.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-runnable-0.12.0.dist-info/entry_points.txt,sha256=amb6ISqKBSIz47um8_6LKnYgpoZ4d_p6-O1-7uUb1cU,1447
-runnable-0.12.0.dist-info/RECORD,,
+runnable-0.12.2.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+runnable-0.12.2.dist-info/METADATA,sha256=8u2DRuU-gQvmW66oAqKABw5BYsCB0sJde6E_u-coEiE,17020
+runnable-0.12.2.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+runnable-0.12.2.dist-info/entry_points.txt,sha256=-csEf-FCAqtOboXaBSzpgaTffz1HwlylYPxnppndpFE,1494
+runnable-0.12.2.dist-info/RECORD,,

{runnable-0.12.0.dist-info → runnable-0.12.2.dist-info}/entry_points.txt

@@ -32,6 +32,7 @@ file-system=runnable.extensions.run_log_store.file_system.implementation:FileSys
 [secrets]
 do-nothing=runnable.secrets:DoNothingSecretManager
 dotenv=runnable.extensions.secrets.dotenv.implementation:DotEnvSecrets
+env-secrets=runnable.secrets:EnvSecretsManager
 
 [tasks]
 notebook=runnable.tasks:NotebookTaskType