runnable 0.11.3__py3-none-any.whl → 0.11.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- runnable/__init__.py +4 -2
- runnable/entrypoints.py +26 -17
- runnable/extensions/executor/argo/implementation.py +31 -26
- runnable/extensions/executor/local_container/implementation.py +0 -1
- runnable/extensions/nodes.py +7 -3
- runnable/tasks.py +31 -38
- runnable/utils.py +20 -2
- {runnable-0.11.3.dist-info → runnable-0.11.5.dist-info}/METADATA +1 -1
- {runnable-0.11.3.dist-info → runnable-0.11.5.dist-info}/RECORD +12 -12
- {runnable-0.11.3.dist-info → runnable-0.11.5.dist-info}/LICENSE +0 -0
- {runnable-0.11.3.dist-info → runnable-0.11.5.dist-info}/WHEEL +0 -0
- {runnable-0.11.3.dist-info → runnable-0.11.5.dist-info}/entry_points.txt +0 -0
    
        runnable/__init__.py
    CHANGED
    
    | @@ -12,7 +12,7 @@ from runnable import defaults | |
| 12 12 | 
             
            dictConfig(defaults.LOGGING_CONFIG)
         | 
| 13 13 | 
             
            logger = logging.getLogger(defaults.LOGGER_NAME)
         | 
| 14 14 |  | 
| 15 | 
            -
            console = Console()
         | 
| 15 | 
            +
            console = Console(record=True)
         | 
| 16 16 | 
             
            console.print(":runner: Lets go!!")
         | 
| 17 17 |  | 
| 18 18 | 
             
            from runnable.sdk import (  # noqa
         | 
| @@ -30,7 +30,8 @@ from runnable.sdk import (  # noqa | |
| 30 30 | 
             
                pickled,
         | 
| 31 31 | 
             
            )
         | 
| 32 32 |  | 
| 33 | 
            -
             | 
| 33 | 
            +
            # Needed to disable ploomber telemetry
         | 
| 34 | 
            +
            os.environ["PLOOMBER_STATS_ENABLED"] = "false"
         | 
| 34 35 |  | 
| 35 36 | 
             
            ## TODO: Summary should be a bit better for catalog.
         | 
| 36 37 | 
             
            ## If the execution fails, hint them about the retry executor.
         | 
| @@ -41,3 +42,4 @@ os.environ["_PLOOMBER_TELEMETRY_DEBUG"] = "false" | |
| 41 42 |  | 
| 42 43 |  | 
| 43 44 | 
             
            # TODO: Think of way of generating dag hash without executor configuration
         | 
| 45 | 
            +
            # Try to get a release
         | 
    
        runnable/entrypoints.py
    CHANGED
    
    | @@ -19,18 +19,11 @@ def get_default_configs() -> RunnableConfig: | |
| 19 19 | 
             
                """
         | 
| 20 20 | 
             
                User can provide extensions as part of their code base, runnable-config.yaml provides the place to put them.
         | 
| 21 21 | 
             
                """
         | 
| 22 | 
            -
                user_configs = {}
         | 
| 22 | 
            +
                user_configs: RunnableConfig = {}
         | 
| 23 23 | 
             
                if utils.does_file_exist(defaults.USER_CONFIG_FILE):
         | 
| 24 | 
            -
                    user_configs = utils.load_yaml(defaults.USER_CONFIG_FILE)
         | 
| 24 | 
            +
                    user_configs = cast(RunnableConfig, utils.load_yaml(defaults.USER_CONFIG_FILE))
         | 
| 25 25 |  | 
| 26 | 
            -
                 | 
| 27 | 
            -
                    return {}
         | 
| 28 | 
            -
             | 
| 29 | 
            -
                user_defaults = user_configs.get("defaults", {})
         | 
| 30 | 
            -
                if user_defaults:
         | 
| 31 | 
            -
                    return user_defaults
         | 
| 32 | 
            -
             | 
| 33 | 
            -
                return {}
         | 
| 26 | 
            +
                return user_configs
         | 
| 34 27 |  | 
| 35 28 |  | 
| 36 29 | 
             
            def prepare_configurations(
         | 
| @@ -198,6 +191,7 @@ def execute( | |
| 198 191 | 
             
                        run_context.progress = progress
         | 
| 199 192 | 
             
                        executor.execute_graph(dag=run_context.dag)  # type: ignore
         | 
| 200 193 |  | 
| 194 | 
            +
                        # Non local executors have no run logs
         | 
| 201 195 | 
             
                        if not executor._local:
         | 
| 202 196 | 
             
                            executor.send_return_code(stage="traversal")
         | 
| 203 197 | 
             
                            return
         | 
| @@ -245,6 +239,8 @@ def execute_single_node( | |
| 245 239 | 
             
                """
         | 
| 246 240 | 
             
                from runnable import nodes
         | 
| 247 241 |  | 
| 242 | 
            +
                console.print(f"Executing the single node: {step_name} with map variable: {map_variable}")
         | 
| 243 | 
            +
             | 
| 248 244 | 
             
                configuration_file = os.environ.get("RUNNABLE_CONFIGURATION_FILE", configuration_file)
         | 
| 249 245 |  | 
| 250 246 | 
             
                run_context = prepare_configurations(
         | 
| @@ -264,19 +260,32 @@ def execute_single_node( | |
| 264 260 |  | 
| 265 261 | 
             
                executor.prepare_for_node_execution()
         | 
| 266 262 |  | 
| 267 | 
            -
                 | 
| 268 | 
            -
             | 
| 269 | 
            -
             | 
| 270 | 
            -
             | 
| 271 | 
            -
             | 
| 272 | 
            -
                 | 
| 263 | 
            +
                # TODO: may be make its own entry point
         | 
| 264 | 
            +
                # if not run_context.dag:
         | 
| 265 | 
            +
                #     # There are a few entry points that make graph dynamically and do not have a dag defined statically.
         | 
| 266 | 
            +
                #     run_log = run_context.run_log_store.get_run_log_by_id(run_id=run_id, full=False)
         | 
| 267 | 
            +
                #     run_context.dag = graph.create_graph(run_log.run_config["pipeline"])
         | 
| 268 | 
            +
                assert run_context.dag
         | 
| 273 269 |  | 
| 274 270 | 
             
                map_variable_dict = utils.json_to_ordered_dict(map_variable)
         | 
| 275 271 |  | 
| 272 | 
            +
                step_internal_name = nodes.BaseNode._get_internal_name_from_command_name(step_name)
         | 
| 276 273 | 
             
                node_to_execute, _ = graph.search_node_by_internal_name(run_context.dag, step_internal_name)
         | 
| 277 274 |  | 
| 278 275 | 
             
                logger.info("Executing the single node of : %s", node_to_execute)
         | 
| 279 | 
            -
                 | 
| 276 | 
            +
                ## This step is where we save the log file
         | 
| 277 | 
            +
                try:
         | 
| 278 | 
            +
                    executor.execute_node(node=node_to_execute, map_variable=map_variable_dict)
         | 
| 279 | 
            +
                except Exception:  # noqa: E722
         | 
| 280 | 
            +
                    log_file_name = utils.make_log_file_name(
         | 
| 281 | 
            +
                        node=node_to_execute,
         | 
| 282 | 
            +
                        map_variable=map_variable_dict,
         | 
| 283 | 
            +
                    )
         | 
| 284 | 
            +
                    console.save_text(log_file_name)
         | 
| 285 | 
            +
             | 
| 286 | 
            +
                    # Put the log file in the catalog
         | 
| 287 | 
            +
                    run_context.catalog_handler.put(name=log_file_name, run_id=run_context.run_id)
         | 
| 288 | 
            +
                    os.remove(log_file_name)
         | 
| 280 289 |  | 
| 281 290 | 
             
                executor.send_return_code(stage="execution")
         | 
| 282 291 |  | 
| @@ -5,7 +5,7 @@ import shlex | |
| 5 5 | 
             
            import string
         | 
| 6 6 | 
             
            from abc import ABC, abstractmethod
         | 
| 7 7 | 
             
            from collections import OrderedDict
         | 
| 8 | 
            -
            from typing import  | 
| 8 | 
            +
            from typing import Dict, List, Optional, Union, cast
         | 
| 9 9 |  | 
| 10 10 | 
             
            from pydantic import (
         | 
| 11 11 | 
             
                BaseModel,
         | 
| @@ -19,7 +19,7 @@ from pydantic.functional_serializers import PlainSerializer | |
| 19 19 | 
             
            from ruamel.yaml import YAML
         | 
| 20 20 | 
             
            from typing_extensions import Annotated
         | 
| 21 21 |  | 
| 22 | 
            -
            from runnable import defaults, exceptions, integration,  | 
| 22 | 
            +
            from runnable import defaults, exceptions, integration, utils
         | 
| 23 23 | 
             
            from runnable.defaults import TypeMapVariable
         | 
| 24 24 | 
             
            from runnable.extensions.executor import GenericExecutor
         | 
| 25 25 | 
             
            from runnable.extensions.nodes import DagNode, MapNode, ParallelNode
         | 
| @@ -378,6 +378,7 @@ class ExecutionNode(NodeRenderer): | |
| 378 378 | 
             
                        self.node,
         | 
| 379 379 | 
             
                        over_write_run_id=self.executor._run_id_placeholder,
         | 
| 380 380 | 
             
                        map_variable=map_variable,
         | 
| 381 | 
            +
                        log_level=self.executor._log_level,
         | 
| 381 382 | 
             
                    )
         | 
| 382 383 |  | 
| 383 384 | 
             
                    inputs = []
         | 
| @@ -502,12 +503,16 @@ class MapNodeRender(NodeRenderer): | |
| 502 503 | 
             
                    self.node = cast(MapNode, self.node)
         | 
| 503 504 | 
             
                    task_template_arguments = []
         | 
| 504 505 | 
             
                    dag_inputs = []
         | 
| 505 | 
            -
             | 
| 506 | 
            -
             | 
| 507 | 
            -
             | 
| 508 | 
            -
             | 
| 506 | 
            +
             | 
| 507 | 
            +
                    if not list_of_iter_values:
         | 
| 508 | 
            +
                        list_of_iter_values = []
         | 
| 509 | 
            +
             | 
| 510 | 
            +
                    for value in list_of_iter_values:
         | 
| 511 | 
            +
                        task_template_arguments.append(Argument(name=value, value="{{inputs.parameters." + value + "}}"))
         | 
| 512 | 
            +
                        dag_inputs.append(Parameter(name=value))
         | 
| 509 513 |  | 
| 510 514 | 
             
                    clean_name = self.executor.get_clean_name(self.node)
         | 
| 515 | 
            +
             | 
| 511 516 | 
             
                    fan_out_template = self.executor._create_fan_out_template(
         | 
| 512 517 | 
             
                        composite_node=self.node, list_of_iter_values=list_of_iter_values
         | 
| 513 518 | 
             
                    )
         | 
| @@ -518,9 +523,6 @@ class MapNodeRender(NodeRenderer): | |
| 518 523 | 
             
                    )
         | 
| 519 524 | 
             
                    fan_in_template.arguments = task_template_arguments if task_template_arguments else None
         | 
| 520 525 |  | 
| 521 | 
            -
                    if not list_of_iter_values:
         | 
| 522 | 
            -
                        list_of_iter_values = []
         | 
| 523 | 
            -
             | 
| 524 526 | 
             
                    list_of_iter_values.append(self.node.iterate_as)
         | 
| 525 527 |  | 
| 526 528 | 
             
                    self.executor._gather_task_templates_of_dag(
         | 
| @@ -580,8 +582,12 @@ class Spec(BaseModel): | |
| 580 582 | 
             
                node_selector: Optional[Dict[str, str]] = Field(default_factory=dict, serialization_alias="nodeSelector")
         | 
| 581 583 | 
             
                tolerations: Optional[List[Toleration]] = Field(default=None, serialization_alias="tolerations")
         | 
| 582 584 | 
             
                parallelism: Optional[int] = Field(default=None, serialization_alias="parallelism")
         | 
| 585 | 
            +
             | 
| 583 586 | 
             
                # TODO: This has to be user driven
         | 
| 584 | 
            -
                pod_gc: Dict[str, str] = Field( | 
| 587 | 
            +
                pod_gc: Dict[str, str] = Field(
         | 
| 588 | 
            +
                    default={"strategy": "OnPodSuccess", "deleteDelayDuration": "600s"},
         | 
| 589 | 
            +
                    serialization_alias="podGC",
         | 
| 590 | 
            +
                )
         | 
| 585 591 |  | 
| 586 592 | 
             
                retry_strategy: Retry = Field(default=Retry(), serialization_alias="retryStrategy")
         | 
| 587 593 | 
             
                service_account_name: Optional[str] = Field(default=None, serialization_alias="serviceAccountName")
         | 
| @@ -674,6 +680,8 @@ class ArgoExecutor(GenericExecutor): | |
| 674 680 | 
             
                service_name: str = "argo"
         | 
| 675 681 | 
             
                _local: bool = False
         | 
| 676 682 |  | 
| 683 | 
            +
                # TODO: Add logging level as option.
         | 
| 684 | 
            +
             | 
| 677 685 | 
             
                model_config = ConfigDict(extra="forbid")
         | 
| 678 686 |  | 
| 679 687 | 
             
                image: str
         | 
| @@ -719,6 +727,7 @@ class ArgoExecutor(GenericExecutor): | |
| 719 727 | 
             
                persistent_volumes: List[UserVolumeMounts] = Field(default_factory=list)
         | 
| 720 728 |  | 
| 721 729 | 
             
                _run_id_placeholder: str = "{{workflow.parameters.run_id}}"
         | 
| 730 | 
            +
                _log_level: str = "{{workflow.parameters.log_level}}"
         | 
| 722 731 | 
             
                _container_templates: List[ContainerTemplate] = []
         | 
| 723 732 | 
             
                _dag_templates: List[DagTemplate] = []
         | 
| 724 733 | 
             
                _clean_names: Dict[str, str] = {}
         | 
| @@ -828,17 +837,7 @@ class ArgoExecutor(GenericExecutor): | |
| 828 837 | 
             
                        iterate_on = self._context.run_log_store.get_parameters(self._context.run_id)[node.iterate_on]
         | 
| 829 838 |  | 
| 830 839 | 
             
                        with open("/tmp/output.txt", mode="w", encoding="utf-8") as myfile:
         | 
| 831 | 
            -
                            json.dump(iterate_on, myfile, indent=4)
         | 
| 832 | 
            -
             | 
| 833 | 
            -
                def _get_parameters(self) -> Dict[str, Any]:
         | 
| 834 | 
            -
                    params = {}
         | 
| 835 | 
            -
                    if self._context.parameters_file:
         | 
| 836 | 
            -
                        # Parameters from the parameters file if defined
         | 
| 837 | 
            -
                        params.update(utils.load_yaml(self._context.parameters_file))
         | 
| 838 | 
            -
                    # parameters from environment variables supersede file based
         | 
| 839 | 
            -
                    params.update(parameters.get_user_set_parameters())
         | 
| 840 | 
            -
             | 
| 841 | 
            -
                    return params
         | 
| 840 | 
            +
                            json.dump(iterate_on.get_value(), myfile, indent=4)
         | 
| 842 841 |  | 
| 843 842 | 
             
                def sanitize_name(self, name):
         | 
| 844 843 | 
             
                    return name.replace(" ", "-").replace(".", "-").replace("_", "-")
         | 
| @@ -886,6 +885,7 @@ class ArgoExecutor(GenericExecutor): | |
| 886 885 |  | 
| 887 886 | 
             
                    if working_on.name == self._context.dag.start_at and self.expose_parameters_as_inputs:
         | 
| 888 887 | 
             
                        for key, value in self._get_parameters().items():
         | 
| 888 | 
            +
                            value = value.get_value()  # type: ignore
         | 
| 889 889 | 
             
                            # Get the value from work flow parameters for dynamic behavior
         | 
| 890 890 | 
             
                            if isinstance(value, int) or isinstance(value, float) or isinstance(value, str):
         | 
| 891 891 | 
             
                                env_var = EnvVar(
         | 
| @@ -943,6 +943,7 @@ class ArgoExecutor(GenericExecutor): | |
| 943 943 | 
             
                        node=composite_node,
         | 
| 944 944 | 
             
                        run_id=self._run_id_placeholder,
         | 
| 945 945 | 
             
                        map_variable=map_variable,
         | 
| 946 | 
            +
                        log_level=self._log_level,
         | 
| 946 947 | 
             
                    )
         | 
| 947 948 |  | 
| 948 949 | 
             
                    outputs = []
         | 
| @@ -984,6 +985,7 @@ class ArgoExecutor(GenericExecutor): | |
| 984 985 | 
             
                        node=composite_node,
         | 
| 985 986 | 
             
                        run_id=self._run_id_placeholder,
         | 
| 986 987 | 
             
                        map_variable=map_variable,
         | 
| 988 | 
            +
                        log_level=self._log_level,
         | 
| 987 989 | 
             
                    )
         | 
| 988 990 |  | 
| 989 991 | 
             
                    step_config = {"command": command, "type": "task", "next": "dummy"}
         | 
| @@ -1033,6 +1035,8 @@ class ArgoExecutor(GenericExecutor): | |
| 1033 1035 | 
             
                        if working_on.node_type not in ["success", "fail"] and working_on._get_on_failure_node():
         | 
| 1034 1036 | 
             
                            failure_node = dag.get_node_by_name(working_on._get_on_failure_node())
         | 
| 1035 1037 |  | 
| 1038 | 
            +
                            # same logic, if a template exists, retrieve it
         | 
| 1039 | 
            +
                            # if not, create a new one
         | 
| 1036 1040 | 
             
                            render_obj = get_renderer(working_on)(executor=self, node=failure_node)
         | 
| 1037 1041 | 
             
                            render_obj.render(list_of_iter_values=list_of_iter_values.copy())
         | 
| 1038 1042 |  | 
| @@ -1083,18 +1087,19 @@ class ArgoExecutor(GenericExecutor): | |
| 1083 1087 | 
             
                    # Expose "simple" parameters as workflow arguments for dynamic behavior
         | 
| 1084 1088 | 
             
                    if self.expose_parameters_as_inputs:
         | 
| 1085 1089 | 
             
                        for key, value in self._get_parameters().items():
         | 
| 1090 | 
            +
                            value = value.get_value()  # type: ignore
         | 
| 1086 1091 | 
             
                            if isinstance(value, dict) or isinstance(value, list):
         | 
| 1087 1092 | 
             
                                continue
         | 
| 1088 | 
            -
             | 
| 1093 | 
            +
             | 
| 1094 | 
            +
                            env_var = EnvVar(name=key, value=value)  # type: ignore
         | 
| 1089 1095 | 
             
                            arguments.append(env_var)
         | 
| 1090 1096 |  | 
| 1091 1097 | 
             
                    run_id_var = EnvVar(name="run_id", value="{{workflow.uid}}")
         | 
| 1098 | 
            +
                    log_level_var = EnvVar(name="log_level", value=defaults.LOG_LEVEL)
         | 
| 1092 1099 | 
             
                    arguments.append(run_id_var)
         | 
| 1100 | 
            +
                    arguments.append(log_level_var)
         | 
| 1093 1101 |  | 
| 1094 | 
            -
                    #  | 
| 1095 | 
            -
             | 
| 1096 | 
            -
                    # original_run_id_var = EnvVar(name="original_run_id")
         | 
| 1097 | 
            -
                    # arguments.append(original_run_id_var)
         | 
| 1102 | 
            +
                    # TODO: Can we do reruns?
         | 
| 1098 1103 |  | 
| 1099 1104 | 
             
                    for volume in self.spec.volumes:
         | 
| 1100 1105 | 
             
                        self._container_volumes.append(ContainerVolume(name=volume.name, mount_path=volume.mount_path))
         | 
| @@ -202,7 +202,6 @@ class LocalContainerExecutor(GenericExecutor): | |
| 202 202 | 
             
                                f"Please provide a docker_image using executor_config of the step {node.name} or at global config"
         | 
| 203 203 | 
             
                            )
         | 
| 204 204 |  | 
| 205 | 
            -
                        print("container", self._volumes)
         | 
| 206 205 | 
             
                        # TODO: Should consider using getpass.getuser() when running the docker container? Volume permissions
         | 
| 207 206 | 
             
                        container = client.containers.create(
         | 
| 208 207 | 
             
                            image=docker_image,
         | 
    
        runnable/extensions/nodes.py
    CHANGED
    
    | @@ -15,7 +15,7 @@ from pydantic import ( | |
| 15 15 | 
             
                field_validator,
         | 
| 16 16 | 
             
            )
         | 
| 17 17 |  | 
| 18 | 
            -
            from runnable import datastore, defaults, utils
         | 
| 18 | 
            +
            from runnable import console, datastore, defaults, utils
         | 
| 19 19 | 
             
            from runnable.datastore import (
         | 
| 20 20 | 
             
                JsonParameter,
         | 
| 21 21 | 
             
                MetricParameter,
         | 
| @@ -96,11 +96,10 @@ class TaskNode(ExecutableNode): | |
| 96 96 | 
             
                            attempt_number=attempt_number,
         | 
| 97 97 | 
             
                        )
         | 
| 98 98 |  | 
| 99 | 
            -
                    logger. | 
| 99 | 
            +
                    logger.info(f"attempt_log: {attempt_log}")
         | 
| 100 100 | 
             
                    logger.info(f"Step {self.name} completed with status: {attempt_log.status}")
         | 
| 101 101 |  | 
| 102 102 | 
             
                    step_log.status = attempt_log.status
         | 
| 103 | 
            -
             | 
| 104 103 | 
             
                    step_log.attempts.append(attempt_log)
         | 
| 105 104 |  | 
| 106 105 | 
             
                    return step_log
         | 
| @@ -347,6 +346,7 @@ class ParallelNode(CompositeNode): | |
| 347 346 | 
             
                    for internal_branch_name, _ in self.branches.items():
         | 
| 348 347 | 
             
                        effective_branch_name = self._resolve_map_placeholders(internal_branch_name, map_variable=map_variable)
         | 
| 349 348 | 
             
                        branch_log = self._context.run_log_store.get_branch_log(effective_branch_name, self._context.run_id)
         | 
| 349 | 
            +
             | 
| 350 350 | 
             
                        if branch_log.status != defaults.SUCCESS:
         | 
| 351 351 | 
             
                            step_success_bool = False
         | 
| 352 352 |  | 
| @@ -498,6 +498,8 @@ class MapNode(CompositeNode): | |
| 498 498 | 
             
                            self.internal_name + "." + str(iter_variable), map_variable=map_variable
         | 
| 499 499 | 
             
                        )
         | 
| 500 500 | 
             
                        branch_log = self._context.run_log_store.create_branch_log(effective_branch_name)
         | 
| 501 | 
            +
             | 
| 502 | 
            +
                        console.print(f"Branch log created for {effective_branch_name}: {branch_log}")
         | 
| 501 503 | 
             
                        branch_log.status = defaults.PROCESSING
         | 
| 502 504 | 
             
                        self._context.run_log_store.add_branch_log(branch_log, self._context.run_id)
         | 
| 503 505 |  | 
| @@ -589,6 +591,8 @@ class MapNode(CompositeNode): | |
| 589 591 | 
             
                            self.internal_name + "." + str(iter_variable), map_variable=map_variable
         | 
| 590 592 | 
             
                        )
         | 
| 591 593 | 
             
                        branch_log = self._context.run_log_store.get_branch_log(effective_branch_name, self._context.run_id)
         | 
| 594 | 
            +
                        # console.print(f"Branch log for {effective_branch_name}: {branch_log}")
         | 
| 595 | 
            +
             | 
| 592 596 | 
             
                        if branch_log.status != defaults.SUCCESS:
         | 
| 593 597 | 
             
                            step_success_bool = False
         | 
| 594 598 |  | 
    
        runnable/tasks.py
    CHANGED
    
    | @@ -14,13 +14,10 @@ from string import Template | |
| 14 14 | 
             
            from typing import Any, Dict, List, Literal, Tuple
         | 
| 15 15 |  | 
| 16 16 | 
             
            from pydantic import BaseModel, ConfigDict, Field, field_validator
         | 
| 17 | 
            -
             | 
| 18 | 
            -
            # from rich import print
         | 
| 19 | 
            -
            from rich.console import Console
         | 
| 20 17 | 
             
            from stevedore import driver
         | 
| 21 18 |  | 
| 22 19 | 
             
            import runnable.context as context
         | 
| 23 | 
            -
            from runnable import defaults, exceptions, parameters, utils
         | 
| 20 | 
            +
            from runnable import console, defaults, exceptions, parameters, utils
         | 
| 24 21 | 
             
            from runnable.datastore import (
         | 
| 25 22 | 
             
                JsonParameter,
         | 
| 26 23 | 
             
                MetricParameter,
         | 
| @@ -147,41 +144,26 @@ class BaseTaskType(BaseModel): | |
| 147 144 | 
             
                            if context_param in params:
         | 
| 148 145 | 
             
                                params[param_name].value = params[context_param].value
         | 
| 149 146 |  | 
| 147 | 
            +
                    console.log("Parameters available for the execution:")
         | 
| 148 | 
            +
                    console.log(params)
         | 
| 149 | 
            +
             | 
| 150 150 | 
             
                    logger.debug(f"Resolved parameters: {params}")
         | 
| 151 151 |  | 
| 152 152 | 
             
                    if not allow_complex:
         | 
| 153 153 | 
             
                        params = {key: value for key, value in params.items() if isinstance(value, JsonParameter)}
         | 
| 154 154 |  | 
| 155 | 
            -
                    log_file_name = self._context.executor._context_node.internal_name
         | 
| 156 | 
            -
                    if map_variable:
         | 
| 157 | 
            -
                        for _, value in map_variable.items():
         | 
| 158 | 
            -
                            log_file_name += "_" + str(value)
         | 
| 159 | 
            -
             | 
| 160 | 
            -
                    log_file_name = "".join(x for x in log_file_name if x.isalnum()) + ".execution.log"
         | 
| 161 | 
            -
             | 
| 162 | 
            -
                    log_file = open(log_file_name, "w")
         | 
| 163 | 
            -
             | 
| 164 155 | 
             
                    parameters_in = copy.deepcopy(params)
         | 
| 165 156 | 
             
                    f = io.StringIO()
         | 
| 166 | 
            -
                    task_console = Console(file=io.StringIO())
         | 
| 167 157 | 
             
                    try:
         | 
| 168 158 | 
             
                        with contextlib.redirect_stdout(f):
         | 
| 169 159 | 
             
                            # with contextlib.nullcontext():
         | 
| 170 | 
            -
                            yield params | 
| 171 | 
            -
                            print(task_console.file.getvalue())  # type: ignore
         | 
| 160 | 
            +
                            yield params
         | 
| 172 161 | 
             
                    except Exception as e:  # pylint: disable=broad-except
         | 
| 162 | 
            +
                        console.log(e, style=defaults.error_style)
         | 
| 173 163 | 
             
                        logger.exception(e)
         | 
| 174 164 | 
             
                    finally:
         | 
| 175 | 
            -
                        task_console = None  # type: ignore
         | 
| 176 165 | 
             
                        print(f.getvalue())  # print to console
         | 
| 177 | 
            -
                        log_file.write(f.getvalue())  # Print to file
         | 
| 178 | 
            -
             | 
| 179 166 | 
             
                        f.close()
         | 
| 180 | 
            -
                        log_file.close()
         | 
| 181 | 
            -
             | 
| 182 | 
            -
                        # Put the log file in the catalog
         | 
| 183 | 
            -
                        self._context.catalog_handler.put(name=log_file.name, run_id=context.run_context.run_id)
         | 
| 184 | 
            -
                        os.remove(log_file.name)
         | 
| 185 167 |  | 
| 186 168 | 
             
                        # Update parameters
         | 
| 187 169 | 
             
                        # This should only update the parameters that are changed at the root level.
         | 
| @@ -233,7 +215,7 @@ class PythonTaskType(BaseTaskType):  # pylint: disable=too-few-public-methods | |
| 233 215 | 
             
                    """Execute the notebook as defined by the command."""
         | 
| 234 216 | 
             
                    attempt_log = StepAttempt(status=defaults.FAIL, start_time=str(datetime.now()))
         | 
| 235 217 |  | 
| 236 | 
            -
                    with self.execution_context(map_variable=map_variable) as  | 
| 218 | 
            +
                    with self.execution_context(map_variable=map_variable) as params, self.expose_secrets() as _:
         | 
| 237 219 | 
             
                        module, func = utils.get_module_and_attr_names(self.command)
         | 
| 238 220 | 
             
                        sys.path.insert(0, os.getcwd())  # Need to add the current directory to path
         | 
| 239 221 | 
             
                        imported_module = importlib.import_module(module)
         | 
| @@ -243,9 +225,10 @@ class PythonTaskType(BaseTaskType):  # pylint: disable=too-few-public-methods | |
| 243 225 | 
             
                            try:
         | 
| 244 226 | 
             
                                filtered_parameters = parameters.filter_arguments_for_func(f, params.copy(), map_variable)
         | 
| 245 227 | 
             
                                logger.info(f"Calling {func} from {module} with {filtered_parameters}")
         | 
| 228 | 
            +
             | 
| 246 229 | 
             
                                user_set_parameters = f(**filtered_parameters)  # This is a tuple or single value
         | 
| 247 230 | 
             
                            except Exception as e:
         | 
| 248 | 
            -
                                 | 
| 231 | 
            +
                                console.log(e, style=defaults.error_style, markup=False)
         | 
| 249 232 | 
             
                                raise exceptions.CommandCallError(f"Function call: {self.command} did not succeed.\n") from e
         | 
| 250 233 |  | 
| 251 234 | 
             
                            attempt_log.input_parameters = params.copy()
         | 
| @@ -289,8 +272,8 @@ class PythonTaskType(BaseTaskType):  # pylint: disable=too-few-public-methods | |
| 289 272 | 
             
                        except Exception as _e:
         | 
| 290 273 | 
             
                            msg = f"Call to the function {self.command} did not succeed.\n"
         | 
| 291 274 | 
             
                            attempt_log.message = msg
         | 
| 292 | 
            -
                             | 
| 293 | 
            -
                             | 
| 275 | 
            +
                            console.print_exception(show_locals=False)
         | 
| 276 | 
            +
                            console.log(_e, style=defaults.error_style)
         | 
| 294 277 |  | 
| 295 278 | 
             
                    attempt_log.end_time = str(datetime.now())
         | 
| 296 279 |  | 
| @@ -346,17 +329,17 @@ class NotebookTaskType(BaseTaskType): | |
| 346 329 |  | 
| 347 330 | 
             
                        notebook_output_path = self.notebook_output_path
         | 
| 348 331 |  | 
| 349 | 
            -
                        with self.execution_context( | 
| 350 | 
            -
                             | 
| 351 | 
            -
             | 
| 352 | 
            -
             | 
| 332 | 
            +
                        with self.execution_context(
         | 
| 333 | 
            +
                            map_variable=map_variable, allow_complex=False
         | 
| 334 | 
            +
                        ) as params, self.expose_secrets() as _:
         | 
| 335 | 
            +
                            copy_params = copy.deepcopy(params)
         | 
| 336 | 
            +
             | 
| 353 337 | 
             
                            if map_variable:
         | 
| 354 338 | 
             
                                for key, value in map_variable.items():
         | 
| 355 339 | 
             
                                    notebook_output_path += "_" + str(value)
         | 
| 356 | 
            -
                                     | 
| 340 | 
            +
                                    copy_params[key] = JsonParameter(kind="json", value=value)
         | 
| 357 341 |  | 
| 358 342 | 
             
                            # Remove any {v}_unreduced parameters from the parameters
         | 
| 359 | 
            -
                            copy_params = copy.deepcopy(params)
         | 
| 360 343 | 
             
                            unprocessed_params = [k for k, v in copy_params.items() if not v.reduced]
         | 
| 361 344 |  | 
| 362 345 | 
             
                            for key in list(copy_params.keys()):
         | 
| @@ -397,6 +380,9 @@ class NotebookTaskType(BaseTaskType): | |
| 397 380 | 
             
                                    )
         | 
| 398 381 | 
             
                            except PicklingError as e:
         | 
| 399 382 | 
             
                                logger.exception("Notebooks cannot return objects")
         | 
| 383 | 
            +
                                console.log("Notebooks cannot return objects", style=defaults.error_style)
         | 
| 384 | 
            +
                                console.log(e, style=defaults.error_style)
         | 
| 385 | 
            +
             | 
| 400 386 | 
             
                                logger.exception(e)
         | 
| 401 387 | 
             
                                raise
         | 
| 402 388 |  | 
| @@ -413,6 +399,9 @@ class NotebookTaskType(BaseTaskType): | |
| 413 399 | 
             
                        )
         | 
| 414 400 | 
             
                        logger.exception(msg)
         | 
| 415 401 | 
             
                        logger.exception(e)
         | 
| 402 | 
            +
             | 
| 403 | 
            +
                        console.log(msg, style=defaults.error_style)
         | 
| 404 | 
            +
             | 
| 416 405 | 
             
                        attempt_log.status = defaults.FAIL
         | 
| 417 406 |  | 
| 418 407 | 
             
                    attempt_log.end_time = str(datetime.now())
         | 
| @@ -468,7 +457,7 @@ class ShellTaskType(BaseTaskType): | |
| 468 457 | 
             
                            subprocess_env[key] = secret_value
         | 
| 469 458 |  | 
| 470 459 | 
             
                    try:
         | 
| 471 | 
            -
                        with self.execution_context(map_variable=map_variable, allow_complex=False) as  | 
| 460 | 
            +
                        with self.execution_context(map_variable=map_variable, allow_complex=False) as params:
         | 
| 472 461 | 
             
                            subprocess_env.update({k: v.get_value() for k, v in params.items()})
         | 
| 473 462 |  | 
| 474 463 | 
             
                            # Json dumps all runnable environment variables
         | 
| @@ -499,14 +488,14 @@ class ShellTaskType(BaseTaskType): | |
| 499 488 |  | 
| 500 489 | 
             
                            if proc.returncode != 0:
         | 
| 501 490 | 
             
                                msg = ",".join(result[1].split("\n"))
         | 
| 502 | 
            -
                                 | 
| 491 | 
            +
                                console.print(msg, style=defaults.error_style)
         | 
| 503 492 | 
             
                                raise exceptions.CommandCallError(msg)
         | 
| 504 493 |  | 
| 505 494 | 
             
                            # for stderr
         | 
| 506 495 | 
             
                            for line in result[1].split("\n"):
         | 
| 507 496 | 
             
                                if line.strip() == "":
         | 
| 508 497 | 
             
                                    continue
         | 
| 509 | 
            -
                                 | 
| 498 | 
            +
                                console.print(line, style=defaults.warning_style)
         | 
| 510 499 |  | 
| 511 500 | 
             
                            output_parameters: Dict[str, Parameter] = {}
         | 
| 512 501 | 
             
                            metrics: Dict[str, Parameter] = {}
         | 
| @@ -517,7 +506,7 @@ class ShellTaskType(BaseTaskType): | |
| 517 506 | 
             
                                    continue
         | 
| 518 507 |  | 
| 519 508 | 
             
                                logger.info(line)
         | 
| 520 | 
            -
                                 | 
| 509 | 
            +
                                console.print(line)
         | 
| 521 510 |  | 
| 522 511 | 
             
                                if line.strip() == collect_delimiter:
         | 
| 523 512 | 
             
                                    # The lines from now on should be captured
         | 
| @@ -558,6 +547,10 @@ class ShellTaskType(BaseTaskType): | |
| 558 547 | 
             
                        msg = f"Call to the command {self.command} did not succeed"
         | 
| 559 548 | 
             
                        logger.exception(msg)
         | 
| 560 549 | 
             
                        logger.exception(e)
         | 
| 550 | 
            +
             | 
| 551 | 
            +
                        console.log(msg, style=defaults.error_style)
         | 
| 552 | 
            +
                        console.log(e, style=defaults.error_style)
         | 
| 553 | 
            +
             | 
| 561 554 | 
             
                        attempt_log.status = defaults.FAIL
         | 
| 562 555 |  | 
| 563 556 | 
             
                    attempt_log.end_time = str(datetime.now())
         | 
    
        runnable/utils.py
    CHANGED
    
    | @@ -4,6 +4,8 @@ import hashlib | |
| 4 4 | 
             
            import json
         | 
| 5 5 | 
             
            import logging
         | 
| 6 6 | 
             
            import os
         | 
| 7 | 
            +
            import random
         | 
| 8 | 
            +
            import string
         | 
| 7 9 | 
             
            import subprocess
         | 
| 8 10 | 
             
            from collections import OrderedDict
         | 
| 9 11 | 
             
            from datetime import datetime
         | 
| @@ -394,6 +396,7 @@ def get_node_execution_command( | |
| 394 396 | 
             
                node: BaseNode,
         | 
| 395 397 | 
             
                map_variable: TypeMapVariable = None,
         | 
| 396 398 | 
             
                over_write_run_id: str = "",
         | 
| 399 | 
            +
                log_level: str = "",
         | 
| 397 400 | 
             
            ) -> str:
         | 
| 398 401 | 
             
                """A utility function to standardize execution call to a node via command line.
         | 
| 399 402 |  | 
| @@ -410,7 +413,7 @@ def get_node_execution_command( | |
| 410 413 | 
             
                if over_write_run_id:
         | 
| 411 414 | 
             
                    run_id = over_write_run_id
         | 
| 412 415 |  | 
| 413 | 
            -
                log_level = logging.getLevelName(logger.getEffectiveLevel())
         | 
| 416 | 
            +
                log_level = log_level or logging.getLevelName(logger.getEffectiveLevel())
         | 
| 414 417 |  | 
| 415 418 | 
             
                action = f"runnable execute_single_node {run_id} " f"{node._command_friendly_name()}" f" --log-level {log_level}"
         | 
| 416 419 |  | 
| @@ -437,6 +440,7 @@ def get_fan_command( | |
| 437 440 | 
             
                node: BaseNode,
         | 
| 438 441 | 
             
                run_id: str,
         | 
| 439 442 | 
             
                map_variable: TypeMapVariable = None,
         | 
| 443 | 
            +
                log_level: str = "",
         | 
| 440 444 | 
             
            ) -> str:
         | 
| 441 445 | 
             
                """
         | 
| 442 446 | 
             
                An utility function to return the fan "in or out" command
         | 
| @@ -451,7 +455,7 @@ def get_fan_command( | |
| 451 455 | 
             
                Returns:
         | 
| 452 456 | 
             
                    str: The fan in or out command
         | 
| 453 457 | 
             
                """
         | 
| 454 | 
            -
                log_level = logging.getLevelName(logger.getEffectiveLevel())
         | 
| 458 | 
            +
                log_level = log_level or logging.getLevelName(logger.getEffectiveLevel())
         | 
| 455 459 | 
             
                action = (
         | 
| 456 460 | 
             
                    f"runnable fan {run_id} "
         | 
| 457 461 | 
             
                    f"{node._command_friendly_name()} "
         | 
| @@ -614,3 +618,17 @@ def gather_variables() -> dict: | |
| 614 618 | 
             
                        variables[key] = value
         | 
| 615 619 |  | 
| 616 620 | 
             
                return variables
         | 
| 621 | 
            +
             | 
| 622 | 
            +
             | 
| 623 | 
            +
            def make_log_file_name(node: BaseNode, map_variable: TypeMapVariable) -> str:
         | 
| 624 | 
            +
                random_tag = "".join(random.choices(string.ascii_uppercase + string.digits, k=3))
         | 
| 625 | 
            +
                log_file_name = node.name
         | 
| 626 | 
            +
             | 
| 627 | 
            +
                if map_variable:
         | 
| 628 | 
            +
                    for _, value in map_variable.items():
         | 
| 629 | 
            +
                        log_file_name += "_" + str(value)
         | 
| 630 | 
            +
             | 
| 631 | 
            +
                log_file_name += "_" + random_tag
         | 
| 632 | 
            +
                log_file_name = "".join(x for x in log_file_name if x.isalnum()) + ".execution.log"
         | 
| 633 | 
            +
             | 
| 634 | 
            +
                return log_file_name
         | 
| @@ -1,10 +1,10 @@ | |
| 1 | 
            -
            runnable/__init__.py,sha256= | 
| 1 | 
            +
            runnable/__init__.py,sha256=zaZVsNzp9zrpngSMNLzT_aIFkkWSpp9gb_K-GAnr9Cw,1001
         | 
| 2 2 | 
             
            runnable/catalog.py,sha256=22OECi5TrpHErxYIhfx-lJ2vgBUi4-5V9CaYEVm98hE,4138
         | 
| 3 3 | 
             
            runnable/cli.py,sha256=RILUrEfzernuKD3dNdXPBkqN_1OgE5GosYRuInj0FVs,9618
         | 
| 4 4 | 
             
            runnable/context.py,sha256=QhiXJHRcEBfSKB1ijvL5yB9w44x0HCe7VEiwK1cUJ9U,1124
         | 
| 5 5 | 
             
            runnable/datastore.py,sha256=8aQZ15KAMdre7a7G61bNRmcTeJFzOdnx_9O9UP4JQc8,27910
         | 
| 6 6 | 
             
            runnable/defaults.py,sha256=MOX7I2S6yO4FphZaZREFQca94a20oO8uvzXLd6GLKQs,4703
         | 
| 7 | 
            -
            runnable/entrypoints.py,sha256= | 
| 7 | 
            +
            runnable/entrypoints.py,sha256=fLipciON3x2iAkBE9w00TrRPqwLxyLJeKu4V7dlgB-A,17611
         | 
| 8 8 | 
             
            runnable/exceptions.py,sha256=6NIYoTAzdKyGQ9PvW1Hu7b80OS746395KiGDhM7ThH8,2526
         | 
| 9 9 | 
             
            runnable/executor.py,sha256=xfBighQ5t_vejohip000XfxLwsgechUE1ZMIJWrZbUA,14484
         | 
| 10 10 | 
             
            runnable/extensions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
         | 
| @@ -16,7 +16,7 @@ runnable/extensions/catalog/k8s_pvc/implementation.py,sha256=oJDDI0APT7lrtjWmzYJ | |
| 16 16 | 
             
            runnable/extensions/catalog/k8s_pvc/integration.py,sha256=OfrHbNFN8sR-wsVa4os3ajmWJFSd5H4KOHGVAmjRZTQ,1850
         | 
| 17 17 | 
             
            runnable/extensions/executor/__init__.py,sha256=0385OpNSpjyA0GjXlLw7gZtqJFFOHGLmYHzWAGBzU98,26247
         | 
| 18 18 | 
             
            runnable/extensions/executor/argo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
         | 
| 19 | 
            -
            runnable/extensions/executor/argo/implementation.py,sha256= | 
| 19 | 
            +
            runnable/extensions/executor/argo/implementation.py,sha256=RMvHLt1VEHb4_C8dEJomistRztycThHfcX9l4CXVp_s,43910
         | 
| 20 20 | 
             
            runnable/extensions/executor/argo/specification.yaml,sha256=wXQcm2gOQYqy-IOQIhucohS32ZrHKCfGA5zZ0RraPYc,1276
         | 
| 21 21 | 
             
            runnable/extensions/executor/k8s_job/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
         | 
| 22 22 | 
             
            runnable/extensions/executor/k8s_job/implementation_FF.py,sha256=1IfVG1GRcJcVFzQ-WhkJsmzdJuj51QMxXylY9UrWM0U,10259
         | 
| @@ -24,12 +24,12 @@ runnable/extensions/executor/k8s_job/integration_FF.py,sha256=pG6HKhPMgCRIgu1PAn | |
| 24 24 | 
             
            runnable/extensions/executor/local/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
         | 
| 25 25 | 
             
            runnable/extensions/executor/local/implementation.py,sha256=e8Tzv-FgQmJeUXVut96jeNERTR83JVG_zkQZMEjCVAs,2469
         | 
| 26 26 | 
             
            runnable/extensions/executor/local_container/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
         | 
| 27 | 
            -
            runnable/extensions/executor/local_container/implementation.py,sha256= | 
| 27 | 
            +
            runnable/extensions/executor/local_container/implementation.py,sha256=Kley2sjdZMe7E5CHjjy4YaJ5YErNGvZmpVW33h6D0W0,14848
         | 
| 28 28 | 
             
            runnable/extensions/executor/mocked/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
         | 
| 29 29 | 
             
            runnable/extensions/executor/mocked/implementation.py,sha256=ChvlcLGpBxO6QwJcoqhBgKBR6NfWVnMdOWKQhMgcEjY,5762
         | 
| 30 30 | 
             
            runnable/extensions/executor/retry/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
         | 
| 31 31 | 
             
            runnable/extensions/executor/retry/implementation.py,sha256=-g6PBOhSG7IL4D_IlQOcf9H_En9IXiUzCt-6vKeCB6Q,6892
         | 
| 32 | 
            -
            runnable/extensions/nodes.py,sha256= | 
| 32 | 
            +
            runnable/extensions/nodes.py,sha256=SvuaXcy9uqJSG4nj-sFZc-aHswZ0v4HmAoaZNQXcu-Y,32831
         | 
| 33 33 | 
             
            runnable/extensions/run_log_store/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
         | 
| 34 34 | 
             
            runnable/extensions/run_log_store/chunked_file_system/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
         | 
| 35 35 | 
             
            runnable/extensions/run_log_store/chunked_file_system/implementation.py,sha256=EW2P8lr3eH-pIOsMTJPr5eb-iWc48GQ97W15JzkpC_4,3326
         | 
| @@ -55,10 +55,10 @@ runnable/parameters.py,sha256=yZkMDnwnkdYXIwQ8LflBzn50Y0xRGxEvLlxwno6ovvs,5163 | |
| 55 55 | 
             
            runnable/pickler.py,sha256=5SDNf0miMUJ3ZauhQdzwk8_t-9jeOqaTjP5bvRnu9sU,2685
         | 
| 56 56 | 
             
            runnable/sdk.py,sha256=t6d1Q3BoovixqC29QuSjFEwsleVgM0E-pAQlfCfMz_o,27923
         | 
| 57 57 | 
             
            runnable/secrets.py,sha256=dakb7WRloWVo-KpQp6Vy4rwFdGi58BTlT4OifQY106I,2324
         | 
| 58 | 
            -
            runnable/tasks.py,sha256= | 
| 59 | 
            -
            runnable/utils.py,sha256= | 
| 60 | 
            -
            runnable-0.11. | 
| 61 | 
            -
            runnable-0.11. | 
| 62 | 
            -
            runnable-0.11. | 
| 63 | 
            -
            runnable-0.11. | 
| 64 | 
            -
            runnable-0.11. | 
| 58 | 
            +
            runnable/tasks.py,sha256=a5Fkvl58ku9JPo-qDDJahemD-6a7jJlXlBEowvmeKuc,21910
         | 
| 59 | 
            +
            runnable/utils.py,sha256=fXOLoFZKYqh3wQgzA2V-VZOu-dSgLPGqCZIbMmsNzOw,20016
         | 
| 60 | 
            +
            runnable-0.11.5.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
         | 
| 61 | 
            +
            runnable-0.11.5.dist-info/METADATA,sha256=tGgeniqJm3X51QiOeZV52fXjWAOZgOiBTL9qtsNKWOY,17020
         | 
| 62 | 
            +
            runnable-0.11.5.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
         | 
| 63 | 
            +
            runnable-0.11.5.dist-info/entry_points.txt,sha256=amb6ISqKBSIz47um8_6LKnYgpoZ4d_p6-O1-7uUb1cU,1447
         | 
| 64 | 
            +
            runnable-0.11.5.dist-info/RECORD,,
         | 
| 
            File without changes
         | 
| 
            File without changes
         | 
| 
            File without changes
         |