runnable 0.11.0__tar.gz → 0.11.2__tar.gz

{runnable-0.11.0 → runnable-0.11.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: runnable
-Version: 0.11.0
+Version: 0.11.2
 Summary: A Compute agnostic pipelining software
 Home-page: https://github.com/vijayvammi/runnable
 License: Apache-2.0
@@ -15,13 +15,12 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Provides-Extra: database
 Provides-Extra: docker
-Provides-Extra: mlflow
 Provides-Extra: notebook
 Requires-Dist: click
 Requires-Dist: click-plugins (>=1.1.1,<2.0.0)
 Requires-Dist: dill (>=0.3.8,<0.4.0)
 Requires-Dist: docker ; extra == "docker"
-Requires-Dist: mlflow-skinny ; extra == "mlflow"
+Requires-Dist: mlflow-skinny
 Requires-Dist: ploomber-engine (>=0.0.31,<0.0.32) ; extra == "notebook"
 Requires-Dist: pydantic (>=2.5,<3.0)
 Requires-Dist: rich (>=13.5.2,<14.0.0)

{runnable-0.11.0 → runnable-0.11.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "runnable"
-version = "0.11.0"
+version = "0.11.2"
 description = "A Compute agnostic pipelining software"
 authors = ["Vijay Vammi <mesanthu@gmail.com>"]
 license = "Apache-2.0"
@@ -49,11 +49,18 @@ scikit-learn = "^1.4.1.post1"
 en-core-web-sm = { url = "https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.7.1/en_core_web_sm-3.7.1.tar.gz" }
 matplotlib = "^3.8.3"
 
+
+[tool.poetry.group.release.dependencies]
+python-semantic-release = "^9.4.2"
+
+
+[tool.poetry.group.examples.dependencies]
+pandas = "^2.2.2"
+
 [tool.poetry.extras]
 docker = ['docker']
 notebook = ['ploomber-engine']
 database = ["sqlalchemy"]
-mlflow = ["mlflow-skinny"]
 
 [tool.poetry.group.dev.dependencies]
 pytest = "*"
@@ -66,6 +73,7 @@ ruff = "^0.0.259"
 commit-linter = "^1.0.2"
 black = "^23.3.0"
 gitlint = "^0.19.1"
+pandas = "^2.2.2"
 
 
 [tool.poetry.scripts]
@@ -191,22 +199,16 @@ requires = ["poetry-core>=1.0.0"]
 build-backend = "poetry.core.masonry.api"
 
 [tool.semantic_release]
-tag_commit = false
-major_on_zero = true
-commit_version_number = false
-upload_to_pypi = false
-upload_to_repository = false
-upload_to_release = false
+allow_zero_version = true
+major_on_zero = false
 tag_format = "{version}"
 
+[tool.semantic_release.remote.token]
+env = "GH_TOKEN"
+
 [tool.semantic_release.branches.main]
 match = "main"
 
-[tool.semantic_release.branches."rc"]
-match = "rc"
-prerelease = true
-prerelease_token = "rc"
-
 [tool.semantic_release.remote]
 ignore_token_for_push = true
 

{runnable-0.11.0 → runnable-0.11.2}/runnable/__init__.py

@@ -29,8 +29,13 @@ from runnable.sdk import (  # noqa
     pickled,
 )
 
+## TODO: Summary should be a bit better for catalog.
+## If the execution fails, hint them about the retry executor.
+# Make the retry executor loose!
+
 # TODO: Think of model registry as a central place to store models.
 # TODO: Implement Sagemaker pipelines as a executor.
 
 
 # TODO: Think of way of generating dag hash without executor configuration
+# TODO: Add examples of map parameters and task types

{runnable-0.11.0 → runnable-0.11.2}/runnable/cli.py

@@ -1,3 +1,4 @@
+# A dummy to trigger the PR
 import logging
 
 import click

{runnable-0.11.0 → runnable-0.11.2}/runnable/datastore.py

@@ -312,8 +312,10 @@ class RunLog(BaseModel):
         summary["Catalog Location"] = _context.catalog_handler.get_summary()
         summary["Full Run log present at: "] = _context.run_log_store.get_summary()
 
-        summary["Final Parameters"] = {p: v.description for p, v in self.parameters.items()}
-        summary["Collected metrics"] = {p: v.description for p, v in self.parameters.items() if v.kind == "metric"}
+        run_log = _context.run_log_store.get_run_log_by_id(run_id=_context.run_id, full=True)
+
+        summary["Final Parameters"] = {p: v.description for p, v in run_log.parameters.items()}
+        summary["Collected metrics"] = {p: v.description for p, v in run_log.parameters.items() if v.kind == "metric"}
 
         return summary
 

{runnable-0.11.0 → runnable-0.11.2}/runnable/entrypoints.py

@@ -172,6 +172,7 @@ def execute(
     )
     console.print("Working with context:")
     console.print(run_context)
+    console.rule(style="[dark orange]")
 
     executor = run_context.executor
 
@@ -243,6 +244,7 @@ def execute_single_node(
     )
     console.print("Working with context:")
     console.print(run_context)
+    console.rule(style="[dark orange]")
 
     executor = run_context.executor
     run_context.execution_plan = defaults.EXECUTION_PLAN.CHAINED.value
@@ -296,6 +298,7 @@ def execute_notebook(
 
     console.print("Working with context:")
     console.print(run_context)
+    console.rule(style="[dark orange]")
 
     step_config = {
         "command": notebook_file,
@@ -358,6 +361,7 @@ def execute_function(
 
     console.print("Working with context:")
     console.print(run_context)
+    console.rule(style="[dark orange]")
 
     # Prepare the graph with a single node
     step_config = {
@@ -427,6 +431,7 @@ def fan(
     )
     console.print("Working with context:")
     console.print(run_context)
+    console.rule(style="[dark orange]")
 
     executor = run_context.executor
     run_context.execution_plan = defaults.EXECUTION_PLAN.CHAINED.value

{runnable-0.11.0 → runnable-0.11.2}/runnable/extensions/catalog/file_system/implementation.py

@@ -226,7 +226,7 @@ class FileSystemCatalog(BaseCatalog):
         for cataloged_file in cataloged_files:
             if str(cataloged_file).endswith("execution.log"):
                 continue
-            print(cataloged_file.name)
+
             if cataloged_file.is_file():
                 shutil.copy(cataloged_file, run_catalog / cataloged_file.name)
             else:

{runnable-0.11.0 → runnable-0.11.2}/runnable/extensions/executor/__init__.py

@@ -476,6 +476,8 @@ class GenericExecutor(BaseExecutor):
                 logger.exception(e)
                 raise
 
+            console.rule(style="[dark orange]")
+
             if working_on.node_type in ["success", "fail"]:
                 break
 

{runnable-0.11.0 → runnable-0.11.2}/runnable/extensions/nodes.py

@@ -505,7 +505,7 @@ class MapNode(CompositeNode):
             for _, v in map_variable.items():
                 for branch_return in self.branch_returns:
                     param_name, param_type = branch_return
-                    raw_parameters[f"{param_name}_{v}"] = param_type.copy()
+                    raw_parameters[f"{v}_{param_name}"] = param_type.copy()
         else:
             for branch_return in self.branch_returns:
                 param_name, param_type = branch_return
@@ -606,9 +606,9 @@ class MapNode(CompositeNode):
                     param_name, _ = branch_return
                     to_reduce = []
                     for iter_variable in iterate_on:
-                        to_reduce.append(params[f"{param_name}_{iter_variable}"].get_value())
+                        to_reduce.append(params[f"{iter_variable}_{param_name}"].get_value())
 
-                    param_name = f"{param_name}_{v}"
+                    param_name = f"{v}_{param_name}"
                     params[param_name].value = reducer_f(to_reduce)
                     params[param_name].reduced = True
         else:
@@ -617,7 +617,7 @@ class MapNode(CompositeNode):
 
                 to_reduce = []
                 for iter_variable in iterate_on:
-                    to_reduce.append(params[f"{param_name}_{iter_variable}"].get_value())
+                    to_reduce.append(params[f"{iter_variable}_{param_name}"].get_value())
 
                 params[param_name].value = reducer_f(*to_reduce)
                 params[param_name].reduced = True

{runnable-0.11.0 → runnable-0.11.2}/runnable/extensions/run_log_store/generic_chunked.py

@@ -7,7 +7,16 @@ from string import Template
 from typing import Any, Dict, Optional, Sequence, Union
 
 from runnable import defaults, exceptions
-from runnable.datastore import BaseRunLogStore, BranchLog, RunLog, StepLog
+from runnable.datastore import (
+    BaseRunLogStore,
+    BranchLog,
+    JsonParameter,
+    MetricParameter,
+    ObjectParameter,
+    Parameter,
+    RunLog,
+    StepLog,
+)
 
 logger = logging.getLogger(defaults.LOGGER_NAME)
 
@@ -164,7 +173,9 @@ class ChunkedRunLogStore(BaseRunLogStore):
             raise Exception(f"Name is required during retrieval for {log_type}")
 
         naming_pattern = self.naming_pattern(log_type=log_type, name=name)
+
         matches = self.get_matches(run_id=run_id, name=naming_pattern, multiple_allowed=multiple_allowed)
+
         if matches:
             if not multiple_allowed:
                 contents = self._retrieve(name=matches)  # type: ignore
@@ -370,10 +381,17 @@ class ChunkedRunLogStore(BaseRunLogStore):
         Raises:
             RunLogNotFoundError: If the run log for run_id is not found in the datastore
         """
-        parameters = {}
+        parameters: Dict[str, Parameter] = {}
         try:
             parameters_list = self.retrieve(run_id=run_id, log_type=self.LogTypes.PARAMETER, multiple_allowed=True)
-            parameters = {key: value for param in parameters_list for key, value in param.items()}
+            for param in parameters_list:
+                for key, value in param.items():
+                    if value["kind"] == "json":
+                        parameters[key] = JsonParameter(**value)
+                    if value["kind"] == "metric":
+                        parameters[key] = MetricParameter(**value)
+                    if value["kind"] == "object":
+                        parameters[key] = ObjectParameter(**value)
         except EntityNotFoundError:
             # No parameters are set
             pass
@@ -401,7 +419,7 @@ class ChunkedRunLogStore(BaseRunLogStore):
             self.store(
                 run_id=run_id,
                 log_type=self.LogTypes.PARAMETER,
-                contents={key: value},
+                contents={key: value.model_dump(by_alias=True)},
                 name=key,
             )
 

{runnable-0.11.0 → runnable-0.11.2}/runnable/sdk.py

@@ -15,8 +15,13 @@ from pydantic import (
     field_validator,
     model_validator,
 )
-from rich import print
-from rich.progress import BarColumn, Progress, TextColumn, TimeElapsedColumn
+from rich.progress import (
+    BarColumn,
+    Progress,
+    SpinnerColumn,
+    TextColumn,
+    TimeElapsedColumn,
+)
 from rich.table import Column
 from typing_extensions import Self
 
@@ -71,7 +76,7 @@ class Catalog(BaseModel):
 
 class BaseTraversal(ABC, BaseModel):
     name: str
-    next_node: str = Field(default="", alias="next")
+    next_node: str = Field(default="", serialization_alias="next_node")
     terminate_with_success: bool = Field(default=False, exclude=True)
     terminate_with_failure: bool = Field(default=False, exclude=True)
     on_failure: str = Field(default="", alias="on_failure")
@@ -83,6 +88,12 @@ class BaseTraversal(ABC, BaseModel):
     def internal_name(self) -> str:
         return self.name
 
+    def __hash__(self):
+        """
+        Needed to Uniqueize DataCatalog objects.
+        """
+        return hash(self.name)
+
     def __rshift__(self, other: StepType) -> StepType:
         if self.next_node:
             raise Exception(f"The node {self} already has a next node: {self.next_node}")
@@ -180,6 +191,7 @@ class BaseTask(BaseTraversal):
     catalog: Optional[Catalog] = Field(default=None, alias="catalog")
     overrides: Dict[str, Any] = Field(default_factory=dict, alias="overrides")
     returns: List[Union[str, TaskReturns]] = Field(default_factory=list, alias="returns")
+    secrets: List[str] = Field(default_factory=list)
 
     @field_validator("returns", mode="before")
     @classmethod
@@ -201,7 +213,7 @@ class BaseTask(BaseTraversal):
             if not (self.terminate_with_failure or self.terminate_with_success):
                 raise AssertionError("A node not being terminated must have a user defined next node")
 
-        return TaskNode.parse_from_config(self.model_dump(exclude_none=True))
+        return TaskNode.parse_from_config(self.model_dump(exclude_none=True, by_alias=True))
 
 
 class PythonTask(BaseTask):
@@ -297,9 +309,9 @@ class NotebookTask(BaseTask):
 
     """
 
-    notebook: str = Field(alias="command")
+    notebook: str = Field(serialization_alias="command")
 
-    notebook_output_path: Optional[str] = Field(default=None, alias="notebook_output_path")
+    notebook_output_path: Optional[str] = Field(default=None, alias="notebook_output_path", validate_default=True)
     optional_ploomber_args: Optional[Dict[str, Any]] = Field(default=None, alias="optional_ploomber_args")
 
     @computed_field
@@ -526,7 +538,7 @@ class Pipeline(BaseModel):
     _dag: graph.Graph = PrivateAttr()
     model_config = ConfigDict(extra="forbid")
 
-    def _validate_path(self, path: List[StepType]) -> None:
+    def _validate_path(self, path: List[StepType], failure_path: bool = False) -> None:
         # Check if one and only one step terminates with success
         # Check no more than one step terminates with failure
 
@@ -544,7 +556,7 @@ class Pipeline(BaseModel):
                     raise Exception("A pipeline cannot have more than one step that terminates with failure")
                 reached_failure = True
 
-        if not reached_success:
+        if not reached_success and not reached_failure:
             raise Exception("A pipeline must have at least one step that terminates with success")
 
     def _construct_path(self, path: List[StepType]) -> None:
@@ -594,11 +606,21 @@ class Pipeline(BaseModel):
 
         # Check all paths are valid and construct the path
         paths = [success_path] + on_failure_paths
+        failure_path = False
         for path in paths:
-            self._validate_path(path)
+            self._validate_path(path, failure_path)
             self._construct_path(path)
 
-        all_steps: List[StepType] = [step for step in success_path + on_failure_paths]  # type: ignore
+            failure_path = True
+
+        all_steps: List[StepType] = []
+
+        for path in paths:
+            for step in path:
+                all_steps.append(step)
+
+        seen = set()
+        unique = [x for x in all_steps if not (x in seen or seen.add(x))]  # type: ignore
 
         self._dag = graph.Graph(
             start_at=all_steps[0].name,
@@ -606,7 +628,7 @@ class Pipeline(BaseModel):
             internal_branch_name=self.internal_branch_name,
         )
 
-        for step in all_steps:
+        for step in unique:
             self._dag.add_node(step.create_node())
 
         if self.add_terminal_nodes:
@@ -675,8 +697,9 @@ class Pipeline(BaseModel):
 
         run_context.dag = graph.create_graph(dag_definition)
 
-        print("Working with context:")
-        print(run_context)
+        console.print("Working with context:")
+        console.print(run_context)
+        console.rule(style="[dark orange]")
 
         if not run_context.executor._local:
             # We are not working with non local executor
@@ -693,6 +716,7 @@ class Pipeline(BaseModel):
         run_context.executor.prepare_for_graph_execution()
 
         with Progress(
+            SpinnerColumn(spinner_name="runner"),
             TextColumn("[progress.description]{task.description}", table_column=Column(ratio=2)),
             BarColumn(table_column=Column(ratio=1), style="dark_orange"),
             TimeElapsedColumn(table_column=Column(ratio=1)),

{runnable-0.11.0 → runnable-0.11.2}/runnable/tasks.py

@@ -1,4 +1,5 @@
 import contextlib
+import copy
 import importlib
 import io
 import json
@@ -9,13 +10,14 @@ import sys
 from datetime import datetime
 from pickle import PicklingError
 from string import Template
-from typing import Any, Dict, List, Literal, Tuple
+from typing import Any, Dict, List, Literal, Optional, Tuple
 
 from pydantic import BaseModel, ConfigDict, Field, ValidationInfo, field_validator
+from rich.console import Console
 from stevedore import driver
 
 import runnable.context as context
-from runnable import console, defaults, exceptions, parameters, utils
+from runnable import defaults, exceptions, parameters, utils
 from runnable.datastore import (
     JsonParameter,
     MetricParameter,
@@ -32,6 +34,9 @@ logging.getLogger("stevedore").setLevel(logging.CRITICAL)
 # TODO: Can we add memory peak, cpu usage, etc. to the metrics?
 
 
+console = Console(file=io.StringIO())
+
+
 class TaskReturns(BaseModel):
     name: str
     kind: Literal["json", "object", "metric"] = Field(default="json")
@@ -42,7 +47,7 @@ class BaseTaskType(BaseModel):
 
     task_type: str = Field(serialization_alias="command_type")
     node_name: str = Field(exclude=True)
-    secrets: Dict[str, str] = Field(default_factory=dict)
+    secrets: List[str] = Field(default_factory=list)
     returns: List[TaskReturns] = Field(default_factory=list, alias="returns")
 
     model_config = ConfigDict(extra="forbid")
@@ -69,15 +74,14 @@ class BaseTaskType(BaseModel):
         raise NotImplementedError()
 
     def set_secrets_as_env_variables(self):
-        for key, value in self.secrets.items():
+        for key in self.secrets:
             secret_value = context.run_context.secrets_handler.get(key)
-            self.secrets[value] = secret_value
-            os.environ[value] = secret_value
+            os.environ[key] = secret_value
 
     def delete_secrets_from_env_variables(self):
-        for _, value in self.secrets.items():
-            if value in os.environ:
-                del os.environ[value]
+        for key in self.secrets:
+            if key in os.environ:
+                del os.environ[key]
 
     def execute_command(
         self,
@@ -96,6 +100,20 @@ class BaseTaskType(BaseModel):
         """
         raise NotImplementedError()
 
+    def _diff_parameters(
+        self, parameters_in: Dict[str, Parameter], context_params: Dict[str, Parameter]
+    ) -> Dict[str, Parameter]:
+        diff: Dict[str, Parameter] = {}
+        for param_name, param in context_params.items():
+            if param_name in parameters_in:
+                if parameters_in[param_name] != param:
+                    diff[param_name] = param
+                continue
+
+            diff[param_name] = param
+
+        return diff
+
     @contextlib.contextmanager
     def expose_secrets(self):
         """Context manager to expose secrets to the execution.
@@ -125,7 +143,7 @@ class BaseTaskType(BaseModel):
             if param.reduced is False:
                 context_param = param_name
                 for _, v in map_variable.items():  # type: ignore
-                    context_param = f"{context_param}_{v}"
+                    context_param = f"{v}_{context_param}"
 
                 if context_param in params:
                     params[param_name].value = params[context_param].value
@@ -135,17 +153,23 @@ class BaseTaskType(BaseModel):
         if not allow_complex:
             params = {key: value for key, value in params.items() if isinstance(value, JsonParameter)}
 
-        log_file_name = self.node_name.replace(" ", "_") + ".execution.log"
+        log_file_name = self.node_name  # + ".execution.log"
         if map_variable:
             for _, value in map_variable.items():
                 log_file_name += "_" + str(value)
 
+        log_file_name = "".join(x for x in log_file_name if x.isalnum()) + ".execution.log"
+
         log_file = open(log_file_name, "w")
 
+        parameters_in = copy.deepcopy(params)
+
         f = io.StringIO()
         try:
             with contextlib.redirect_stdout(f):
+                # with contextlib.nullcontext():
                 yield params
+                print(console.file.getvalue())  # type: ignore
         except Exception as e:  # pylint: disable=broad-except
             logger.exception(e)
         finally:
@@ -156,11 +180,13 @@ class BaseTaskType(BaseModel):
             log_file.close()
 
             # Put the log file in the catalog
-            # self._context.catalog_handler.put(name=log_file.name, run_id=context.run_context.run_id)
+            self._context.catalog_handler.put(name=log_file.name, run_id=context.run_context.run_id)
             os.remove(log_file.name)
 
             # Update parameters
-            self._context.run_log_store.set_parameters(parameters=params, run_id=self._context.run_id)
+            # This should only update the parameters that are changed at the root level.
+            diff_parameters = self._diff_parameters(parameters_in=parameters_in, context_params=params)
+            self._context.run_log_store.set_parameters(parameters=diff_parameters, run_id=self._context.run_id)
 
 
 def task_return_to_parameter(task_return: TaskReturns, value: Any) -> Parameter:
@@ -219,8 +245,7 @@ class PythonTaskType(BaseTaskType):  # pylint: disable=too-few-public-methods
                     logger.info(f"Calling {func} from {module} with {filtered_parameters}")
                     user_set_parameters = f(**filtered_parameters)  # This is a tuple or single value
                 except Exception as e:
-                    logger.exception(e)
-                    console.print(e, style=defaults.error_style)
+                    console.log(e, style=defaults.error_style, markup=False)
                     raise exceptions.CommandCallError(f"Function call: {self.command} did not succeed.\n") from e
 
                 attempt_log.input_parameters = params.copy()
@@ -252,7 +277,7 @@ class PythonTaskType(BaseTaskType):  # pylint: disable=too-few-public-methods
                         param_name = task_return.name
                         if map_variable:
                             for _, v in map_variable.items():
-                                param_name = f"{param_name}_{v}"
+                                param_name = f"{v}_{param_name}"
 
                         output_parameters[param_name] = output_parameter
 
@@ -263,9 +288,9 @@ class PythonTaskType(BaseTaskType):  # pylint: disable=too-few-public-methods
                 attempt_log.status = defaults.SUCCESS
             except Exception as _e:
                 msg = f"Call to the function {self.command} did not succeed.\n"
-                logger.exception(_e)
                 attempt_log.message = msg
-                console.print(_e, style=defaults.error_style)
+                console.print_exception(show_locals=False)
+                console.log(_e, style=defaults.error_style)
 
         attempt_log.end_time = str(datetime.now())
 
@@ -277,7 +302,7 @@ class NotebookTaskType(BaseTaskType):
 
     task_type: str = Field(default="notebook", serialization_alias="command_type")
     command: str
-    notebook_output_path: str = Field(default="", validate_default=True)
+    notebook_output_path: Optional[str] = Field(default=None, validate_default=True)
     optional_ploomber_args: dict = {}
 
     @field_validator("command")
@@ -319,7 +344,7 @@ class NotebookTaskType(BaseTaskType):
             import ploomber_engine as pm
             from ploomber_engine.ipython import PloomberClient
 
-            notebook_output_path = self.notebook_output_path
+            notebook_output_path = self.notebook_output_path or ""
 
             with self.execution_context(
                 map_variable=map_variable, allow_complex=False
@@ -424,15 +449,17 @@ class ShellTaskType(BaseTaskType):
 
         # Expose secrets as environment variables
         if self.secrets:
-            for key, value in self.secrets.items():
+            for key in self.secrets:
                 secret_value = context.run_context.secrets_handler.get(key)
-                subprocess_env[value] = secret_value
+                subprocess_env[key] = secret_value
 
         with self.execution_context(map_variable=map_variable, allow_complex=False) as params:
             subprocess_env.update({k: v.get_value() for k, v in params.items()})
 
             # Json dumps all runnable environment variables
             for key, value in subprocess_env.items():
+                if isinstance(value, str):
+                    continue
                 subprocess_env[key] = json.dumps(value)
 
             collect_delimiter = "=== COLLECT ==="
@@ -441,37 +468,80 @@ class ShellTaskType(BaseTaskType):
             logger.info(f"Executing shell command: {command}")
 
             capture = False
-            return_keys = [x.name for x in self.returns]
+            return_keys = {x.name: x for x in self.returns}
 
-            with subprocess.Popen(
+            proc = subprocess.Popen(
                 command,
                 shell=True,
                 env=subprocess_env,
                 stdout=subprocess.PIPE,
                 stderr=subprocess.PIPE,
                 text=True,
-            ) as proc:
-                for line in proc.stdout:  # type: ignore
-                    logger.info(line)
-                    print(line)
-
-                    if line.strip() == collect_delimiter:
-                        # The lines from now on should be captured
-                        capture = True
-                        continue
-
-                    if capture:
-                        key, value = line.strip().split("=", 1)
-                        if key in (return_keys or []):
-                            param_name = Template(key).safe_substitute(map_variable)  # type: ignore
-                            try:
-                                params[param_name] = JsonParameter(kind="json", value=json.loads(value))
-                            except json.JSONDecodeError:
-                                params[param_name] = JsonParameter(kind="json", value=value)
-
-                proc.wait()
-                if proc.returncode == 0:
-                    attempt_log.status = defaults.SUCCESS
+            )
+            result = proc.communicate()
+            logger.debug(result)
+            logger.info(proc.returncode)
+
+            if proc.returncode != 0:
+                msg = ",".join(result[1].split("\n"))
+                attempt_log.status = defaults.FAIL
+                attempt_log.end_time = str(datetime.now())
+                attempt_log.message = msg
+                console.print(msg, style=defaults.error_style)
+                return attempt_log
+
+            # for stderr
+            for line in result[1].split("\n"):
+                if line.strip() == "":
+                    continue
+                console.print(line, style=defaults.warning_style)
+
+            output_parameters: Dict[str, Parameter] = {}
+            metrics: Dict[str, Parameter] = {}
+
+            # only from stdout
+            for line in result[0].split("\n"):
+                if line.strip() == "":
+                    continue
+
+                logger.info(line)
+                console.print(line)
+
+                if line.strip() == collect_delimiter:
+                    # The lines from now on should be captured
+                    capture = True
+                    continue
+
+                if capture:
+                    key, value = line.strip().split("=", 1)
+                    if key in return_keys:
+                        task_return = return_keys[key]
+
+                        try:
+                            value = json.loads(value)
+                        except json.JSONDecodeError:
+                            value = value
+
+                        output_parameter = task_return_to_parameter(
+                            task_return=task_return,
+                            value=value,
+                        )
+
+                        if task_return.kind == "metric":
+                            metrics[task_return.name] = output_parameter
+
+                        param_name = task_return.name
+                        if map_variable:
+                            for _, v in map_variable.items():
+                                param_name = f"{param_name}_{v}"
+
+                        output_parameters[param_name] = output_parameter
+
+                attempt_log.output_parameters = output_parameters
+                attempt_log.user_defined_metrics = metrics
+                params.update(output_parameters)
+
+            attempt_log.status = defaults.SUCCESS
 
         attempt_log.end_time = str(datetime.now())
         return attempt_log

{runnable-0.11.0 → runnable-0.11.2}/runnable/extensions/run_log_store/chunked_file_system/implementation.py

@@ -35,10 +35,10 @@ class ChunkedFileSystemRunLogStore(ChunkedRunLogStore):
             name (str): The suffix of the file name to check in the run log store.
         """
         log_folder = self.log_folder_with_run_id(run_id=run_id)
-
         sub_name = Template(name).safe_substitute({"creation_time": ""})
 
         matches = list(log_folder.glob(f"{sub_name}*"))
+
         if matches:
             if not multiple_allowed:
                 if len(matches) > 1: