runnable 0.11.1__py3-none-any.whl → 0.11.2__py3-none-any.whl

runnable/__init__.py

@@ -38,3 +38,4 @@ from runnable.sdk import (  # noqa
 
 
 # TODO: Think of way of generating dag hash without executor configuration
+# TODO: Add examples of map parameters and task types

runnable/datastore.py

@@ -312,8 +312,10 @@ class RunLog(BaseModel):
         summary["Catalog Location"] = _context.catalog_handler.get_summary()
         summary["Full Run log present at: "] = _context.run_log_store.get_summary()
 
-        summary["Final Parameters"] = {p: v.description for p, v in self.parameters.items()}
-        summary["Collected metrics"] = {p: v.description for p, v in self.parameters.items() if v.kind == "metric"}
+        run_log = _context.run_log_store.get_run_log_by_id(run_id=_context.run_id, full=True)
+
+        summary["Final Parameters"] = {p: v.description for p, v in run_log.parameters.items()}
+        summary["Collected metrics"] = {p: v.description for p, v in run_log.parameters.items() if v.kind == "metric"}
 
         return summary
 

runnable/extensions/catalog/file_system/implementation.py

@@ -226,7 +226,7 @@ class FileSystemCatalog(BaseCatalog):
         for cataloged_file in cataloged_files:
             if str(cataloged_file).endswith("execution.log"):
                 continue
-            print(cataloged_file.name)
+
             if cataloged_file.is_file():
                 shutil.copy(cataloged_file, run_catalog / cataloged_file.name)
             else:

runnable/extensions/nodes.py

@@ -505,7 +505,7 @@ class MapNode(CompositeNode):
             for _, v in map_variable.items():
                 for branch_return in self.branch_returns:
                     param_name, param_type = branch_return
-                    raw_parameters[f"{param_name}_{v}"] = param_type.copy()
+                    raw_parameters[f"{v}_{param_name}"] = param_type.copy()
         else:
             for branch_return in self.branch_returns:
                 param_name, param_type = branch_return
@@ -606,9 +606,9 @@ class MapNode(CompositeNode):
                     param_name, _ = branch_return
                     to_reduce = []
                     for iter_variable in iterate_on:
-                        to_reduce.append(params[f"{param_name}_{iter_variable}"].get_value())
+                        to_reduce.append(params[f"{iter_variable}_{param_name}"].get_value())
 
-                    param_name = f"{param_name}_{v}"
+                    param_name = f"{v}_{param_name}"
                     params[param_name].value = reducer_f(to_reduce)
                     params[param_name].reduced = True
         else:
@@ -617,7 +617,7 @@ class MapNode(CompositeNode):
 
                 to_reduce = []
                 for iter_variable in iterate_on:
-                    to_reduce.append(params[f"{param_name}_{iter_variable}"].get_value())
+                    to_reduce.append(params[f"{iter_variable}_{param_name}"].get_value())
 
                 params[param_name].value = reducer_f(*to_reduce)
                 params[param_name].reduced = True

runnable/extensions/run_log_store/chunked_file_system/implementation.py

@@ -35,10 +35,10 @@ class ChunkedFileSystemRunLogStore(ChunkedRunLogStore):
             name (str): The suffix of the file name to check in the run log store.
         """
         log_folder = self.log_folder_with_run_id(run_id=run_id)
-
         sub_name = Template(name).safe_substitute({"creation_time": ""})
 
         matches = list(log_folder.glob(f"{sub_name}*"))
+
         if matches:
             if not multiple_allowed:
                 if len(matches) > 1:

runnable/extensions/run_log_store/generic_chunked.py

@@ -7,7 +7,16 @@ from string import Template
 from typing import Any, Dict, Optional, Sequence, Union
 
 from runnable import defaults, exceptions
-from runnable.datastore import BaseRunLogStore, BranchLog, RunLog, StepLog
+from runnable.datastore import (
+    BaseRunLogStore,
+    BranchLog,
+    JsonParameter,
+    MetricParameter,
+    ObjectParameter,
+    Parameter,
+    RunLog,
+    StepLog,
+)
 
 logger = logging.getLogger(defaults.LOGGER_NAME)
 
@@ -164,7 +173,9 @@ class ChunkedRunLogStore(BaseRunLogStore):
             raise Exception(f"Name is required during retrieval for {log_type}")
 
         naming_pattern = self.naming_pattern(log_type=log_type, name=name)
+
         matches = self.get_matches(run_id=run_id, name=naming_pattern, multiple_allowed=multiple_allowed)
+
         if matches:
             if not multiple_allowed:
                 contents = self._retrieve(name=matches)  # type: ignore
@@ -370,10 +381,17 @@ class ChunkedRunLogStore(BaseRunLogStore):
         Raises:
             RunLogNotFoundError: If the run log for run_id is not found in the datastore
         """
-        parameters = {}
+        parameters: Dict[str, Parameter] = {}
         try:
             parameters_list = self.retrieve(run_id=run_id, log_type=self.LogTypes.PARAMETER, multiple_allowed=True)
-            parameters = {key: value for param in parameters_list for key, value in param.items()}
+            for param in parameters_list:
+                for key, value in param.items():
+                    if value["kind"] == "json":
+                        parameters[key] = JsonParameter(**value)
+                    if value["kind"] == "metric":
+                        parameters[key] = MetricParameter(**value)
+                    if value["kind"] == "object":
+                        parameters[key] = ObjectParameter(**value)
         except EntityNotFoundError:
             # No parameters are set
             pass
@@ -401,7 +419,7 @@ class ChunkedRunLogStore(BaseRunLogStore):
             self.store(
                 run_id=run_id,
                 log_type=self.LogTypes.PARAMETER,
-                contents={key: value},
+                contents={key: value.model_dump(by_alias=True)},
                 name=key,
             )
 

runnable/tasks.py

@@ -1,4 +1,5 @@
 import contextlib
+import copy
 import importlib
 import io
 import json
@@ -99,6 +100,20 @@ class BaseTaskType(BaseModel):
         """
         raise NotImplementedError()
 
+    def _diff_parameters(
+        self, parameters_in: Dict[str, Parameter], context_params: Dict[str, Parameter]
+    ) -> Dict[str, Parameter]:
+        diff: Dict[str, Parameter] = {}
+        for param_name, param in context_params.items():
+            if param_name in parameters_in:
+                if parameters_in[param_name] != param:
+                    diff[param_name] = param
+                continue
+
+            diff[param_name] = param
+
+        return diff
+
     @contextlib.contextmanager
     def expose_secrets(self):
         """Context manager to expose secrets to the execution.
@@ -128,7 +143,7 @@ class BaseTaskType(BaseModel):
             if param.reduced is False:
                 context_param = param_name
                 for _, v in map_variable.items():  # type: ignore
-                    context_param = f"{context_param}_{v}"
+                    context_param = f"{v}_{context_param}"
 
                 if context_param in params:
                     params[param_name].value = params[context_param].value
@@ -147,6 +162,8 @@ class BaseTaskType(BaseModel):
 
         log_file = open(log_file_name, "w")
 
+        parameters_in = copy.deepcopy(params)
+
         f = io.StringIO()
         try:
             with contextlib.redirect_stdout(f):
@@ -168,7 +185,8 @@ class BaseTaskType(BaseModel):
 
             # Update parameters
             # This should only update the parameters that are changed at the root level.
-            self._context.run_log_store.set_parameters(parameters=params, run_id=self._context.run_id)
+            diff_parameters = self._diff_parameters(parameters_in=parameters_in, context_params=params)
+            self._context.run_log_store.set_parameters(parameters=diff_parameters, run_id=self._context.run_id)
 
 
 def task_return_to_parameter(task_return: TaskReturns, value: Any) -> Parameter:
@@ -259,7 +277,7 @@ class PythonTaskType(BaseTaskType):  # pylint: disable=too-few-public-methods
                         param_name = task_return.name
                         if map_variable:
                             for _, v in map_variable.items():
-                                param_name = f"{param_name}_{v}"
+                                param_name = f"{v}_{param_name}"
 
                         output_parameters[param_name] = output_parameter
 

{runnable-0.11.1.dist-info → runnable-0.11.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: runnable
-Version: 0.11.1
+Version: 0.11.2
 Summary: A Compute agnostic pipelining software
 Home-page: https://github.com/vijayvammi/runnable
 License: Apache-2.0

{runnable-0.11.1.dist-info → runnable-0.11.2.dist-info}/RECORD

@@ -1,8 +1,8 @@
-runnable/__init__.py,sha256=C5ySSfN_vHWFTscnxLx3tfMjKT3Bs9jfUjXnDvun33Y,870
+runnable/__init__.py,sha256=V3Ihmzbb56k0qCNiBxB8ELDhgffhr_qctIy8qL0o4QM,924
 runnable/catalog.py,sha256=22OECi5TrpHErxYIhfx-lJ2vgBUi4-5V9CaYEVm98hE,4138
 runnable/cli.py,sha256=RILUrEfzernuKD3dNdXPBkqN_1OgE5GosYRuInj0FVs,9618
 runnable/context.py,sha256=QhiXJHRcEBfSKB1ijvL5yB9w44x0HCe7VEiwK1cUJ9U,1124
-runnable/datastore.py,sha256=EgKi4_b5g6KbInpjMyw8Xwr-EgcSGi1Lx2u5vp4amSQ,27672
+runnable/datastore.py,sha256=ViyAyjZQuJkRE1Q8CEEkVJXRKCmozQPe4_ZPl1X3wxo,27773
 runnable/defaults.py,sha256=MOX7I2S6yO4FphZaZREFQca94a20oO8uvzXLd6GLKQs,4703
 runnable/entrypoints.py,sha256=a8M7vb954as_ni7lM0t65czXQj2AHjB-KrQJ3zt3sWo,16397
 runnable/exceptions.py,sha256=6NIYoTAzdKyGQ9PvW1Hu7b80OS746395KiGDhM7ThH8,2526
@@ -10,7 +10,7 @@ runnable/executor.py,sha256=xfBighQ5t_vejohip000XfxLwsgechUE1ZMIJWrZbUA,14484
 runnable/extensions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 runnable/extensions/catalog/__init__.py,sha256=uXZ6D-Myr_J4HnBA4F5Hd7LZ0IAjQiFQYxRhMzejhQc,761
 runnable/extensions/catalog/file_system/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-runnable/extensions/catalog/file_system/implementation.py,sha256=9j920o9SULdcVp1Mr8FgeuV-Sv5bR3w5tcohChxHnak,9130
+runnable/extensions/catalog/file_system/implementation.py,sha256=mFPsAwPMNGWbHczpQ84o3mfkPkOEz5zjsT7a3rqNzoE,9092
 runnable/extensions/catalog/k8s_pvc/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 runnable/extensions/catalog/k8s_pvc/implementation.py,sha256=oJDDI0APT7lrtjWmzYJRDHLGn3Vhbn2MdFSRYvFBUpY,436
 runnable/extensions/catalog/k8s_pvc/integration.py,sha256=OfrHbNFN8sR-wsVa4os3ajmWJFSd5H4KOHGVAmjRZTQ,1850
@@ -29,10 +29,10 @@ runnable/extensions/executor/mocked/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQ
 runnable/extensions/executor/mocked/implementation.py,sha256=ChdUyUsiXXjG_v80d0uLp76Nz4jqqGEry36gs9gNn9k,5082
 runnable/extensions/executor/retry/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 runnable/extensions/executor/retry/implementation.py,sha256=ZBSYpxSiAIt-SXPD-qIPP-MMo8b7sQ6UKOTJemAjXlI,6625
-runnable/extensions/nodes.py,sha256=5soHRhfT8FY2vnQa4kvRqeVphTq_t-GSw-ExNZfgB30,31965
+runnable/extensions/nodes.py,sha256=Z2LuAxeZpx1pKZmI7G2u90jAm0sdi4U2pqCIFmm0JB4,31965
 runnable/extensions/run_log_store/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 runnable/extensions/run_log_store/chunked_file_system/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-runnable/extensions/run_log_store/chunked_file_system/implementation.py,sha256=wtOeREr9QyIuMHLCT7o_eDCJVCDsBvwmk89kos3dhfQ,3326
+runnable/extensions/run_log_store/chunked_file_system/implementation.py,sha256=EW2P8lr3eH-pIOsMTJPr5eb-iWc48GQ97W15JzkpC_4,3326
 runnable/extensions/run_log_store/chunked_k8s_pvc/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 runnable/extensions/run_log_store/chunked_k8s_pvc/implementation.py,sha256=iGzy-s1eT_kAJP7XgzDLmEMOGrBLvACIiGE_wM62jGE,579
 runnable/extensions/run_log_store/chunked_k8s_pvc/integration.py,sha256=atzdTy5HJ-bZsd6AzDP8kYRI1TshKxviBKeqY359TUs,1979
@@ -40,7 +40,7 @@ runnable/extensions/run_log_store/db/implementation_FF.py,sha256=oEiG5ASWYYbwlBb
 runnable/extensions/run_log_store/db/integration_FF.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 runnable/extensions/run_log_store/file_system/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 runnable/extensions/run_log_store/file_system/implementation.py,sha256=WxxfGCaDAB5zHMM3zv9aeDwXZ4DhtyzjXOjfjvyDoZ4,4288
-runnable/extensions/run_log_store/generic_chunked.py,sha256=rcY5f-MIYUUiM5iQnDHICOh7cKiOUSCeaxcBG9_fz-U,19390
+runnable/extensions/run_log_store/generic_chunked.py,sha256=PtYK1dheKYdxODwu_ygpGRIHIepgLVaIORSqvsrg0No,19876
 runnable/extensions/run_log_store/k8s_pvc/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 runnable/extensions/run_log_store/k8s_pvc/implementation.py,sha256=tLgXy9HUB_vlFVQ0Itk6PpNU3GlCOILN4vA3fm80jXI,542
 runnable/extensions/run_log_store/k8s_pvc/integration.py,sha256=lxQg327mwC0ykhNp5Kg34a9g8o1DzJAhfkiqMGmsABs,1873
@@ -57,10 +57,10 @@ runnable/parameters.py,sha256=KGGW8_uoIK2hd3EwzzBmoHBOrai3fh-SESNPpJRTfj4,5161
 runnable/pickler.py,sha256=5SDNf0miMUJ3ZauhQdzwk8_t-9jeOqaTjP5bvRnu9sU,2685
 runnable/sdk.py,sha256=JsM27GUc3c57ZepK996FHtfzXP6FGs8MP-s96RC-_fo,27648
 runnable/secrets.py,sha256=dakb7WRloWVo-KpQp6Vy4rwFdGi58BTlT4OifQY106I,2324
-runnable/tasks.py,sha256=CH0W7evAZK5hco0medJS6DwHRr06bng7oB7xk0Xh6zQ,20618
+runnable/tasks.py,sha256=CKmZoQAHAQgQLGEX3S0l6qvDL5hcqHoUyTXH_gHe61M,21261
 runnable/utils.py,sha256=okZFGbJWqStl5Rq5vLhNUQZDv_vhcT58bq9MDrTVxhc,19449
-runnable-0.11.1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-runnable-0.11.1.dist-info/METADATA,sha256=IKuld8eA9gcb6y7YZaO-3Dpmw2x4U2wnPg0eag1fewU,17020
-runnable-0.11.1.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-runnable-0.11.1.dist-info/entry_points.txt,sha256=Wy-dimdD2REO2a36Ri84fqGqA5iwGy2RIbdgRNtCNdM,1540
-runnable-0.11.1.dist-info/RECORD,,
+runnable-0.11.2.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+runnable-0.11.2.dist-info/METADATA,sha256=5FKWYUkN4EidqFwOckXPOY0DZFqJykmaJdZAf1w__Yo,17020
+runnable-0.11.2.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+runnable-0.11.2.dist-info/entry_points.txt,sha256=Wy-dimdD2REO2a36Ri84fqGqA5iwGy2RIbdgRNtCNdM,1540
+runnable-0.11.2.dist-info/RECORD,,