runnable 0.10.0__py3-none-any.whl → 0.11.0__py3-none-any.whl

runnable/sdk.py

@@ -16,9 +16,11 @@ from pydantic import (
     model_validator,
 )
 from rich import print
+from rich.progress import BarColumn, Progress, TextColumn, TimeElapsedColumn
+from rich.table import Column
 from typing_extensions import Self
 
-from runnable import defaults, entrypoints, graph, utils
+from runnable import console, defaults, entrypoints, graph, utils
 from runnable.extensions.nodes import (
     FailNode,
     MapNode,
@@ -39,6 +41,10 @@ def pickled(name: str) -> TaskReturns:
     return TaskReturns(name=name, kind="object")
 
 
+def metric(name: str) -> TaskReturns:
+    return TaskReturns(name=name, kind="metric")
+
+
 class Catalog(BaseModel):
     """
     Use to instruct a task to sync data from/to the central catalog.
@@ -360,7 +366,7 @@ class Stub(BaseTraversal):
 
     """
 
-    model_config = ConfigDict(extra="allow")
+    model_config = ConfigDict(extra="ignore")
     catalog: Optional[Catalog] = Field(default=None, alias="catalog")
 
     def create_node(self) -> StubNode:
@@ -686,8 +692,27 @@ class Pipeline(BaseModel):
         # Prepare for graph execution
         run_context.executor.prepare_for_graph_execution()
 
-        logger.info("Executing the graph")
-        run_context.executor.execute_graph(dag=run_context.dag)
+        with Progress(
+            TextColumn("[progress.description]{task.description}", table_column=Column(ratio=2)),
+            BarColumn(table_column=Column(ratio=1), style="dark_orange"),
+            TimeElapsedColumn(table_column=Column(ratio=1)),
+            console=console,
+            expand=True,
+        ) as progress:
+            try:
+                run_context.progress = progress
+                pipeline_execution_task = progress.add_task("[dark_orange] Starting execution .. ", total=1)
+                run_context.executor.execute_graph(dag=run_context.dag)
+
+                run_log = run_context.run_log_store.get_run_log_by_id(run_id=run_context.run_id, full=False)
+
+                if run_log.status == defaults.SUCCESS:
+                    progress.update(pipeline_execution_task, description="[green] Success", completed=True)
+                else:
+                    progress.update(pipeline_execution_task, description="[red] Failed", completed=True)
+            except Exception as e:  # noqa: E722
+                console.print(e, style=defaults.error_style)
+                progress.update(pipeline_execution_task, description="[red] Errored execution", completed=True)
 
         if run_context.executor._local:
             return run_context.run_log_store.get_run_log_by_id(run_id=run_context.run_id)

runnable/tasks.py

@@ -15,8 +15,14 @@ from pydantic import BaseModel, ConfigDict, Field, ValidationInfo, field_validat
 from stevedore import driver
 
 import runnable.context as context
-from runnable import defaults, parameters, utils
-from runnable.datastore import JsonParameter, ObjectParameter, Parameter, StepAttempt
+from runnable import console, defaults, exceptions, parameters, utils
+from runnable.datastore import (
+    JsonParameter,
+    MetricParameter,
+    ObjectParameter,
+    Parameter,
+    StepAttempt,
+)
 from runnable.defaults import TypeMapVariable
 
 logger = logging.getLogger(defaults.LOGGER_NAME)
@@ -28,7 +34,7 @@ logging.getLogger("stevedore").setLevel(logging.CRITICAL)
 
 class TaskReturns(BaseModel):
     name: str
-    kind: Literal["json", "object"] = Field(default="json")
+    kind: Literal["json", "object", "metric"] = Field(default="json")
 
 
 class BaseTaskType(BaseModel):
@@ -41,6 +47,9 @@ class BaseTaskType(BaseModel):
 
     model_config = ConfigDict(extra="forbid")
 
+    def get_summary(self) -> Dict[str, Any]:
+        return self.model_dump(by_alias=True, exclude_none=True)
+
     @property
     def _context(self):
         return context.run_context
@@ -98,12 +107,15 @@ class BaseTaskType(BaseModel):
         self.set_secrets_as_env_variables()
         try:
             yield
+        except Exception as e:  # pylint: disable=broad-except
+            logger.exception(e)
         finally:
             self.delete_secrets_from_env_variables()
 
     @contextlib.contextmanager
     def execution_context(self, map_variable: TypeMapVariable = None, allow_complex: bool = True):
         params = self._context.run_log_store.get_parameters(run_id=self._context.run_id).copy()
+        logger.info(f"Parameters available for the execution: {params}")
 
         for param_name, param in params.items():
             # Any access to unreduced param should be replaced.
@@ -118,6 +130,8 @@ class BaseTaskType(BaseModel):
                 if context_param in params:
                     params[param_name].value = params[context_param].value
 
+        logger.debug(f"Resolved parameters: {params}")
+
         if not allow_complex:
             params = {key: value for key, value in params.items() if isinstance(value, JsonParameter)}
 
@@ -132,6 +146,8 @@ class BaseTaskType(BaseModel):
         try:
             with contextlib.redirect_stdout(f):
                 yield params
+        except Exception as e:  # pylint: disable=broad-except
+            logger.exception(e)
         finally:
             print(f.getvalue())  # print to console
             log_file.write(f.getvalue())  # Print to file
@@ -140,15 +156,12 @@ class BaseTaskType(BaseModel):
             log_file.close()
 
             # Put the log file in the catalog
-            catalog_handler = context.run_context.catalog_handler
-            catalog_handler.put(name=log_file.name, run_id=context.run_context.run_id)
+            # self._context.catalog_handler.put(name=log_file.name, run_id=context.run_context.run_id)
             os.remove(log_file.name)
 
             # Update parameters
             self._context.run_log_store.set_parameters(parameters=params, run_id=self._context.run_id)
 
-            return True  # To suppress exceptions
-
 
 def task_return_to_parameter(task_return: TaskReturns, value: Any) -> Parameter:
     # implicit support for pydantic models
@@ -161,6 +174,9 @@ def task_return_to_parameter(task_return: TaskReturns, value: Any) -> Parameter:
     if task_return.kind == "json":
         return JsonParameter(kind="json", value=value)
 
+    if task_return.kind == "metric":
+        return MetricParameter(kind="metric", value=value)
+
     if task_return.kind == "object":
         obj = ObjectParameter(value=task_return.name, kind="object")
         obj.put_object(data=value)
@@ -197,13 +213,23 @@ class PythonTaskType(BaseTaskType):  # pylint: disable=too-few-public-methods
             imported_module = importlib.import_module(module)
             f = getattr(imported_module, func)
 
-            filtered_parameters = parameters.filter_arguments_for_func(f, params.copy(), map_variable)
-            logger.info(f"Calling {func} from {module} with {filtered_parameters}")
-
             try:
-                user_set_parameters = f(**filtered_parameters)  # This is a tuple or single value
+                try:
+                    filtered_parameters = parameters.filter_arguments_for_func(f, params.copy(), map_variable)
+                    logger.info(f"Calling {func} from {module} with {filtered_parameters}")
+                    user_set_parameters = f(**filtered_parameters)  # This is a tuple or single value
+                except Exception as e:
+                    logger.exception(e)
+                    console.print(e, style=defaults.error_style)
+                    raise exceptions.CommandCallError(f"Function call: {self.command} did not succeed.\n") from e
+
                 attempt_log.input_parameters = params.copy()
 
+                if map_variable:
+                    attempt_log.input_parameters.update(
+                        {k: JsonParameter(value=v, kind="json") for k, v in map_variable.items()}
+                    )
+
                 if self.returns:
                     if not isinstance(user_set_parameters, tuple):  # make it a tuple
                         user_set_parameters = (user_set_parameters,)
@@ -212,6 +238,7 @@ class PythonTaskType(BaseTaskType):  # pylint: disable=too-few-public-methods
                         raise ValueError("Returns task signature does not match the function returns")
 
                     output_parameters: Dict[str, Parameter] = {}
+                    metrics: Dict[str, Parameter] = {}
 
                     for i, task_return in enumerate(self.returns):
                         output_parameter = task_return_to_parameter(
@@ -219,6 +246,9 @@ class PythonTaskType(BaseTaskType):  # pylint: disable=too-few-public-methods
                             value=user_set_parameters[i],
                         )
 
+                        if task_return.kind == "metric":
+                            metrics[task_return.name] = output_parameter
+
                         param_name = task_return.name
                         if map_variable:
                             for _, v in map_variable.items():
@@ -227,14 +257,15 @@ class PythonTaskType(BaseTaskType):  # pylint: disable=too-few-public-methods
                         output_parameters[param_name] = output_parameter
 
                     attempt_log.output_parameters = output_parameters
+                    attempt_log.user_defined_metrics = metrics
                     params.update(output_parameters)
 
                 attempt_log.status = defaults.SUCCESS
             except Exception as _e:
-                msg = f"Call to the function {self.command} with {filtered_parameters} did not succeed.\n"
-                logger.exception(msg)
+                msg = f"Call to the function {self.command} did not succeed.\n"
                 logger.exception(_e)
-                attempt_log.status = defaults.FAIL
+                attempt_log.message = msg
+                console.print(_e, style=defaults.error_style)
 
         attempt_log.end_time = str(datetime.now())
 
@@ -296,7 +327,6 @@ class NotebookTaskType(BaseTaskType):
                 if map_variable:
                     for key, value in map_variable.items():
                         notebook_output_path += "_" + str(value)
-
                         params[key] = value
 
                 notebook_params = {k: v.get_value() for k, v in params.items()}

runnable/utils.py

@@ -538,7 +538,7 @@ def get_provider_by_name_and_type(service_type: str, service_details: defaults.S
     if "config" in service_details:
         service_config = service_details.get("config", {})
 
-    logger.info(f"Trying to get a service of {service_type} of the name {service_name} with config: {service_config}")
+    logger.debug(f"Trying to get a service of {service_type} of the name {service_name} with config: {service_config}")
     try:
         mgr = driver.DriverManager(
             namespace=namespace,
@@ -548,6 +548,7 @@ def get_provider_by_name_and_type(service_type: str, service_details: defaults.S
         )
         return mgr.driver
     except Exception as _e:
+        logger.exception(f"Could not find the service of type: {service_type} with config: {service_details}")
         raise Exception(f"Could not find the service of type: {service_type} with config: {service_details}") from _e
 
 

{runnable-0.10.0.dist-info → runnable-0.11.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: runnable
-Version: 0.10.0
+Version: 0.11.0
 Summary: A Compute agnostic pipelining software
 Home-page: https://github.com/vijayvammi/runnable
 License: Apache-2.0

{runnable-0.10.0.dist-info → runnable-0.11.0.dist-info}/RECORD

@@ -1,49 +1,45 @@
-runnable/__init__.py,sha256=MEJVptRwgHJyQa07gpc0sGcdxoxwX-crhoyAZUEYjBs,619
-runnable/catalog.py,sha256=OUaQ73DWfTsMmq2sKlBn0aDz031mupladNGVuF3pWm0,3985
-runnable/cli.py,sha256=AZiZf2eRV7zMA7APg6dyTHWqK1--bQwdLiYP8olaKis,9589
-runnable/context.py,sha256=GOp-dRPMgYsbgjGy39yVsaWgu5l4RgUxN9-4nggoPmg,1048
-runnable/datastore.py,sha256=xtO6atOh1Zf_olvdzh3W_XisAviL26uZPR491N7tn-E,25354
-runnable/defaults.py,sha256=lN9HUjTaGGYBC8OPCGTNZXJ4xlOOzqUt-zDjY7GLnLI,4770
-runnable/entrypoints.py,sha256=tDQNSX3yx7zU5tqa1hxn3j4yHXfJ9Svg9CpPmEdBQ2o,15359
-runnable/exceptions.py,sha256=R__RzUWs7Ow7m7yqawi2w09XXI4OqmA67yeXkECM0xw,2419
+runnable/__init__.py,sha256=BzuufxKGqgYvd-v4fwhH7lgYGCzOnwY7ca5pZZasgx8,720
+runnable/catalog.py,sha256=22OECi5TrpHErxYIhfx-lJ2vgBUi4-5V9CaYEVm98hE,4138
+runnable/cli.py,sha256=gDKk-eg1Vlf0NAWEFqdFEUfcDJdBawRPgSMpNqpbsOc,9590
+runnable/context.py,sha256=QhiXJHRcEBfSKB1ijvL5yB9w44x0HCe7VEiwK1cUJ9U,1124
+runnable/datastore.py,sha256=EgKi4_b5g6KbInpjMyw8Xwr-EgcSGi1Lx2u5vp4amSQ,27672
+runnable/defaults.py,sha256=MOX7I2S6yO4FphZaZREFQca94a20oO8uvzXLd6GLKQs,4703
+runnable/entrypoints.py,sha256=-GbwFCUPkLfXpvslD1abXtlfwZevSocqc6QK8TPL40Q,16197
+runnable/exceptions.py,sha256=6NIYoTAzdKyGQ9PvW1Hu7b80OS746395KiGDhM7ThH8,2526
 runnable/executor.py,sha256=xfBighQ5t_vejohip000XfxLwsgechUE1ZMIJWrZbUA,14484
-runnable/experiment_tracker.py,sha256=bX2Vr73f3bsdnWqxjMSSiKA-WwqkUHfUzJQqZoQBpvY,3668
 runnable/extensions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 runnable/extensions/catalog/__init__.py,sha256=uXZ6D-Myr_J4HnBA4F5Hd7LZ0IAjQiFQYxRhMzejhQc,761
 runnable/extensions/catalog/file_system/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-runnable/extensions/catalog/file_system/implementation.py,sha256=UNrJFV_tyMpknFKoCMXbBt-CWL4UpDxcwnMh43zVfpc,8958
+runnable/extensions/catalog/file_system/implementation.py,sha256=9j920o9SULdcVp1Mr8FgeuV-Sv5bR3w5tcohChxHnak,9130
 runnable/extensions/catalog/k8s_pvc/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 runnable/extensions/catalog/k8s_pvc/implementation.py,sha256=oJDDI0APT7lrtjWmzYJRDHLGn3Vhbn2MdFSRYvFBUpY,436
 runnable/extensions/catalog/k8s_pvc/integration.py,sha256=OfrHbNFN8sR-wsVa4os3ajmWJFSd5H4KOHGVAmjRZTQ,1850
-runnable/extensions/executor/__init__.py,sha256=tIWUBWu5MNqHrfbOD83AgP59OkM2aJh3JUUQItIP6i4,24187
+runnable/extensions/executor/__init__.py,sha256=FTXtI_etk_eyKRIDpCiTrYfV5yCkfJMl-cuXVDzEqY8,26649
 runnable/extensions/executor/argo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-runnable/extensions/executor/argo/implementation.py,sha256=wlDSD5RfZmrdQ65abXTZSdh4KUGv-IzQtbHVtDXNUgQ,43795
+runnable/extensions/executor/argo/implementation.py,sha256=_BfxCe742S6uV-7PuQ53KjzwY-8Rq-5y9txOXMYf20U,43670
 runnable/extensions/executor/argo/specification.yaml,sha256=wXQcm2gOQYqy-IOQIhucohS32ZrHKCfGA5zZ0RraPYc,1276
 runnable/extensions/executor/k8s_job/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 runnable/extensions/executor/k8s_job/implementation_FF.py,sha256=1IfVG1GRcJcVFzQ-WhkJsmzdJuj51QMxXylY9UrWM0U,10259
 runnable/extensions/executor/k8s_job/integration_FF.py,sha256=pG6HKhPMgCRIgu1PAnBvsfJQE1FxcjuSiC2I-Hn5sWo,2165
 runnable/extensions/executor/local/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-runnable/extensions/executor/local/implementation.py,sha256=r9dSf2lSBGHihbGNhq_GPe3fHKLjf4KI3l-B4w2b5Ls,2468
+runnable/extensions/executor/local/implementation.py,sha256=e8Tzv-FgQmJeUXVut96jeNERTR83JVG_zkQZMEjCVAs,2469
 runnable/extensions/executor/local_container/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 runnable/extensions/executor/local_container/implementation.py,sha256=6kYMgdgE5JxZkVAidxsBSpqkHvyKMfEctgZWSZQEpXA,13979
 runnable/extensions/executor/mocked/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 runnable/extensions/executor/mocked/implementation.py,sha256=ChdUyUsiXXjG_v80d0uLp76Nz4jqqGEry36gs9gNn9k,5082
 runnable/extensions/executor/retry/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 runnable/extensions/executor/retry/implementation.py,sha256=ZBSYpxSiAIt-SXPD-qIPP-MMo8b7sQ6UKOTJemAjXlI,6625
-runnable/extensions/experiment_tracker/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-runnable/extensions/experiment_tracker/mlflow/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-runnable/extensions/experiment_tracker/mlflow/implementation.py,sha256=sc1Wm1LCf7wBX0BYVx3YVdwsR72AE0qIrzl7cEfIl58,3045
-runnable/extensions/nodes.py,sha256=Em-vs21ZfhyvFh-s6NZVhcUydSrX_mY8mb8-NUAO_w8,29883
+runnable/extensions/nodes.py,sha256=5soHRhfT8FY2vnQa4kvRqeVphTq_t-GSw-ExNZfgB30,31965
 runnable/extensions/run_log_store/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 runnable/extensions/run_log_store/chunked_file_system/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-runnable/extensions/run_log_store/chunked_file_system/implementation.py,sha256=koHNG-Cv7mpt-rTNC3tiLBR8HcjnQ9L-EvQ4dtKkGRA,3170
+runnable/extensions/run_log_store/chunked_file_system/implementation.py,sha256=wtOeREr9QyIuMHLCT7o_eDCJVCDsBvwmk89kos3dhfQ,3326
 runnable/extensions/run_log_store/chunked_k8s_pvc/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 runnable/extensions/run_log_store/chunked_k8s_pvc/implementation.py,sha256=iGzy-s1eT_kAJP7XgzDLmEMOGrBLvACIiGE_wM62jGE,579
 runnable/extensions/run_log_store/chunked_k8s_pvc/integration.py,sha256=atzdTy5HJ-bZsd6AzDP8kYRI1TshKxviBKeqY359TUs,1979
 runnable/extensions/run_log_store/db/implementation_FF.py,sha256=oEiG5ASWYYbwlBbnryKarQENB-L_yOsnZahbj2U0GdQ,5155
 runnable/extensions/run_log_store/db/integration_FF.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 runnable/extensions/run_log_store/file_system/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-runnable/extensions/run_log_store/file_system/implementation.py,sha256=PcaM8IKcj-b2iNE9Zup2eC6Y2-987uQzzb0skdd1QX4,4114
+runnable/extensions/run_log_store/file_system/implementation.py,sha256=WxxfGCaDAB5zHMM3zv9aeDwXZ4DhtyzjXOjfjvyDoZ4,4288
 runnable/extensions/run_log_store/generic_chunked.py,sha256=rcY5f-MIYUUiM5iQnDHICOh7cKiOUSCeaxcBG9_fz-U,19390
 runnable/extensions/run_log_store/k8s_pvc/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 runnable/extensions/run_log_store/k8s_pvc/implementation.py,sha256=tLgXy9HUB_vlFVQ0Itk6PpNU3GlCOILN4vA3fm80jXI,542
@@ -53,18 +49,18 @@ runnable/extensions/secrets/dotenv/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQe
 runnable/extensions/secrets/dotenv/implementation.py,sha256=3J5pofWahdZbnwnETwpspE5-PKyvmZF_vkfwA1X_bkA,3365
 runnable/extensions/secrets/env_secrets/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 runnable/extensions/secrets/env_secrets/implementation.py,sha256=5XiHdJvIr0-jkl4fGfEf26UsgE5Q2Z4oCc0RwjlJdJA,1236
-runnable/graph.py,sha256=w7qEnTuh6lvfeVSAbqNlwMufPqWL9espJI5s_G2prHM,15871
-runnable/integration.py,sha256=_jEm5PXl8pbvCmUGKjtssfZ2YtYnMzuRnjr2NySMqzs,7176
+runnable/graph.py,sha256=18IpGYw5kgHP32m12WwXscx-kG5Kx-AuWS1LFbMfBLg,16202
+runnable/integration.py,sha256=eb9qJVZR7Ehg0N1UnGPuyjJvoA-xQ1-xP7AlZHUXHqM,6705
 runnable/names.py,sha256=vn92Kv9ANROYSZX6Z4z1v_WA3WiEdIYmG6KEStBFZug,8134
-runnable/nodes.py,sha256=-vHtjyeRPfmKCmWjuYQItwdZIG3Rpv-IJlGQNbqj99c,16387
-runnable/parameters.py,sha256=-DtO-3LnsOt-wa9vqk3Mpq8OB-WVZf5xgEhtaointy0,5091
+runnable/nodes.py,sha256=UqR-bJx0Hi7uLSUw_saB7VsNdFh3POKtdgsEPsasHfE,16576
+runnable/parameters.py,sha256=KGGW8_uoIK2hd3EwzzBmoHBOrai3fh-SESNPpJRTfj4,5161
 runnable/pickler.py,sha256=5SDNf0miMUJ3ZauhQdzwk8_t-9jeOqaTjP5bvRnu9sU,2685
-runnable/sdk.py,sha256=_WXAfRgSOzZzqhh3d_t_sCUBLf7_0tHXz48a00ie0H4,25697
+runnable/sdk.py,sha256=hx29PEDYjJIWaTZp3ZhyLDwuulG1HIQ2q7A4HVE1WkM,26998
 runnable/secrets.py,sha256=dakb7WRloWVo-KpQp6Vy4rwFdGi58BTlT4OifQY106I,2324
-runnable/tasks.py,sha256=LSc6Un-vB0bGTayuxcR0VY6D7gilg8WQKIDEgP6Ge90,17663
-runnable/utils.py,sha256=mnxLzGjJKht5z46gz5EumNJaaY33dec26F8wRQQRsGY,19337
-runnable-0.10.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-runnable-0.10.0.dist-info/METADATA,sha256=N1kPIVDGBJbWTYdSfuDR6AxQ4x4xoS-eM63RXo_xhKI,17063
-runnable-0.10.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-runnable-0.10.0.dist-info/entry_points.txt,sha256=_elJX0RSR4u9IWIl8fwYNL-YBXoYaanYd415TJBmTRE,1710
-runnable-0.10.0.dist-info/RECORD,,
+runnable/tasks.py,sha256=T8vVLo-yWHanrXKHFJvNdjQXKuWmJ13_-lxZzmf1mQM,18908
+runnable/utils.py,sha256=okZFGbJWqStl5Rq5vLhNUQZDv_vhcT58bq9MDrTVxhc,19449
+runnable-0.11.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+runnable-0.11.0.dist-info/METADATA,sha256=M7YR0Y_j7zMnXCKHFWGk4mlWKYZTdkOyZa3M0v3NokI,17063
+runnable-0.11.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+runnable-0.11.0.dist-info/entry_points.txt,sha256=Wy-dimdD2REO2a36Ri84fqGqA5iwGy2RIbdgRNtCNdM,1540
+runnable-0.11.0.dist-info/RECORD,,

{runnable-0.10.0.dist-info → runnable-0.11.0.dist-info}/entry_points.txt

@@ -12,10 +12,6 @@ local-container=runnable.extensions.executor.local_container.implementation:Loca
 mocked=runnable.extensions.executor.mocked.implementation:MockedExecutor
 retry=runnable.extensions.executor.retry.implementation:RetryExecutor
 
-[experiment_tracker]
-do-nothing=runnable.experiment_tracker:DoNothingTracker
-mlflow=runnable.extensions.experiment_tracker.mlflow.implementation:MLFlowExperimentTracker
-
 [nodes]
 dag=runnable.extensions.nodes:DagNode
 fail=runnable.extensions.nodes:FailNode

runnable/experiment_tracker.py

@@ -1,139 +0,0 @@
-import contextlib
-import json
-import logging
-import os
-from abc import ABC, abstractmethod
-from collections import defaultdict
-from typing import Any, ContextManager, Dict, Tuple, Union
-
-from pydantic import BaseModel, ConfigDict
-
-import runnable.context as context
-from runnable import defaults
-from runnable.utils import remove_prefix
-
-logger = logging.getLogger(defaults.LOGGER_NAME)
-
-
-def retrieve_step_details(key: str) -> Tuple[str, int]:
-    key = remove_prefix(key, defaults.TRACK_PREFIX)
-    data = key.split(defaults.STEP_INDICATOR)
-
-    key = data[0].lower()
-    step = 0
-
-    if len(data) > 1:
-        step = int(data[1])
-
-    return key, step
-
-
-def get_tracked_data() -> Dict[str, Any]:
-    tracked_data: Dict[str, Any] = defaultdict(dict)
-    for env_var, value in os.environ.items():
-        if env_var.startswith(defaults.TRACK_PREFIX):
-            key, step = retrieve_step_details(env_var)
-
-            # print(value, type(value))
-            try:
-                value = json.loads(value)
-            except json.decoder.JSONDecodeError:
-                logger.warning(f"Tracker {key} could not be JSON decoded, adding the literal value")
-
-            tracked_data[key][step] = value
-            del os.environ[env_var]
-
-    for key, value in tracked_data.items():
-        if len(value) == 1:
-            tracked_data[key] = value[0]
-
-    return tracked_data
-
-
-# --8<-- [start:docs]
-
-
-class BaseExperimentTracker(ABC, BaseModel):
-    """
-    Base Experiment tracker class definition.
-    """
-
-    service_name: str = ""
-    service_type: str = "experiment_tracker"
-
-    @property
-    def _context(self):
-        return context.run_context
-
-    model_config = ConfigDict(extra="forbid")
-
-    @property
-    def client_context(self) -> ContextManager:
-        """
-        Returns the client context.
-        """
-        return contextlib.nullcontext()
-
-    def publish_data(self, tracked_data: Dict[str, Any]):
-        for key, value in tracked_data.items():
-            if isinstance(value, dict):
-                for key2, value2 in value.items():
-                    self.log_metric(key, value2, step=key2)
-                continue
-            self.log_metric(key, value)
-
-    @abstractmethod
-    def log_metric(self, key: str, value: Union[int, float], step: int = 0):
-        """
-        Sets the metric in the experiment tracking.
-
-        Args:
-            key (str): The key against you want to store the value
-            value (float): The value of the metric
-            step (int): Optional step at which it was recorded
-
-        Raises:
-            NotImplementedError: Base class, hence not implemented
-        """
-        raise NotImplementedError
-
-    @abstractmethod
-    def log_parameter(self, key: str, value: Any):
-        """
-        Logs a parameter in the experiment tracking.
-
-        Args:
-            key (str): The key against you want to store the value
-            value (any): The value of the metric
-
-        Raises:
-            NotImplementedError: Base class, hence not implemented
-        """
-        pass
-
-
-# --8<-- [end:docs]
-
-
-class DoNothingTracker(BaseExperimentTracker):
-    """
-    A Do nothing tracker
-    """
-
-    service_name: str = "do-nothing"
-
-    def log_metric(self, key: str, value: Union[int, float], step: int = 0):
-        """
-        Sets the metric in the experiment tracking.
-
-        Args:
-            key (str): The key against you want to store the value
-            value (float): The value of the metric
-        """
-        ...
-
-    def log_parameter(self, key: str, value: Any):
-        """
-        Since this is a Do nothing tracker, we don't need to log anything.
-        """
-        ...

runnable/extensions/experiment_tracker/mlflow/implementation.py

@@ -1,94 +0,0 @@
-import functools
-import logging
-from typing import Any, Union
-
-from pydantic import ConfigDict, PrivateAttr
-
-from runnable import defaults
-from runnable.experiment_tracker import BaseExperimentTracker
-
-logger = logging.getLogger(defaults.NAME)
-
-
-class MLFlowExperimentTracker(BaseExperimentTracker):
-    """
-    A MLFlow experiment tracker.
-
-    TODO: Need to set up credentials from secrets
-    """
-
-    service_name: str = "mlflow"
-
-    server_url: str
-    autolog: bool = False
-
-    _default_experiment_name: str = PrivateAttr(default="Default")
-    _active_run_id: str = PrivateAttr(default="")
-    _client: Any = PrivateAttr(default=None)
-
-    model_config = ConfigDict(extra="forbid")
-
-    def model_post_init(self, __context: Any) -> None:
-        try:
-            import mlflow
-        except ImportError:
-            raise Exception("You need to install mlflow to use MLFlowExperimentTracker.")
-
-        self._client = mlflow
-
-        self._client.set_tracking_uri(self.server_url)
-
-        if self.autolog:
-            self._client.autolog(log_models=False)
-
-    @functools.cached_property
-    def experiment_id(self):
-        experiment_name = self._default_experiment_name
-
-        # If a tag is provided, we should create that as our experiment
-        if self._context.tag:
-            experiment_name = self._context.tag
-
-        experiment = self._client.get_experiment_by_name(experiment_name)
-        if not experiment:
-            # Create the experiment and get it.
-            experiment = self._client.create_experiment(experiment_name)
-            experiment = self._client.get_experiment(experiment)
-
-        return experiment.experiment_id
-
-    @functools.cached_property
-    def run_name(self):
-        return self._context.run_id
-
-    @property
-    def client_context(self):
-        if self._active_run_id:
-            return self._client.start_run(
-                run_id=self._active_run_id, experiment_id=self.experiment_id, run_name=self.run_name
-            )
-
-        active_run = self._client.start_run(run_name=self.run_name, experiment_id=self.experiment_id)
-        self._active_run_id = active_run.info.run_id
-        return active_run
-
-    def log_metric(self, key: str, value: Union[int, float], step: int = 0):
-        """
-        Sets the metric in the experiment tracking.
-
-        Args:
-            key (str): The key against you want to store the value
-            value (Any): The value of the metric
-        """
-        if not isinstance(value, float) or isinstance(value, int):
-            msg = f"Only float/int values are accepted as metrics. Setting the metric {key} as parameter {key}_{step}"
-            logger.warning(msg)
-            self.log_parameter(key=key, value=value, step=step)
-            return
-
-        with self.client_context as _:
-            self._client.log_metric(key, float(value), step=step or None)
-
-    def log_parameter(self, key: str, value: Any, step: int = 0):
-        with self.client_context as _:
-            self._client.log_param(key + f"_{str(step)}", value)