runnable 0.9.1__py3-none-any.whl → 0.11.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- runnable/__init__.py +13 -9
- runnable/catalog.py +8 -1
- runnable/cli.py +1 -0
- runnable/context.py +5 -3
- runnable/datastore.py +96 -12
- runnable/defaults.py +9 -9
- runnable/entrypoints.py +38 -24
- runnable/exceptions.py +4 -0
- runnable/extensions/catalog/file_system/implementation.py +8 -1
- runnable/extensions/executor/__init__.py +85 -29
- runnable/extensions/executor/argo/implementation.py +8 -4
- runnable/extensions/executor/local/implementation.py +1 -0
- runnable/extensions/nodes.py +90 -13
- runnable/extensions/run_log_store/chunked_file_system/implementation.py +6 -1
- runnable/extensions/run_log_store/file_system/implementation.py +6 -0
- runnable/graph.py +11 -0
- runnable/integration.py +4 -17
- runnable/nodes.py +9 -0
- runnable/parameters.py +3 -1
- runnable/sdk.py +123 -18
- runnable/tasks.py +45 -15
- runnable/utils.py +2 -1
- {runnable-0.9.1.dist-info → runnable-0.11.0.dist-info}/METADATA +1 -1
- {runnable-0.9.1.dist-info → runnable-0.11.0.dist-info}/RECORD +27 -31
- {runnable-0.9.1.dist-info → runnable-0.11.0.dist-info}/entry_points.txt +0 -4
- runnable/experiment_tracker.py +0 -139
- runnable/extensions/experiment_tracker/__init__.py +0 -0
- runnable/extensions/experiment_tracker/mlflow/__init__.py +0 -0
- runnable/extensions/experiment_tracker/mlflow/implementation.py +0 -94
- {runnable-0.9.1.dist-info → runnable-0.11.0.dist-info}/LICENSE +0 -0
- {runnable-0.9.1.dist-info → runnable-0.11.0.dist-info}/WHEEL +0 -0
runnable/sdk.py
CHANGED
@@ -6,25 +6,45 @@ from abc import ABC, abstractmethod
|
|
6
6
|
from pathlib import Path
|
7
7
|
from typing import Any, Callable, Dict, List, Optional, Union
|
8
8
|
|
9
|
-
from pydantic import
|
9
|
+
from pydantic import (
|
10
|
+
BaseModel,
|
11
|
+
ConfigDict,
|
12
|
+
Field,
|
13
|
+
PrivateAttr,
|
14
|
+
computed_field,
|
15
|
+
field_validator,
|
16
|
+
model_validator,
|
17
|
+
)
|
10
18
|
from rich import print
|
19
|
+
from rich.progress import BarColumn, Progress, TextColumn, TimeElapsedColumn
|
20
|
+
from rich.table import Column
|
11
21
|
from typing_extensions import Self
|
12
22
|
|
13
|
-
from runnable import defaults, entrypoints, graph, utils
|
14
|
-
from runnable.extensions.nodes import
|
23
|
+
from runnable import console, defaults, entrypoints, graph, utils
|
24
|
+
from runnable.extensions.nodes import (
|
25
|
+
FailNode,
|
26
|
+
MapNode,
|
27
|
+
ParallelNode,
|
28
|
+
StubNode,
|
29
|
+
SuccessNode,
|
30
|
+
TaskNode,
|
31
|
+
)
|
15
32
|
from runnable.nodes import TraversalNode
|
16
33
|
from runnable.tasks import TaskReturns
|
17
34
|
|
18
35
|
logger = logging.getLogger(defaults.LOGGER_NAME)
|
19
36
|
|
20
|
-
StepType = Union["Stub", "PythonTask", "NotebookTask", "ShellTask", "
|
21
|
-
TraversalTypes = Union["Stub", "PythonTask", "NotebookTask", "ShellTask", "Parallel", "Map"]
|
37
|
+
StepType = Union["Stub", "PythonTask", "NotebookTask", "ShellTask", "Parallel", "Map"]
|
22
38
|
|
23
39
|
|
24
40
|
def pickled(name: str) -> TaskReturns:
|
25
41
|
return TaskReturns(name=name, kind="object")
|
26
42
|
|
27
43
|
|
44
|
+
def metric(name: str) -> TaskReturns:
|
45
|
+
return TaskReturns(name=name, kind="metric")
|
46
|
+
|
47
|
+
|
28
48
|
class Catalog(BaseModel):
|
29
49
|
"""
|
30
50
|
Use to instruct a task to sync data from/to the central catalog.
|
@@ -346,7 +366,7 @@ class Stub(BaseTraversal):
|
|
346
366
|
|
347
367
|
"""
|
348
368
|
|
349
|
-
model_config = ConfigDict(extra="
|
369
|
+
model_config = ConfigDict(extra="ignore")
|
350
370
|
catalog: Optional[Catalog] = Field(default=None, alias="catalog")
|
351
371
|
|
352
372
|
def create_node(self) -> StubNode:
|
@@ -496,8 +516,7 @@ class Pipeline(BaseModel):
|
|
496
516
|
|
497
517
|
"""
|
498
518
|
|
499
|
-
steps: List[StepType]
|
500
|
-
start_at: TraversalTypes
|
519
|
+
steps: List[Union[StepType, List[StepType]]]
|
501
520
|
name: str = ""
|
502
521
|
description: str = ""
|
503
522
|
add_terminal_nodes: bool = True # Adds "success" and "fail" nodes
|
@@ -507,20 +526,87 @@ class Pipeline(BaseModel):
|
|
507
526
|
_dag: graph.Graph = PrivateAttr()
|
508
527
|
model_config = ConfigDict(extra="forbid")
|
509
528
|
|
529
|
+
def _validate_path(self, path: List[StepType]) -> None:
|
530
|
+
# Check if one and only one step terminates with success
|
531
|
+
# Check no more than one step terminates with failure
|
532
|
+
|
533
|
+
reached_success = False
|
534
|
+
reached_failure = False
|
535
|
+
|
536
|
+
for step in path:
|
537
|
+
if step.terminate_with_success:
|
538
|
+
if reached_success:
|
539
|
+
raise Exception("A pipeline cannot have more than one step that terminates with success")
|
540
|
+
reached_success = True
|
541
|
+
continue
|
542
|
+
if step.terminate_with_failure:
|
543
|
+
if reached_failure:
|
544
|
+
raise Exception("A pipeline cannot have more than one step that terminates with failure")
|
545
|
+
reached_failure = True
|
546
|
+
|
547
|
+
if not reached_success:
|
548
|
+
raise Exception("A pipeline must have at least one step that terminates with success")
|
549
|
+
|
550
|
+
def _construct_path(self, path: List[StepType]) -> None:
|
551
|
+
prev_step = path[0]
|
552
|
+
|
553
|
+
for step in path:
|
554
|
+
if step == prev_step:
|
555
|
+
continue
|
556
|
+
|
557
|
+
if prev_step.terminate_with_success or prev_step.terminate_with_failure:
|
558
|
+
raise Exception(f"A step that terminates with success/failure cannot have a next step: {prev_step}")
|
559
|
+
|
560
|
+
if prev_step.next_node and prev_step.next_node not in ["success", "fail"]:
|
561
|
+
raise Exception(f"Step already has a next node: {prev_step} ")
|
562
|
+
|
563
|
+
prev_step.next_node = step.name
|
564
|
+
prev_step = step
|
565
|
+
|
510
566
|
def model_post_init(self, __context: Any) -> None:
|
511
|
-
|
567
|
+
"""
|
568
|
+
The sequence of steps can either be:
|
569
|
+
[step1, step2,..., stepN, [step11, step12,..., step1N], [step21, step22,...,]]
|
570
|
+
indicates:
|
571
|
+
- step1 > step2 > ... > stepN
|
572
|
+
- We expect terminate with success or fail to be explicitly stated on a step.
|
573
|
+
- If it is stated, the step cannot have a next step defined apart from "success" and "fail".
|
574
|
+
|
575
|
+
The inner list of steps is only to accommodate on-failure behaviors.
|
576
|
+
- For sake of simplicity, lets assume that it has the same behavior as the happy pipeline.
|
577
|
+
- A task which was already seen should not be part of this.
|
578
|
+
- There should be at least one step which terminates with success
|
579
|
+
|
580
|
+
Any definition of pipeline should have one node that terminates with success.
|
581
|
+
"""
|
582
|
+
|
583
|
+
success_path: List[StepType] = []
|
584
|
+
on_failure_paths: List[List[StepType]] = []
|
585
|
+
|
586
|
+
for step in self.steps:
|
587
|
+
if isinstance(step, (Stub, PythonTask, NotebookTask, ShellTask, Parallel, Map)):
|
588
|
+
success_path.append(step)
|
589
|
+
continue
|
590
|
+
on_failure_paths.append(step)
|
591
|
+
|
592
|
+
if not success_path:
|
593
|
+
raise Exception("There should be some success path")
|
594
|
+
|
595
|
+
# Check all paths are valid and construct the path
|
596
|
+
paths = [success_path] + on_failure_paths
|
597
|
+
for path in paths:
|
598
|
+
self._validate_path(path)
|
599
|
+
self._construct_path(path)
|
600
|
+
|
601
|
+
all_steps: List[StepType] = [step for step in success_path + on_failure_paths] # type: ignore
|
512
602
|
|
513
603
|
self._dag = graph.Graph(
|
514
|
-
start_at=
|
604
|
+
start_at=all_steps[0].name,
|
515
605
|
description=self.description,
|
516
606
|
internal_branch_name=self.internal_branch_name,
|
517
607
|
)
|
518
608
|
|
519
|
-
for step in
|
520
|
-
if step.name == self.start_at.name:
|
521
|
-
if isinstance(step, Success) or isinstance(step, Fail):
|
522
|
-
raise Exception("A success or fail node cannot be the start_at of the graph")
|
523
|
-
assert step.next_node
|
609
|
+
for step in all_steps:
|
524
610
|
self._dag.add_node(step.create_node())
|
525
611
|
|
526
612
|
if self.add_terminal_nodes:
|
@@ -569,7 +655,7 @@ class Pipeline(BaseModel):
|
|
569
655
|
py_to_yaml = os.environ.get("RUNNABLE_PY_TO_YAML", "false")
|
570
656
|
|
571
657
|
if py_to_yaml == "true":
|
572
|
-
return
|
658
|
+
return {}
|
573
659
|
|
574
660
|
logger.setLevel(log_level)
|
575
661
|
|
@@ -606,8 +692,27 @@ class Pipeline(BaseModel):
|
|
606
692
|
# Prepare for graph execution
|
607
693
|
run_context.executor.prepare_for_graph_execution()
|
608
694
|
|
609
|
-
|
610
|
-
|
695
|
+
with Progress(
|
696
|
+
TextColumn("[progress.description]{task.description}", table_column=Column(ratio=2)),
|
697
|
+
BarColumn(table_column=Column(ratio=1), style="dark_orange"),
|
698
|
+
TimeElapsedColumn(table_column=Column(ratio=1)),
|
699
|
+
console=console,
|
700
|
+
expand=True,
|
701
|
+
) as progress:
|
702
|
+
try:
|
703
|
+
run_context.progress = progress
|
704
|
+
pipeline_execution_task = progress.add_task("[dark_orange] Starting execution .. ", total=1)
|
705
|
+
run_context.executor.execute_graph(dag=run_context.dag)
|
706
|
+
|
707
|
+
run_log = run_context.run_log_store.get_run_log_by_id(run_id=run_context.run_id, full=False)
|
708
|
+
|
709
|
+
if run_log.status == defaults.SUCCESS:
|
710
|
+
progress.update(pipeline_execution_task, description="[green] Success", completed=True)
|
711
|
+
else:
|
712
|
+
progress.update(pipeline_execution_task, description="[red] Failed", completed=True)
|
713
|
+
except Exception as e: # noqa: E722
|
714
|
+
console.print(e, style=defaults.error_style)
|
715
|
+
progress.update(pipeline_execution_task, description="[red] Errored execution", completed=True)
|
611
716
|
|
612
717
|
if run_context.executor._local:
|
613
718
|
return run_context.run_log_store.get_run_log_by_id(run_id=run_context.run_id)
|
runnable/tasks.py
CHANGED
@@ -15,8 +15,14 @@ from pydantic import BaseModel, ConfigDict, Field, ValidationInfo, field_validat
|
|
15
15
|
from stevedore import driver
|
16
16
|
|
17
17
|
import runnable.context as context
|
18
|
-
from runnable import defaults, parameters, utils
|
19
|
-
from runnable.datastore import
|
18
|
+
from runnable import console, defaults, exceptions, parameters, utils
|
19
|
+
from runnable.datastore import (
|
20
|
+
JsonParameter,
|
21
|
+
MetricParameter,
|
22
|
+
ObjectParameter,
|
23
|
+
Parameter,
|
24
|
+
StepAttempt,
|
25
|
+
)
|
20
26
|
from runnable.defaults import TypeMapVariable
|
21
27
|
|
22
28
|
logger = logging.getLogger(defaults.LOGGER_NAME)
|
@@ -28,7 +34,7 @@ logging.getLogger("stevedore").setLevel(logging.CRITICAL)
|
|
28
34
|
|
29
35
|
class TaskReturns(BaseModel):
|
30
36
|
name: str
|
31
|
-
kind: Literal["json", "object"] = Field(default="json")
|
37
|
+
kind: Literal["json", "object", "metric"] = Field(default="json")
|
32
38
|
|
33
39
|
|
34
40
|
class BaseTaskType(BaseModel):
|
@@ -41,6 +47,9 @@ class BaseTaskType(BaseModel):
|
|
41
47
|
|
42
48
|
model_config = ConfigDict(extra="forbid")
|
43
49
|
|
50
|
+
def get_summary(self) -> Dict[str, Any]:
|
51
|
+
return self.model_dump(by_alias=True, exclude_none=True)
|
52
|
+
|
44
53
|
@property
|
45
54
|
def _context(self):
|
46
55
|
return context.run_context
|
@@ -98,12 +107,15 @@ class BaseTaskType(BaseModel):
|
|
98
107
|
self.set_secrets_as_env_variables()
|
99
108
|
try:
|
100
109
|
yield
|
110
|
+
except Exception as e: # pylint: disable=broad-except
|
111
|
+
logger.exception(e)
|
101
112
|
finally:
|
102
113
|
self.delete_secrets_from_env_variables()
|
103
114
|
|
104
115
|
@contextlib.contextmanager
|
105
116
|
def execution_context(self, map_variable: TypeMapVariable = None, allow_complex: bool = True):
|
106
117
|
params = self._context.run_log_store.get_parameters(run_id=self._context.run_id).copy()
|
118
|
+
logger.info(f"Parameters available for the execution: {params}")
|
107
119
|
|
108
120
|
for param_name, param in params.items():
|
109
121
|
# Any access to unreduced param should be replaced.
|
@@ -118,6 +130,8 @@ class BaseTaskType(BaseModel):
|
|
118
130
|
if context_param in params:
|
119
131
|
params[param_name].value = params[context_param].value
|
120
132
|
|
133
|
+
logger.debug(f"Resolved parameters: {params}")
|
134
|
+
|
121
135
|
if not allow_complex:
|
122
136
|
params = {key: value for key, value in params.items() if isinstance(value, JsonParameter)}
|
123
137
|
|
@@ -132,6 +146,8 @@ class BaseTaskType(BaseModel):
|
|
132
146
|
try:
|
133
147
|
with contextlib.redirect_stdout(f):
|
134
148
|
yield params
|
149
|
+
except Exception as e: # pylint: disable=broad-except
|
150
|
+
logger.exception(e)
|
135
151
|
finally:
|
136
152
|
print(f.getvalue()) # print to console
|
137
153
|
log_file.write(f.getvalue()) # Print to file
|
@@ -140,15 +156,12 @@ class BaseTaskType(BaseModel):
|
|
140
156
|
log_file.close()
|
141
157
|
|
142
158
|
# Put the log file in the catalog
|
143
|
-
catalog_handler =
|
144
|
-
catalog_handler.put(name=log_file.name, run_id=context.run_context.run_id)
|
159
|
+
# self._context.catalog_handler.put(name=log_file.name, run_id=context.run_context.run_id)
|
145
160
|
os.remove(log_file.name)
|
146
161
|
|
147
162
|
# Update parameters
|
148
163
|
self._context.run_log_store.set_parameters(parameters=params, run_id=self._context.run_id)
|
149
164
|
|
150
|
-
return True # To suppress exceptions
|
151
|
-
|
152
165
|
|
153
166
|
def task_return_to_parameter(task_return: TaskReturns, value: Any) -> Parameter:
|
154
167
|
# implicit support for pydantic models
|
@@ -161,6 +174,9 @@ def task_return_to_parameter(task_return: TaskReturns, value: Any) -> Parameter:
|
|
161
174
|
if task_return.kind == "json":
|
162
175
|
return JsonParameter(kind="json", value=value)
|
163
176
|
|
177
|
+
if task_return.kind == "metric":
|
178
|
+
return MetricParameter(kind="metric", value=value)
|
179
|
+
|
164
180
|
if task_return.kind == "object":
|
165
181
|
obj = ObjectParameter(value=task_return.name, kind="object")
|
166
182
|
obj.put_object(data=value)
|
@@ -197,13 +213,23 @@ class PythonTaskType(BaseTaskType): # pylint: disable=too-few-public-methods
|
|
197
213
|
imported_module = importlib.import_module(module)
|
198
214
|
f = getattr(imported_module, func)
|
199
215
|
|
200
|
-
filtered_parameters = parameters.filter_arguments_for_func(f, params.copy(), map_variable)
|
201
|
-
logger.info(f"Calling {func} from {module} with {filtered_parameters}")
|
202
|
-
|
203
216
|
try:
|
204
|
-
|
217
|
+
try:
|
218
|
+
filtered_parameters = parameters.filter_arguments_for_func(f, params.copy(), map_variable)
|
219
|
+
logger.info(f"Calling {func} from {module} with {filtered_parameters}")
|
220
|
+
user_set_parameters = f(**filtered_parameters) # This is a tuple or single value
|
221
|
+
except Exception as e:
|
222
|
+
logger.exception(e)
|
223
|
+
console.print(e, style=defaults.error_style)
|
224
|
+
raise exceptions.CommandCallError(f"Function call: {self.command} did not succeed.\n") from e
|
225
|
+
|
205
226
|
attempt_log.input_parameters = params.copy()
|
206
227
|
|
228
|
+
if map_variable:
|
229
|
+
attempt_log.input_parameters.update(
|
230
|
+
{k: JsonParameter(value=v, kind="json") for k, v in map_variable.items()}
|
231
|
+
)
|
232
|
+
|
207
233
|
if self.returns:
|
208
234
|
if not isinstance(user_set_parameters, tuple): # make it a tuple
|
209
235
|
user_set_parameters = (user_set_parameters,)
|
@@ -212,6 +238,7 @@ class PythonTaskType(BaseTaskType): # pylint: disable=too-few-public-methods
|
|
212
238
|
raise ValueError("Returns task signature does not match the function returns")
|
213
239
|
|
214
240
|
output_parameters: Dict[str, Parameter] = {}
|
241
|
+
metrics: Dict[str, Parameter] = {}
|
215
242
|
|
216
243
|
for i, task_return in enumerate(self.returns):
|
217
244
|
output_parameter = task_return_to_parameter(
|
@@ -219,6 +246,9 @@ class PythonTaskType(BaseTaskType): # pylint: disable=too-few-public-methods
|
|
219
246
|
value=user_set_parameters[i],
|
220
247
|
)
|
221
248
|
|
249
|
+
if task_return.kind == "metric":
|
250
|
+
metrics[task_return.name] = output_parameter
|
251
|
+
|
222
252
|
param_name = task_return.name
|
223
253
|
if map_variable:
|
224
254
|
for _, v in map_variable.items():
|
@@ -227,14 +257,15 @@ class PythonTaskType(BaseTaskType): # pylint: disable=too-few-public-methods
|
|
227
257
|
output_parameters[param_name] = output_parameter
|
228
258
|
|
229
259
|
attempt_log.output_parameters = output_parameters
|
260
|
+
attempt_log.user_defined_metrics = metrics
|
230
261
|
params.update(output_parameters)
|
231
262
|
|
232
263
|
attempt_log.status = defaults.SUCCESS
|
233
264
|
except Exception as _e:
|
234
|
-
msg = f"Call to the function {self.command}
|
235
|
-
logger.exception(msg)
|
265
|
+
msg = f"Call to the function {self.command} did not succeed.\n"
|
236
266
|
logger.exception(_e)
|
237
|
-
attempt_log.
|
267
|
+
attempt_log.message = msg
|
268
|
+
console.print(_e, style=defaults.error_style)
|
238
269
|
|
239
270
|
attempt_log.end_time = str(datetime.now())
|
240
271
|
|
@@ -296,7 +327,6 @@ class NotebookTaskType(BaseTaskType):
|
|
296
327
|
if map_variable:
|
297
328
|
for key, value in map_variable.items():
|
298
329
|
notebook_output_path += "_" + str(value)
|
299
|
-
|
300
330
|
params[key] = value
|
301
331
|
|
302
332
|
notebook_params = {k: v.get_value() for k, v in params.items()}
|
runnable/utils.py
CHANGED
@@ -538,7 +538,7 @@ def get_provider_by_name_and_type(service_type: str, service_details: defaults.S
|
|
538
538
|
if "config" in service_details:
|
539
539
|
service_config = service_details.get("config", {})
|
540
540
|
|
541
|
-
logger.
|
541
|
+
logger.debug(f"Trying to get a service of {service_type} of the name {service_name} with config: {service_config}")
|
542
542
|
try:
|
543
543
|
mgr = driver.DriverManager(
|
544
544
|
namespace=namespace,
|
@@ -548,6 +548,7 @@ def get_provider_by_name_and_type(service_type: str, service_details: defaults.S
|
|
548
548
|
)
|
549
549
|
return mgr.driver
|
550
550
|
except Exception as _e:
|
551
|
+
logger.exception(f"Could not find the service of type: {service_type} with config: {service_details}")
|
551
552
|
raise Exception(f"Could not find the service of type: {service_type} with config: {service_details}") from _e
|
552
553
|
|
553
554
|
|
@@ -1,49 +1,45 @@
|
|
1
|
-
runnable/__init__.py,sha256=
|
2
|
-
runnable/catalog.py,sha256=
|
3
|
-
runnable/cli.py,sha256=
|
4
|
-
runnable/context.py,sha256=
|
5
|
-
runnable/datastore.py,sha256=
|
6
|
-
runnable/defaults.py,sha256=
|
7
|
-
runnable/entrypoints.py,sha256
|
8
|
-
runnable/exceptions.py,sha256=
|
1
|
+
runnable/__init__.py,sha256=BzuufxKGqgYvd-v4fwhH7lgYGCzOnwY7ca5pZZasgx8,720
|
2
|
+
runnable/catalog.py,sha256=22OECi5TrpHErxYIhfx-lJ2vgBUi4-5V9CaYEVm98hE,4138
|
3
|
+
runnable/cli.py,sha256=gDKk-eg1Vlf0NAWEFqdFEUfcDJdBawRPgSMpNqpbsOc,9590
|
4
|
+
runnable/context.py,sha256=QhiXJHRcEBfSKB1ijvL5yB9w44x0HCe7VEiwK1cUJ9U,1124
|
5
|
+
runnable/datastore.py,sha256=EgKi4_b5g6KbInpjMyw8Xwr-EgcSGi1Lx2u5vp4amSQ,27672
|
6
|
+
runnable/defaults.py,sha256=MOX7I2S6yO4FphZaZREFQca94a20oO8uvzXLd6GLKQs,4703
|
7
|
+
runnable/entrypoints.py,sha256=-GbwFCUPkLfXpvslD1abXtlfwZevSocqc6QK8TPL40Q,16197
|
8
|
+
runnable/exceptions.py,sha256=6NIYoTAzdKyGQ9PvW1Hu7b80OS746395KiGDhM7ThH8,2526
|
9
9
|
runnable/executor.py,sha256=xfBighQ5t_vejohip000XfxLwsgechUE1ZMIJWrZbUA,14484
|
10
|
-
runnable/experiment_tracker.py,sha256=bX2Vr73f3bsdnWqxjMSSiKA-WwqkUHfUzJQqZoQBpvY,3668
|
11
10
|
runnable/extensions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
12
11
|
runnable/extensions/catalog/__init__.py,sha256=uXZ6D-Myr_J4HnBA4F5Hd7LZ0IAjQiFQYxRhMzejhQc,761
|
13
12
|
runnable/extensions/catalog/file_system/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
14
|
-
runnable/extensions/catalog/file_system/implementation.py,sha256=
|
13
|
+
runnable/extensions/catalog/file_system/implementation.py,sha256=9j920o9SULdcVp1Mr8FgeuV-Sv5bR3w5tcohChxHnak,9130
|
15
14
|
runnable/extensions/catalog/k8s_pvc/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
16
15
|
runnable/extensions/catalog/k8s_pvc/implementation.py,sha256=oJDDI0APT7lrtjWmzYJRDHLGn3Vhbn2MdFSRYvFBUpY,436
|
17
16
|
runnable/extensions/catalog/k8s_pvc/integration.py,sha256=OfrHbNFN8sR-wsVa4os3ajmWJFSd5H4KOHGVAmjRZTQ,1850
|
18
|
-
runnable/extensions/executor/__init__.py,sha256=
|
17
|
+
runnable/extensions/executor/__init__.py,sha256=FTXtI_etk_eyKRIDpCiTrYfV5yCkfJMl-cuXVDzEqY8,26649
|
19
18
|
runnable/extensions/executor/argo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
20
|
-
runnable/extensions/executor/argo/implementation.py,sha256=
|
19
|
+
runnable/extensions/executor/argo/implementation.py,sha256=_BfxCe742S6uV-7PuQ53KjzwY-8Rq-5y9txOXMYf20U,43670
|
21
20
|
runnable/extensions/executor/argo/specification.yaml,sha256=wXQcm2gOQYqy-IOQIhucohS32ZrHKCfGA5zZ0RraPYc,1276
|
22
21
|
runnable/extensions/executor/k8s_job/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
23
22
|
runnable/extensions/executor/k8s_job/implementation_FF.py,sha256=1IfVG1GRcJcVFzQ-WhkJsmzdJuj51QMxXylY9UrWM0U,10259
|
24
23
|
runnable/extensions/executor/k8s_job/integration_FF.py,sha256=pG6HKhPMgCRIgu1PAnBvsfJQE1FxcjuSiC2I-Hn5sWo,2165
|
25
24
|
runnable/extensions/executor/local/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
26
|
-
runnable/extensions/executor/local/implementation.py,sha256=
|
25
|
+
runnable/extensions/executor/local/implementation.py,sha256=e8Tzv-FgQmJeUXVut96jeNERTR83JVG_zkQZMEjCVAs,2469
|
27
26
|
runnable/extensions/executor/local_container/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
28
27
|
runnable/extensions/executor/local_container/implementation.py,sha256=6kYMgdgE5JxZkVAidxsBSpqkHvyKMfEctgZWSZQEpXA,13979
|
29
28
|
runnable/extensions/executor/mocked/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
30
29
|
runnable/extensions/executor/mocked/implementation.py,sha256=ChdUyUsiXXjG_v80d0uLp76Nz4jqqGEry36gs9gNn9k,5082
|
31
30
|
runnable/extensions/executor/retry/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
32
31
|
runnable/extensions/executor/retry/implementation.py,sha256=ZBSYpxSiAIt-SXPD-qIPP-MMo8b7sQ6UKOTJemAjXlI,6625
|
33
|
-
runnable/extensions/
|
34
|
-
runnable/extensions/experiment_tracker/mlflow/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
35
|
-
runnable/extensions/experiment_tracker/mlflow/implementation.py,sha256=sc1Wm1LCf7wBX0BYVx3YVdwsR72AE0qIrzl7cEfIl58,3045
|
36
|
-
runnable/extensions/nodes.py,sha256=Em-vs21ZfhyvFh-s6NZVhcUydSrX_mY8mb8-NUAO_w8,29883
|
32
|
+
runnable/extensions/nodes.py,sha256=5soHRhfT8FY2vnQa4kvRqeVphTq_t-GSw-ExNZfgB30,31965
|
37
33
|
runnable/extensions/run_log_store/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
38
34
|
runnable/extensions/run_log_store/chunked_file_system/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
39
|
-
runnable/extensions/run_log_store/chunked_file_system/implementation.py,sha256=
|
35
|
+
runnable/extensions/run_log_store/chunked_file_system/implementation.py,sha256=wtOeREr9QyIuMHLCT7o_eDCJVCDsBvwmk89kos3dhfQ,3326
|
40
36
|
runnable/extensions/run_log_store/chunked_k8s_pvc/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
41
37
|
runnable/extensions/run_log_store/chunked_k8s_pvc/implementation.py,sha256=iGzy-s1eT_kAJP7XgzDLmEMOGrBLvACIiGE_wM62jGE,579
|
42
38
|
runnable/extensions/run_log_store/chunked_k8s_pvc/integration.py,sha256=atzdTy5HJ-bZsd6AzDP8kYRI1TshKxviBKeqY359TUs,1979
|
43
39
|
runnable/extensions/run_log_store/db/implementation_FF.py,sha256=oEiG5ASWYYbwlBbnryKarQENB-L_yOsnZahbj2U0GdQ,5155
|
44
40
|
runnable/extensions/run_log_store/db/integration_FF.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
45
41
|
runnable/extensions/run_log_store/file_system/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
46
|
-
runnable/extensions/run_log_store/file_system/implementation.py,sha256=
|
42
|
+
runnable/extensions/run_log_store/file_system/implementation.py,sha256=WxxfGCaDAB5zHMM3zv9aeDwXZ4DhtyzjXOjfjvyDoZ4,4288
|
47
43
|
runnable/extensions/run_log_store/generic_chunked.py,sha256=rcY5f-MIYUUiM5iQnDHICOh7cKiOUSCeaxcBG9_fz-U,19390
|
48
44
|
runnable/extensions/run_log_store/k8s_pvc/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
49
45
|
runnable/extensions/run_log_store/k8s_pvc/implementation.py,sha256=tLgXy9HUB_vlFVQ0Itk6PpNU3GlCOILN4vA3fm80jXI,542
|
@@ -53,18 +49,18 @@ runnable/extensions/secrets/dotenv/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQe
|
|
53
49
|
runnable/extensions/secrets/dotenv/implementation.py,sha256=3J5pofWahdZbnwnETwpspE5-PKyvmZF_vkfwA1X_bkA,3365
|
54
50
|
runnable/extensions/secrets/env_secrets/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
55
51
|
runnable/extensions/secrets/env_secrets/implementation.py,sha256=5XiHdJvIr0-jkl4fGfEf26UsgE5Q2Z4oCc0RwjlJdJA,1236
|
56
|
-
runnable/graph.py,sha256=
|
57
|
-
runnable/integration.py,sha256=
|
52
|
+
runnable/graph.py,sha256=18IpGYw5kgHP32m12WwXscx-kG5Kx-AuWS1LFbMfBLg,16202
|
53
|
+
runnable/integration.py,sha256=eb9qJVZR7Ehg0N1UnGPuyjJvoA-xQ1-xP7AlZHUXHqM,6705
|
58
54
|
runnable/names.py,sha256=vn92Kv9ANROYSZX6Z4z1v_WA3WiEdIYmG6KEStBFZug,8134
|
59
|
-
runnable/nodes.py,sha256
|
60
|
-
runnable/parameters.py,sha256
|
55
|
+
runnable/nodes.py,sha256=UqR-bJx0Hi7uLSUw_saB7VsNdFh3POKtdgsEPsasHfE,16576
|
56
|
+
runnable/parameters.py,sha256=KGGW8_uoIK2hd3EwzzBmoHBOrai3fh-SESNPpJRTfj4,5161
|
61
57
|
runnable/pickler.py,sha256=5SDNf0miMUJ3ZauhQdzwk8_t-9jeOqaTjP5bvRnu9sU,2685
|
62
|
-
runnable/sdk.py,sha256=
|
58
|
+
runnable/sdk.py,sha256=hx29PEDYjJIWaTZp3ZhyLDwuulG1HIQ2q7A4HVE1WkM,26998
|
63
59
|
runnable/secrets.py,sha256=dakb7WRloWVo-KpQp6Vy4rwFdGi58BTlT4OifQY106I,2324
|
64
|
-
runnable/tasks.py,sha256=
|
65
|
-
runnable/utils.py,sha256=
|
66
|
-
runnable-0.
|
67
|
-
runnable-0.
|
68
|
-
runnable-0.
|
69
|
-
runnable-0.
|
70
|
-
runnable-0.
|
60
|
+
runnable/tasks.py,sha256=T8vVLo-yWHanrXKHFJvNdjQXKuWmJ13_-lxZzmf1mQM,18908
|
61
|
+
runnable/utils.py,sha256=okZFGbJWqStl5Rq5vLhNUQZDv_vhcT58bq9MDrTVxhc,19449
|
62
|
+
runnable-0.11.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
63
|
+
runnable-0.11.0.dist-info/METADATA,sha256=M7YR0Y_j7zMnXCKHFWGk4mlWKYZTdkOyZa3M0v3NokI,17063
|
64
|
+
runnable-0.11.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
|
65
|
+
runnable-0.11.0.dist-info/entry_points.txt,sha256=Wy-dimdD2REO2a36Ri84fqGqA5iwGy2RIbdgRNtCNdM,1540
|
66
|
+
runnable-0.11.0.dist-info/RECORD,,
|
@@ -12,10 +12,6 @@ local-container=runnable.extensions.executor.local_container.implementation:Loca
|
|
12
12
|
mocked=runnable.extensions.executor.mocked.implementation:MockedExecutor
|
13
13
|
retry=runnable.extensions.executor.retry.implementation:RetryExecutor
|
14
14
|
|
15
|
-
[experiment_tracker]
|
16
|
-
do-nothing=runnable.experiment_tracker:DoNothingTracker
|
17
|
-
mlflow=runnable.extensions.experiment_tracker.mlflow.implementation:MLFlowExperimentTracker
|
18
|
-
|
19
15
|
[nodes]
|
20
16
|
dag=runnable.extensions.nodes:DagNode
|
21
17
|
fail=runnable.extensions.nodes:FailNode
|
runnable/experiment_tracker.py
DELETED
@@ -1,139 +0,0 @@
|
|
1
|
-
import contextlib
|
2
|
-
import json
|
3
|
-
import logging
|
4
|
-
import os
|
5
|
-
from abc import ABC, abstractmethod
|
6
|
-
from collections import defaultdict
|
7
|
-
from typing import Any, ContextManager, Dict, Tuple, Union
|
8
|
-
|
9
|
-
from pydantic import BaseModel, ConfigDict
|
10
|
-
|
11
|
-
import runnable.context as context
|
12
|
-
from runnable import defaults
|
13
|
-
from runnable.utils import remove_prefix
|
14
|
-
|
15
|
-
logger = logging.getLogger(defaults.LOGGER_NAME)
|
16
|
-
|
17
|
-
|
18
|
-
def retrieve_step_details(key: str) -> Tuple[str, int]:
|
19
|
-
key = remove_prefix(key, defaults.TRACK_PREFIX)
|
20
|
-
data = key.split(defaults.STEP_INDICATOR)
|
21
|
-
|
22
|
-
key = data[0].lower()
|
23
|
-
step = 0
|
24
|
-
|
25
|
-
if len(data) > 1:
|
26
|
-
step = int(data[1])
|
27
|
-
|
28
|
-
return key, step
|
29
|
-
|
30
|
-
|
31
|
-
def get_tracked_data() -> Dict[str, Any]:
|
32
|
-
tracked_data: Dict[str, Any] = defaultdict(dict)
|
33
|
-
for env_var, value in os.environ.items():
|
34
|
-
if env_var.startswith(defaults.TRACK_PREFIX):
|
35
|
-
key, step = retrieve_step_details(env_var)
|
36
|
-
|
37
|
-
# print(value, type(value))
|
38
|
-
try:
|
39
|
-
value = json.loads(value)
|
40
|
-
except json.decoder.JSONDecodeError:
|
41
|
-
logger.warning(f"Tracker {key} could not be JSON decoded, adding the literal value")
|
42
|
-
|
43
|
-
tracked_data[key][step] = value
|
44
|
-
del os.environ[env_var]
|
45
|
-
|
46
|
-
for key, value in tracked_data.items():
|
47
|
-
if len(value) == 1:
|
48
|
-
tracked_data[key] = value[0]
|
49
|
-
|
50
|
-
return tracked_data
|
51
|
-
|
52
|
-
|
53
|
-
# --8<-- [start:docs]
|
54
|
-
|
55
|
-
|
56
|
-
class BaseExperimentTracker(ABC, BaseModel):
|
57
|
-
"""
|
58
|
-
Base Experiment tracker class definition.
|
59
|
-
"""
|
60
|
-
|
61
|
-
service_name: str = ""
|
62
|
-
service_type: str = "experiment_tracker"
|
63
|
-
|
64
|
-
@property
|
65
|
-
def _context(self):
|
66
|
-
return context.run_context
|
67
|
-
|
68
|
-
model_config = ConfigDict(extra="forbid")
|
69
|
-
|
70
|
-
@property
|
71
|
-
def client_context(self) -> ContextManager:
|
72
|
-
"""
|
73
|
-
Returns the client context.
|
74
|
-
"""
|
75
|
-
return contextlib.nullcontext()
|
76
|
-
|
77
|
-
def publish_data(self, tracked_data: Dict[str, Any]):
|
78
|
-
for key, value in tracked_data.items():
|
79
|
-
if isinstance(value, dict):
|
80
|
-
for key2, value2 in value.items():
|
81
|
-
self.log_metric(key, value2, step=key2)
|
82
|
-
continue
|
83
|
-
self.log_metric(key, value)
|
84
|
-
|
85
|
-
@abstractmethod
|
86
|
-
def log_metric(self, key: str, value: Union[int, float], step: int = 0):
|
87
|
-
"""
|
88
|
-
Sets the metric in the experiment tracking.
|
89
|
-
|
90
|
-
Args:
|
91
|
-
key (str): The key against you want to store the value
|
92
|
-
value (float): The value of the metric
|
93
|
-
step (int): Optional step at which it was recorded
|
94
|
-
|
95
|
-
Raises:
|
96
|
-
NotImplementedError: Base class, hence not implemented
|
97
|
-
"""
|
98
|
-
raise NotImplementedError
|
99
|
-
|
100
|
-
@abstractmethod
|
101
|
-
def log_parameter(self, key: str, value: Any):
|
102
|
-
"""
|
103
|
-
Logs a parameter in the experiment tracking.
|
104
|
-
|
105
|
-
Args:
|
106
|
-
key (str): The key against you want to store the value
|
107
|
-
value (any): The value of the metric
|
108
|
-
|
109
|
-
Raises:
|
110
|
-
NotImplementedError: Base class, hence not implemented
|
111
|
-
"""
|
112
|
-
pass
|
113
|
-
|
114
|
-
|
115
|
-
# --8<-- [end:docs]
|
116
|
-
|
117
|
-
|
118
|
-
class DoNothingTracker(BaseExperimentTracker):
|
119
|
-
"""
|
120
|
-
A Do nothing tracker
|
121
|
-
"""
|
122
|
-
|
123
|
-
service_name: str = "do-nothing"
|
124
|
-
|
125
|
-
def log_metric(self, key: str, value: Union[int, float], step: int = 0):
|
126
|
-
"""
|
127
|
-
Sets the metric in the experiment tracking.
|
128
|
-
|
129
|
-
Args:
|
130
|
-
key (str): The key against you want to store the value
|
131
|
-
value (float): The value of the metric
|
132
|
-
"""
|
133
|
-
...
|
134
|
-
|
135
|
-
def log_parameter(self, key: str, value: Any):
|
136
|
-
"""
|
137
|
-
Since this is a Do nothing tracker, we don't need to log anything.
|
138
|
-
"""
|
139
|
-
...
|
File without changes
|
File without changes
|