runnable 0.12.3__py3-none-any.whl → 0.14.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- runnable/__init__.py +0 -11
- runnable/catalog.py +27 -5
- runnable/cli.py +122 -26
- runnable/datastore.py +71 -35
- runnable/defaults.py +0 -1
- runnable/entrypoints.py +107 -32
- runnable/exceptions.py +6 -2
- runnable/executor.py +28 -9
- runnable/graph.py +37 -12
- runnable/integration.py +7 -2
- runnable/nodes.py +15 -17
- runnable/parameters.py +27 -8
- runnable/pickler.py +1 -1
- runnable/sdk.py +101 -33
- runnable/secrets.py +3 -1
- runnable/tasks.py +246 -34
- runnable/utils.py +41 -13
- {runnable-0.12.3.dist-info → runnable-0.14.0.dist-info}/METADATA +25 -31
- runnable-0.14.0.dist-info/RECORD +24 -0
- {runnable-0.12.3.dist-info → runnable-0.14.0.dist-info}/WHEEL +1 -1
- runnable-0.14.0.dist-info/entry_points.txt +40 -0
- runnable/extensions/__init__.py +0 -0
- runnable/extensions/catalog/__init__.py +0 -21
- runnable/extensions/catalog/file_system/__init__.py +0 -0
- runnable/extensions/catalog/file_system/implementation.py +0 -234
- runnable/extensions/catalog/k8s_pvc/__init__.py +0 -0
- runnable/extensions/catalog/k8s_pvc/implementation.py +0 -16
- runnable/extensions/catalog/k8s_pvc/integration.py +0 -59
- runnable/extensions/executor/__init__.py +0 -649
- runnable/extensions/executor/argo/__init__.py +0 -0
- runnable/extensions/executor/argo/implementation.py +0 -1194
- runnable/extensions/executor/argo/specification.yaml +0 -51
- runnable/extensions/executor/k8s_job/__init__.py +0 -0
- runnable/extensions/executor/k8s_job/implementation_FF.py +0 -259
- runnable/extensions/executor/k8s_job/integration_FF.py +0 -69
- runnable/extensions/executor/local/__init__.py +0 -0
- runnable/extensions/executor/local/implementation.py +0 -71
- runnable/extensions/executor/local_container/__init__.py +0 -0
- runnable/extensions/executor/local_container/implementation.py +0 -446
- runnable/extensions/executor/mocked/__init__.py +0 -0
- runnable/extensions/executor/mocked/implementation.py +0 -154
- runnable/extensions/executor/retry/__init__.py +0 -0
- runnable/extensions/executor/retry/implementation.py +0 -168
- runnable/extensions/nodes.py +0 -855
- runnable/extensions/run_log_store/__init__.py +0 -0
- runnable/extensions/run_log_store/chunked_file_system/__init__.py +0 -0
- runnable/extensions/run_log_store/chunked_file_system/implementation.py +0 -111
- runnable/extensions/run_log_store/chunked_k8s_pvc/__init__.py +0 -0
- runnable/extensions/run_log_store/chunked_k8s_pvc/implementation.py +0 -21
- runnable/extensions/run_log_store/chunked_k8s_pvc/integration.py +0 -61
- runnable/extensions/run_log_store/db/implementation_FF.py +0 -157
- runnable/extensions/run_log_store/db/integration_FF.py +0 -0
- runnable/extensions/run_log_store/file_system/__init__.py +0 -0
- runnable/extensions/run_log_store/file_system/implementation.py +0 -140
- runnable/extensions/run_log_store/generic_chunked.py +0 -557
- runnable/extensions/run_log_store/k8s_pvc/__init__.py +0 -0
- runnable/extensions/run_log_store/k8s_pvc/implementation.py +0 -21
- runnable/extensions/run_log_store/k8s_pvc/integration.py +0 -56
- runnable/extensions/secrets/__init__.py +0 -0
- runnable/extensions/secrets/dotenv/__init__.py +0 -0
- runnable/extensions/secrets/dotenv/implementation.py +0 -100
- runnable-0.12.3.dist-info/RECORD +0 -64
- runnable-0.12.3.dist-info/entry_points.txt +0 -41
- {runnable-0.12.3.dist-info → runnable-0.14.0.dist-info/licenses}/LICENSE +0 -0
runnable/parameters.py
CHANGED
@@ -15,6 +15,8 @@ from runnable.utils import remove_prefix
|
|
15
15
|
|
16
16
|
logger = logging.getLogger(defaults.LOGGER_NAME)
|
17
17
|
|
18
|
+
# TODO: Revisit this, it might be a bit too complicated than required
|
19
|
+
|
18
20
|
|
19
21
|
def get_user_set_parameters(remove: bool = False) -> Dict[str, JsonParameter]:
|
20
22
|
"""
|
@@ -34,9 +36,13 @@ def get_user_set_parameters(remove: bool = False) -> Dict[str, JsonParameter]:
|
|
34
36
|
if env_var.startswith(defaults.PARAMETER_PREFIX):
|
35
37
|
key = remove_prefix(env_var, defaults.PARAMETER_PREFIX)
|
36
38
|
try:
|
37
|
-
parameters[key.lower()] = JsonParameter(
|
39
|
+
parameters[key.lower()] = JsonParameter(
|
40
|
+
kind="json", value=json.loads(value)
|
41
|
+
)
|
38
42
|
except json.decoder.JSONDecodeError:
|
39
|
-
logger.warning(
|
43
|
+
logger.warning(
|
44
|
+
f"Parameter {key} could not be JSON decoded, adding the literal value"
|
45
|
+
)
|
40
46
|
parameters[key.lower()] = JsonParameter(kind="json", value=value)
|
41
47
|
|
42
48
|
if remove:
|
@@ -52,7 +58,9 @@ def serialize_parameter_as_str(value: Any) -> str:
|
|
52
58
|
|
53
59
|
|
54
60
|
def filter_arguments_for_func(
|
55
|
-
func: Callable[..., Any],
|
61
|
+
func: Callable[..., Any],
|
62
|
+
params: Dict[str, Any],
|
63
|
+
map_variable: TypeMapVariable = None,
|
56
64
|
) -> Dict[str, Any]:
|
57
65
|
"""
|
58
66
|
Inspects the function to be called as part of the pipeline to find the arguments of the function.
|
@@ -96,11 +104,16 @@ def filter_arguments_for_func(
|
|
96
104
|
# No parameter of this name was provided
|
97
105
|
if value.default == inspect.Parameter.empty:
|
98
106
|
# No default value is given in the function signature. error as parameter is required.
|
99
|
-
raise ValueError(
|
107
|
+
raise ValueError(
|
108
|
+
f"Parameter {name} is required for {func.__name__} but not provided"
|
109
|
+
)
|
100
110
|
# default value is given in the function signature, nothing further to do.
|
101
111
|
continue
|
102
112
|
|
103
|
-
if type(value.annotation) in [
|
113
|
+
if type(value.annotation) in [
|
114
|
+
BaseModel,
|
115
|
+
pydantic._internal._model_construction.ModelMetaclass,
|
116
|
+
]:
|
104
117
|
# We try to cast it as a pydantic model if asked
|
105
118
|
named_param = params[name].get_value()
|
106
119
|
|
@@ -110,7 +123,9 @@ def filter_arguments_for_func(
|
|
110
123
|
|
111
124
|
bound_model = bind_args_for_pydantic_model(named_param, value.annotation)
|
112
125
|
bound_args[name] = bound_model
|
113
|
-
unassigned_params = unassigned_params.difference(
|
126
|
+
unassigned_params = unassigned_params.difference(
|
127
|
+
bound_model.model_fields.keys()
|
128
|
+
)
|
114
129
|
|
115
130
|
elif value.annotation in [str, int, float, bool]:
|
116
131
|
# Cast it if its a primitive type. Ensure the type matches the annotation.
|
@@ -120,12 +135,16 @@ def filter_arguments_for_func(
|
|
120
135
|
|
121
136
|
unassigned_params.remove(name)
|
122
137
|
|
123
|
-
params = {
|
138
|
+
params = {
|
139
|
+
key: params[key] for key in unassigned_params
|
140
|
+
} # remove keys from params if they are assigned
|
124
141
|
|
125
142
|
return bound_args
|
126
143
|
|
127
144
|
|
128
|
-
def bind_args_for_pydantic_model(
|
145
|
+
def bind_args_for_pydantic_model(
|
146
|
+
params: Dict[str, Any], model: Type[BaseModel]
|
147
|
+
) -> BaseModel:
|
129
148
|
class EasyModel(model): # type: ignore
|
130
149
|
model_config = ConfigDict(extra="ignore")
|
131
150
|
|
runnable/pickler.py
CHANGED
@@ -9,7 +9,7 @@ import runnable.context as context
|
|
9
9
|
|
10
10
|
class BasePickler(ABC, BaseModel):
|
11
11
|
"""
|
12
|
-
The base class for all
|
12
|
+
The base class for all pickler.
|
13
13
|
|
14
14
|
We are still in the process of hardening the design of this class.
|
15
15
|
For now, we are just going to use pickle.
|
runnable/sdk.py
CHANGED
@@ -26,8 +26,7 @@ from rich.progress import (
|
|
26
26
|
from rich.table import Column
|
27
27
|
from typing_extensions import Self
|
28
28
|
|
29
|
-
from
|
30
|
-
from runnable.extensions.nodes import (
|
29
|
+
from extensions.nodes.nodes import (
|
31
30
|
FailNode,
|
32
31
|
MapNode,
|
33
32
|
ParallelNode,
|
@@ -35,9 +34,12 @@ from runnable.extensions.nodes import (
|
|
35
34
|
SuccessNode,
|
36
35
|
TaskNode,
|
37
36
|
)
|
37
|
+
from runnable import console, defaults, entrypoints, exceptions, graph, utils
|
38
38
|
from runnable.nodes import TraversalNode
|
39
39
|
from runnable.tasks import TaskReturns
|
40
40
|
|
41
|
+
# TODO: This might have to be an extension
|
42
|
+
|
41
43
|
logger = logging.getLogger(defaults.LOGGER_NAME)
|
42
44
|
|
43
45
|
StepType = Union["Stub", "PythonTask", "NotebookTask", "ShellTask", "Parallel", "Map"]
|
@@ -66,7 +68,9 @@ class Catalog(BaseModel):
|
|
66
68
|
|
67
69
|
"""
|
68
70
|
|
69
|
-
model_config = ConfigDict(
|
71
|
+
model_config = ConfigDict(
|
72
|
+
extra="forbid"
|
73
|
+
) # Need to be for command, would be validated later
|
70
74
|
# Note: compute_data_folder was confusing to explain, might be introduced later.
|
71
75
|
# compute_data_folder: str = Field(default="", alias="compute_data_folder")
|
72
76
|
get: List[str] = Field(default_factory=list, alias="get")
|
@@ -95,14 +99,18 @@ class BaseTraversal(ABC, BaseModel):
|
|
95
99
|
|
96
100
|
def __rshift__(self, other: StepType) -> StepType:
|
97
101
|
if self.next_node:
|
98
|
-
raise Exception(
|
102
|
+
raise Exception(
|
103
|
+
f"The node {self} already has a next node: {self.next_node}"
|
104
|
+
)
|
99
105
|
self.next_node = other.name
|
100
106
|
|
101
107
|
return other
|
102
108
|
|
103
109
|
def __lshift__(self, other: TraversalNode) -> TraversalNode:
|
104
110
|
if other.next_node:
|
105
|
-
raise Exception(
|
111
|
+
raise Exception(
|
112
|
+
f"The {other} node already has a next node: {other.next_node}"
|
113
|
+
)
|
106
114
|
other.next_node = self.name
|
107
115
|
|
108
116
|
return other
|
@@ -112,7 +120,9 @@ class BaseTraversal(ABC, BaseModel):
|
|
112
120
|
assert not isinstance(node, Fail)
|
113
121
|
|
114
122
|
if node.next_node:
|
115
|
-
raise Exception(
|
123
|
+
raise Exception(
|
124
|
+
f"The {node} node already has a next node: {node.next_node}"
|
125
|
+
)
|
116
126
|
|
117
127
|
node.next_node = self.name
|
118
128
|
return self
|
@@ -124,7 +134,9 @@ class BaseTraversal(ABC, BaseModel):
|
|
124
134
|
|
125
135
|
if self.terminate_with_failure or self.terminate_with_success:
|
126
136
|
if self.next_node and self.next_node not in ["success", "fail"]:
|
127
|
-
raise AssertionError(
|
137
|
+
raise AssertionError(
|
138
|
+
"A node being terminated cannot have a user defined next node"
|
139
|
+
)
|
128
140
|
|
129
141
|
if self.terminate_with_failure:
|
130
142
|
self.next_node = "fail"
|
@@ -135,8 +147,7 @@ class BaseTraversal(ABC, BaseModel):
|
|
135
147
|
return self
|
136
148
|
|
137
149
|
@abstractmethod
|
138
|
-
def create_node(self) -> TraversalNode:
|
139
|
-
...
|
150
|
+
def create_node(self) -> TraversalNode: ...
|
140
151
|
|
141
152
|
|
142
153
|
class BaseTask(BaseTraversal):
|
@@ -146,12 +157,16 @@ class BaseTask(BaseTraversal):
|
|
146
157
|
|
147
158
|
catalog: Optional[Catalog] = Field(default=None, alias="catalog")
|
148
159
|
overrides: Dict[str, Any] = Field(default_factory=dict, alias="overrides")
|
149
|
-
returns: List[Union[str, TaskReturns]] = Field(
|
160
|
+
returns: List[Union[str, TaskReturns]] = Field(
|
161
|
+
default_factory=list, alias="returns"
|
162
|
+
)
|
150
163
|
secrets: List[str] = Field(default_factory=list)
|
151
164
|
|
152
165
|
@field_validator("returns", mode="before")
|
153
166
|
@classmethod
|
154
|
-
def serialize_returns(
|
167
|
+
def serialize_returns(
|
168
|
+
cls, returns: List[Union[str, TaskReturns]]
|
169
|
+
) -> List[TaskReturns]:
|
155
170
|
task_returns = []
|
156
171
|
|
157
172
|
for x in returns:
|
@@ -167,9 +182,13 @@ class BaseTask(BaseTraversal):
|
|
167
182
|
def create_node(self) -> TaskNode:
|
168
183
|
if not self.next_node:
|
169
184
|
if not (self.terminate_with_failure or self.terminate_with_success):
|
170
|
-
raise AssertionError(
|
185
|
+
raise AssertionError(
|
186
|
+
"A node not being terminated must have a user defined next node"
|
187
|
+
)
|
171
188
|
|
172
|
-
return TaskNode.parse_from_config(
|
189
|
+
return TaskNode.parse_from_config(
|
190
|
+
self.model_dump(exclude_none=True, by_alias=True)
|
191
|
+
)
|
173
192
|
|
174
193
|
|
175
194
|
class PythonTask(BaseTask):
|
@@ -326,7 +345,9 @@ class NotebookTask(BaseTask):
|
|
326
345
|
"""
|
327
346
|
|
328
347
|
notebook: str = Field(serialization_alias="command")
|
329
|
-
optional_ploomber_args: Optional[Dict[str, Any]] = Field(
|
348
|
+
optional_ploomber_args: Optional[Dict[str, Any]] = Field(
|
349
|
+
default=None, alias="optional_ploomber_args"
|
350
|
+
)
|
330
351
|
|
331
352
|
@computed_field
|
332
353
|
def command_type(self) -> str:
|
@@ -416,7 +437,9 @@ class Stub(BaseTraversal):
|
|
416
437
|
def create_node(self) -> StubNode:
|
417
438
|
if not self.next_node:
|
418
439
|
if not (self.terminate_with_failure or self.terminate_with_success):
|
419
|
-
raise AssertionError(
|
440
|
+
raise AssertionError(
|
441
|
+
"A node not being terminated must have a user defined next node"
|
442
|
+
)
|
420
443
|
|
421
444
|
return StubNode.parse_from_config(self.model_dump(exclude_none=True))
|
422
445
|
|
@@ -439,14 +462,23 @@ class Parallel(BaseTraversal):
|
|
439
462
|
@computed_field # type: ignore
|
440
463
|
@property
|
441
464
|
def graph_branches(self) -> Dict[str, graph.Graph]:
|
442
|
-
return {
|
465
|
+
return {
|
466
|
+
name: pipeline._dag.model_copy() for name, pipeline in self.branches.items()
|
467
|
+
}
|
443
468
|
|
444
469
|
def create_node(self) -> ParallelNode:
|
445
470
|
if not self.next_node:
|
446
471
|
if not (self.terminate_with_failure or self.terminate_with_success):
|
447
|
-
raise AssertionError(
|
472
|
+
raise AssertionError(
|
473
|
+
"A node not being terminated must have a user defined next node"
|
474
|
+
)
|
448
475
|
|
449
|
-
node = ParallelNode(
|
476
|
+
node = ParallelNode(
|
477
|
+
name=self.name,
|
478
|
+
branches=self.graph_branches,
|
479
|
+
internal_name="",
|
480
|
+
next_node=self.next_node,
|
481
|
+
)
|
450
482
|
return node
|
451
483
|
|
452
484
|
|
@@ -483,7 +515,9 @@ class Map(BaseTraversal):
|
|
483
515
|
def create_node(self) -> MapNode:
|
484
516
|
if not self.next_node:
|
485
517
|
if not (self.terminate_with_failure or self.terminate_with_success):
|
486
|
-
raise AssertionError(
|
518
|
+
raise AssertionError(
|
519
|
+
"A node not being terminated must have a user defined next node"
|
520
|
+
)
|
487
521
|
|
488
522
|
node = MapNode(
|
489
523
|
name=self.name,
|
@@ -596,16 +630,22 @@ class Pipeline(BaseModel):
|
|
596
630
|
for step in path:
|
597
631
|
if step.terminate_with_success:
|
598
632
|
if reached_success:
|
599
|
-
raise Exception(
|
633
|
+
raise Exception(
|
634
|
+
"A pipeline cannot have more than one step that terminates with success"
|
635
|
+
)
|
600
636
|
reached_success = True
|
601
637
|
continue
|
602
638
|
if step.terminate_with_failure:
|
603
639
|
if reached_failure:
|
604
|
-
raise Exception(
|
640
|
+
raise Exception(
|
641
|
+
"A pipeline cannot have more than one step that terminates with failure"
|
642
|
+
)
|
605
643
|
reached_failure = True
|
606
644
|
|
607
645
|
if not reached_success and not reached_failure:
|
608
|
-
raise Exception(
|
646
|
+
raise Exception(
|
647
|
+
"A pipeline must have at least one step that terminates with success"
|
648
|
+
)
|
609
649
|
|
610
650
|
def _construct_path(self, path: List[StepType]) -> None:
|
611
651
|
prev_step = path[0]
|
@@ -615,7 +655,9 @@ class Pipeline(BaseModel):
|
|
615
655
|
continue
|
616
656
|
|
617
657
|
if prev_step.terminate_with_success or prev_step.terminate_with_failure:
|
618
|
-
raise Exception(
|
658
|
+
raise Exception(
|
659
|
+
f"A step that terminates with success/failure cannot have a next step: {prev_step}"
|
660
|
+
)
|
619
661
|
|
620
662
|
if prev_step.next_node and prev_step.next_node not in ["success", "fail"]:
|
621
663
|
raise Exception(f"Step already has a next node: {prev_step} ")
|
@@ -646,7 +688,9 @@ class Pipeline(BaseModel):
|
|
646
688
|
on_failure_paths: List[List[StepType]] = []
|
647
689
|
|
648
690
|
for step in self.steps:
|
649
|
-
if isinstance(
|
691
|
+
if isinstance(
|
692
|
+
step, (Stub, PythonTask, NotebookTask, ShellTask, Parallel, Map)
|
693
|
+
):
|
650
694
|
success_path.append(step)
|
651
695
|
continue
|
652
696
|
# on_failure_paths.append(step)
|
@@ -731,7 +775,9 @@ class Pipeline(BaseModel):
|
|
731
775
|
logger.setLevel(log_level)
|
732
776
|
|
733
777
|
run_id = utils.generate_run_id(run_id=run_id)
|
734
|
-
configuration_file = os.environ.get(
|
778
|
+
configuration_file = os.environ.get(
|
779
|
+
"RUNNABLE_CONFIGURATION_FILE", configuration_file
|
780
|
+
)
|
735
781
|
run_context = entrypoints.prepare_configurations(
|
736
782
|
configuration_file=configuration_file,
|
737
783
|
run_id=run_id,
|
@@ -740,7 +786,9 @@ class Pipeline(BaseModel):
|
|
740
786
|
)
|
741
787
|
|
742
788
|
run_context.execution_plan = defaults.EXECUTION_PLAN.CHAINED.value
|
743
|
-
utils.set_runnable_environment_variables(
|
789
|
+
utils.set_runnable_environment_variables(
|
790
|
+
run_id=run_id, configuration_file=configuration_file, tag=tag
|
791
|
+
)
|
744
792
|
|
745
793
|
dag_definition = self._dag.model_dump(by_alias=True, exclude_none=True)
|
746
794
|
|
@@ -767,7 +815,9 @@ class Pipeline(BaseModel):
|
|
767
815
|
|
768
816
|
with Progress(
|
769
817
|
SpinnerColumn(spinner_name="runner"),
|
770
|
-
TextColumn(
|
818
|
+
TextColumn(
|
819
|
+
"[progress.description]{task.description}", table_column=Column(ratio=2)
|
820
|
+
),
|
771
821
|
BarColumn(table_column=Column(ratio=1), style="dark_orange"),
|
772
822
|
TimeElapsedColumn(table_column=Column(ratio=1)),
|
773
823
|
console=console,
|
@@ -775,23 +825,41 @@ class Pipeline(BaseModel):
|
|
775
825
|
) as progress:
|
776
826
|
try:
|
777
827
|
run_context.progress = progress
|
778
|
-
pipeline_execution_task = progress.add_task(
|
828
|
+
pipeline_execution_task = progress.add_task(
|
829
|
+
"[dark_orange] Starting execution .. ", total=1
|
830
|
+
)
|
779
831
|
run_context.executor.execute_graph(dag=run_context.dag)
|
780
832
|
|
781
833
|
if not run_context.executor._local:
|
782
834
|
return {}
|
783
835
|
|
784
|
-
run_log = run_context.run_log_store.get_run_log_by_id(
|
836
|
+
run_log = run_context.run_log_store.get_run_log_by_id(
|
837
|
+
run_id=run_context.run_id, full=False
|
838
|
+
)
|
785
839
|
|
786
840
|
if run_log.status == defaults.SUCCESS:
|
787
|
-
progress.update(
|
841
|
+
progress.update(
|
842
|
+
pipeline_execution_task,
|
843
|
+
description="[green] Success",
|
844
|
+
completed=True,
|
845
|
+
)
|
788
846
|
else:
|
789
|
-
progress.update(
|
847
|
+
progress.update(
|
848
|
+
pipeline_execution_task,
|
849
|
+
description="[red] Failed",
|
850
|
+
completed=True,
|
851
|
+
)
|
790
852
|
raise exceptions.ExecutionFailedError(run_context.run_id)
|
791
853
|
except Exception as e: # noqa: E722
|
792
854
|
console.print(e, style=defaults.error_style)
|
793
|
-
progress.update(
|
855
|
+
progress.update(
|
856
|
+
pipeline_execution_task,
|
857
|
+
description="[red] Errored execution",
|
858
|
+
completed=True,
|
859
|
+
)
|
794
860
|
raise
|
795
861
|
|
796
862
|
if run_context.executor._local:
|
797
|
-
return run_context.run_log_store.get_run_log_by_id(
|
863
|
+
return run_context.run_log_store.get_run_log_by_id(
|
864
|
+
run_id=run_context.run_id
|
865
|
+
)
|
runnable/secrets.py
CHANGED
@@ -92,4 +92,6 @@ class EnvSecretsManager(BaseSecrets):
|
|
92
92
|
try:
|
93
93
|
return os.environ[name]
|
94
94
|
except KeyError:
|
95
|
-
raise exceptions.SecretNotFoundError(
|
95
|
+
raise exceptions.SecretNotFoundError(
|
96
|
+
secret_name=name, secret_setting="environment variables"
|
97
|
+
)
|