runnable 0.34.0a3__py3-none-any.whl → 0.36.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- extensions/job_executor/__init__.py +3 -4
- extensions/job_executor/emulate.py +106 -0
- extensions/job_executor/k8s.py +8 -8
- extensions/job_executor/local_container.py +13 -14
- extensions/nodes/__init__.py +0 -0
- extensions/nodes/conditional.py +243 -0
- extensions/nodes/fail.py +72 -0
- extensions/nodes/map.py +350 -0
- extensions/nodes/parallel.py +159 -0
- extensions/nodes/stub.py +89 -0
- extensions/nodes/success.py +72 -0
- extensions/nodes/task.py +92 -0
- extensions/pipeline_executor/__init__.py +24 -26
- extensions/pipeline_executor/argo.py +50 -41
- extensions/pipeline_executor/emulate.py +112 -0
- extensions/pipeline_executor/local.py +4 -4
- extensions/pipeline_executor/local_container.py +19 -79
- extensions/pipeline_executor/mocked.py +4 -4
- extensions/pipeline_executor/retry.py +6 -10
- extensions/tasks/torch.py +1 -1
- runnable/__init__.py +2 -9
- runnable/catalog.py +1 -21
- runnable/cli.py +0 -59
- runnable/context.py +519 -28
- runnable/datastore.py +51 -54
- runnable/defaults.py +12 -34
- runnable/entrypoints.py +82 -440
- runnable/exceptions.py +35 -34
- runnable/executor.py +13 -20
- runnable/names.py +1 -1
- runnable/nodes.py +18 -16
- runnable/parameters.py +2 -2
- runnable/sdk.py +117 -164
- runnable/tasks.py +62 -21
- runnable/utils.py +6 -268
- {runnable-0.34.0a3.dist-info → runnable-0.36.0.dist-info}/METADATA +1 -2
- runnable-0.36.0.dist-info/RECORD +74 -0
- {runnable-0.34.0a3.dist-info → runnable-0.36.0.dist-info}/entry_points.txt +9 -8
- extensions/nodes/nodes.py +0 -778
- extensions/nodes/torch.py +0 -273
- extensions/nodes/torch_config.py +0 -76
- runnable-0.34.0a3.dist-info/RECORD +0 -67
- {runnable-0.34.0a3.dist-info → runnable-0.36.0.dist-info}/WHEEL +0 -0
- {runnable-0.34.0a3.dist-info → runnable-0.36.0.dist-info}/licenses/LICENSE +0 -0
extensions/nodes/task.py
ADDED
@@ -0,0 +1,92 @@
|
|
1
|
+
import logging
|
2
|
+
from datetime import datetime
|
3
|
+
from typing import Any, Dict
|
4
|
+
|
5
|
+
from pydantic import ConfigDict, Field
|
6
|
+
|
7
|
+
from runnable import datastore, defaults
|
8
|
+
from runnable.datastore import StepLog
|
9
|
+
from runnable.defaults import MapVariableType
|
10
|
+
from runnable.nodes import ExecutableNode
|
11
|
+
from runnable.tasks import BaseTaskType, create_task
|
12
|
+
|
13
|
+
logger = logging.getLogger(defaults.LOGGER_NAME)
|
14
|
+
|
15
|
+
|
16
|
+
class TaskNode(ExecutableNode):
|
17
|
+
"""
|
18
|
+
A node of type Task.
|
19
|
+
|
20
|
+
This node does the actual function execution of the graph in all cases.
|
21
|
+
"""
|
22
|
+
|
23
|
+
executable: BaseTaskType = Field(exclude=True)
|
24
|
+
node_type: str = Field(default="task", serialization_alias="type")
|
25
|
+
|
26
|
+
# It is technically not allowed as parse_from_config filters them.
|
27
|
+
# This is just to get the task level configuration to be present during serialization.
|
28
|
+
model_config = ConfigDict(extra="allow")
|
29
|
+
|
30
|
+
@classmethod
|
31
|
+
def parse_from_config(cls, config: Dict[str, Any]) -> "TaskNode":
|
32
|
+
# separate task config from node config
|
33
|
+
task_config = {
|
34
|
+
k: v for k, v in config.items() if k not in TaskNode.model_fields.keys()
|
35
|
+
}
|
36
|
+
node_config = {
|
37
|
+
k: v for k, v in config.items() if k in TaskNode.model_fields.keys()
|
38
|
+
}
|
39
|
+
|
40
|
+
executable = create_task(task_config)
|
41
|
+
return cls(executable=executable, **node_config, **task_config)
|
42
|
+
|
43
|
+
def get_summary(self) -> Dict[str, Any]:
|
44
|
+
summary = {
|
45
|
+
"name": self.name,
|
46
|
+
"type": self.node_type,
|
47
|
+
"executable": self.executable.get_summary(),
|
48
|
+
"catalog": self._get_catalog_settings(),
|
49
|
+
}
|
50
|
+
|
51
|
+
return summary
|
52
|
+
|
53
|
+
def execute(
|
54
|
+
self,
|
55
|
+
mock=False,
|
56
|
+
map_variable: MapVariableType = None,
|
57
|
+
attempt_number: int = 1,
|
58
|
+
) -> StepLog:
|
59
|
+
"""
|
60
|
+
All that we do in runnable is to come to this point where we actually execute the command.
|
61
|
+
|
62
|
+
Args:
|
63
|
+
executor (_type_): The executor class
|
64
|
+
mock (bool, optional): If we should just mock and not execute. Defaults to False.
|
65
|
+
map_variable (dict, optional): If the node is part of internal branch. Defaults to None.
|
66
|
+
|
67
|
+
Returns:
|
68
|
+
StepAttempt: The attempt object
|
69
|
+
"""
|
70
|
+
step_log = self._context.run_log_store.get_step_log(
|
71
|
+
self._get_step_log_name(map_variable), self._context.run_id
|
72
|
+
)
|
73
|
+
|
74
|
+
if not mock:
|
75
|
+
# Do not run if we are mocking the execution, could be useful for caching and dry runs
|
76
|
+
attempt_log = self.executable.execute_command(map_variable=map_variable)
|
77
|
+
attempt_log.attempt_number = attempt_number
|
78
|
+
else:
|
79
|
+
attempt_log = datastore.StepAttempt(
|
80
|
+
status=defaults.SUCCESS,
|
81
|
+
start_time=str(datetime.now()),
|
82
|
+
end_time=str(datetime.now()),
|
83
|
+
attempt_number=attempt_number,
|
84
|
+
)
|
85
|
+
|
86
|
+
logger.info(f"attempt_log: {attempt_log}")
|
87
|
+
logger.info(f"Step {self.name} completed with status: {attempt_log.status}")
|
88
|
+
|
89
|
+
step_log.status = attempt_log.status
|
90
|
+
step_log.attempts.append(attempt_log)
|
91
|
+
|
92
|
+
return step_log
|
@@ -13,7 +13,7 @@ from runnable import (
|
|
13
13
|
utils,
|
14
14
|
)
|
15
15
|
from runnable.datastore import DataCatalog, JsonParameter, RunLog, StepLog
|
16
|
-
from runnable.defaults import
|
16
|
+
from runnable.defaults import MapVariableType
|
17
17
|
from runnable.executor import BasePipelineExecutor
|
18
18
|
from runnable.graph import Graph
|
19
19
|
from runnable.nodes import BaseNode
|
@@ -40,7 +40,7 @@ class GenericPipelineExecutor(BasePipelineExecutor):
|
|
40
40
|
|
41
41
|
@property
|
42
42
|
def _context(self):
|
43
|
-
assert context.run_context
|
43
|
+
assert isinstance(context.run_context, context.PipelineContext)
|
44
44
|
return context.run_context
|
45
45
|
|
46
46
|
def _get_parameters(self) -> Dict[str, JsonParameter]:
|
@@ -104,7 +104,7 @@ class GenericPipelineExecutor(BasePipelineExecutor):
|
|
104
104
|
)
|
105
105
|
|
106
106
|
# Update run_config
|
107
|
-
run_config =
|
107
|
+
run_config = self._context.model_dump()
|
108
108
|
logger.debug(f"run_config as seen by executor: {run_config}")
|
109
109
|
self._context.run_log_store.set_run_config(
|
110
110
|
run_id=self._context.run_id, run_config=run_config
|
@@ -154,12 +154,12 @@ class GenericPipelineExecutor(BasePipelineExecutor):
|
|
154
154
|
data_catalogs = []
|
155
155
|
for name_pattern in node_catalog_settings.get(stage) or []:
|
156
156
|
if stage == "get":
|
157
|
-
data_catalog = self._context.
|
157
|
+
data_catalog = self._context.catalog.get(
|
158
158
|
name=name_pattern,
|
159
159
|
)
|
160
160
|
|
161
161
|
elif stage == "put":
|
162
|
-
data_catalog = self._context.
|
162
|
+
data_catalog = self._context.catalog.put(
|
163
163
|
name=name_pattern, allow_file_not_found_exc=allow_file_no_found_exc
|
164
164
|
)
|
165
165
|
else:
|
@@ -189,14 +189,15 @@ class GenericPipelineExecutor(BasePipelineExecutor):
|
|
189
189
|
map_variable=map_variable,
|
190
190
|
)
|
191
191
|
task_console.save_text(log_file_name)
|
192
|
+
task_console.export_text(clear=True)
|
192
193
|
# Put the log file in the catalog
|
193
|
-
self._context.
|
194
|
+
self._context.catalog.put(name=log_file_name)
|
194
195
|
os.remove(log_file_name)
|
195
196
|
|
196
197
|
def _execute_node(
|
197
198
|
self,
|
198
199
|
node: BaseNode,
|
199
|
-
map_variable:
|
200
|
+
map_variable: MapVariableType = None,
|
200
201
|
mock: bool = False,
|
201
202
|
):
|
202
203
|
"""
|
@@ -250,6 +251,10 @@ class GenericPipelineExecutor(BasePipelineExecutor):
|
|
250
251
|
console.print(f"Summary of the step: {step_log.internal_name}")
|
251
252
|
console.print(step_log.get_summary(), style=defaults.info_style)
|
252
253
|
|
254
|
+
self.add_task_log_to_catalog(
|
255
|
+
name=self._context_node.internal_name, map_variable=map_variable
|
256
|
+
)
|
257
|
+
|
253
258
|
self._context_node = None
|
254
259
|
|
255
260
|
self._context.run_log_store.add_step_log(step_log, self._context.run_id)
|
@@ -266,7 +271,7 @@ class GenericPipelineExecutor(BasePipelineExecutor):
|
|
266
271
|
"""
|
267
272
|
step_log.code_identities.append(utils.get_git_code_identity())
|
268
273
|
|
269
|
-
def execute_from_graph(self, node: BaseNode, map_variable:
|
274
|
+
def execute_from_graph(self, node: BaseNode, map_variable: MapVariableType = None):
|
270
275
|
"""
|
271
276
|
This is the entry point to from the graph execution.
|
272
277
|
|
@@ -315,8 +320,6 @@ class GenericPipelineExecutor(BasePipelineExecutor):
|
|
315
320
|
node.execute_as_graph(map_variable=map_variable)
|
316
321
|
return
|
317
322
|
|
318
|
-
task_console.export_text(clear=True)
|
319
|
-
|
320
323
|
task_name = node._resolve_map_placeholders(node.internal_name, map_variable)
|
321
324
|
console.print(
|
322
325
|
f":runner: Executing the node {task_name} ... ", style="bold color(208)"
|
@@ -324,7 +327,7 @@ class GenericPipelineExecutor(BasePipelineExecutor):
|
|
324
327
|
self.trigger_node_execution(node=node, map_variable=map_variable)
|
325
328
|
|
326
329
|
def trigger_node_execution(
|
327
|
-
self, node: BaseNode, map_variable:
|
330
|
+
self, node: BaseNode, map_variable: MapVariableType = None
|
328
331
|
):
|
329
332
|
"""
|
330
333
|
Call this method only if we are responsible for traversing the graph via
|
@@ -342,7 +345,7 @@ class GenericPipelineExecutor(BasePipelineExecutor):
|
|
342
345
|
pass
|
343
346
|
|
344
347
|
def _get_status_and_next_node_name(
|
345
|
-
self, current_node: BaseNode, dag: Graph, map_variable:
|
348
|
+
self, current_node: BaseNode, dag: Graph, map_variable: MapVariableType = None
|
346
349
|
) -> tuple[str, str]:
|
347
350
|
"""
|
348
351
|
Given the current node and the graph, returns the name of the next node to execute.
|
@@ -380,7 +383,7 @@ class GenericPipelineExecutor(BasePipelineExecutor):
|
|
380
383
|
|
381
384
|
return step_log.status, next_node_name
|
382
385
|
|
383
|
-
def execute_graph(self, dag: Graph, map_variable:
|
386
|
+
def execute_graph(self, dag: Graph, map_variable: MapVariableType = None):
|
384
387
|
"""
|
385
388
|
The parallelization is controlled by the nodes and not by this function.
|
386
389
|
|
@@ -409,7 +412,7 @@ class GenericPipelineExecutor(BasePipelineExecutor):
|
|
409
412
|
dag.internal_branch_name or "Graph",
|
410
413
|
map_variable,
|
411
414
|
)
|
412
|
-
branch_execution_task =
|
415
|
+
branch_execution_task = context.progress.add_task(
|
413
416
|
f"[dark_orange]Executing {branch_task_name}",
|
414
417
|
total=1,
|
415
418
|
)
|
@@ -429,7 +432,7 @@ class GenericPipelineExecutor(BasePipelineExecutor):
|
|
429
432
|
|
430
433
|
depth = " " * ((task_name.count(".")) or 1 - 1)
|
431
434
|
|
432
|
-
task_execution =
|
435
|
+
task_execution = context.progress.add_task(
|
433
436
|
f"{depth}Executing {task_name}", total=1
|
434
437
|
)
|
435
438
|
|
@@ -440,20 +443,20 @@ class GenericPipelineExecutor(BasePipelineExecutor):
|
|
440
443
|
)
|
441
444
|
|
442
445
|
if status == defaults.SUCCESS:
|
443
|
-
|
446
|
+
context.progress.update(
|
444
447
|
task_execution,
|
445
448
|
description=f"{depth}[green] {task_name} Completed",
|
446
449
|
completed=True,
|
447
450
|
overflow="fold",
|
448
451
|
)
|
449
452
|
else:
|
450
|
-
|
453
|
+
context.progress.update(
|
451
454
|
task_execution,
|
452
455
|
description=f"{depth}[red] {task_name} Failed",
|
453
456
|
completed=True,
|
454
457
|
) # type ignore
|
455
458
|
except Exception as e: # noqa: E722
|
456
|
-
|
459
|
+
context.progress.update(
|
457
460
|
task_execution,
|
458
461
|
description=f"{depth}[red] {task_name} Errored",
|
459
462
|
completed=True,
|
@@ -461,11 +464,6 @@ class GenericPipelineExecutor(BasePipelineExecutor):
|
|
461
464
|
console.print(e, style=defaults.error_style)
|
462
465
|
logger.exception(e)
|
463
466
|
raise
|
464
|
-
finally:
|
465
|
-
# Add task log to the catalog
|
466
|
-
self.add_task_log_to_catalog(
|
467
|
-
name=working_on.internal_name, map_variable=map_variable
|
468
|
-
)
|
469
467
|
|
470
468
|
console.rule(style="[dark orange]")
|
471
469
|
|
@@ -475,7 +473,7 @@ class GenericPipelineExecutor(BasePipelineExecutor):
|
|
475
473
|
current_node = next_node_name
|
476
474
|
|
477
475
|
if branch_execution_task:
|
478
|
-
|
476
|
+
context.progress.update(
|
479
477
|
branch_execution_task,
|
480
478
|
description=f"[green3] {branch_task_name} completed",
|
481
479
|
completed=True,
|
@@ -567,7 +565,7 @@ class GenericPipelineExecutor(BasePipelineExecutor):
|
|
567
565
|
|
568
566
|
return effective_node_config
|
569
567
|
|
570
|
-
def fan_out(self, node: BaseNode, map_variable:
|
568
|
+
def fan_out(self, node: BaseNode, map_variable: MapVariableType = None):
|
571
569
|
"""
|
572
570
|
This method is used to appropriately fan-out the execution of a composite node.
|
573
571
|
This is only useful when we want to execute a composite node during 3rd party orchestrators.
|
@@ -599,7 +597,7 @@ class GenericPipelineExecutor(BasePipelineExecutor):
|
|
599
597
|
|
600
598
|
node.fan_out(map_variable=map_variable)
|
601
599
|
|
602
|
-
def fan_in(self, node: BaseNode, map_variable:
|
600
|
+
def fan_in(self, node: BaseNode, map_variable: MapVariableType = None):
|
603
601
|
"""
|
604
602
|
This method is used to appropriately fan-in after the execution of a composite node.
|
605
603
|
This is only useful when we want to execute a composite node during 3rd party orchestrators.
|
@@ -20,13 +20,16 @@ from pydantic import (
|
|
20
20
|
from pydantic.alias_generators import to_camel
|
21
21
|
from ruamel.yaml import YAML
|
22
22
|
|
23
|
-
from extensions.nodes.
|
23
|
+
from extensions.nodes.conditional import ConditionalNode
|
24
|
+
from extensions.nodes.map import MapNode
|
25
|
+
from extensions.nodes.parallel import ParallelNode
|
26
|
+
from extensions.nodes.task import TaskNode
|
24
27
|
|
25
28
|
# TODO: Should be part of a wider refactor
|
26
29
|
# from extensions.nodes.torch import TorchNode
|
27
30
|
from extensions.pipeline_executor import GenericPipelineExecutor
|
28
|
-
from runnable import defaults
|
29
|
-
from runnable.defaults import
|
31
|
+
from runnable import defaults
|
32
|
+
from runnable.defaults import MapVariableType
|
30
33
|
from runnable.graph import Graph, search_node_by_internal_name
|
31
34
|
from runnable.nodes import BaseNode
|
32
35
|
|
@@ -307,6 +310,7 @@ class DagTask(BaseModelWIthConfig):
|
|
307
310
|
template: str # Should be name of a container template or dag template
|
308
311
|
arguments: Optional[Arguments] = Field(default=None)
|
309
312
|
with_param: Optional[str] = Field(default=None)
|
313
|
+
when_param: Optional[str] = Field(default=None, serialization_alias="when")
|
310
314
|
depends: Optional[str] = Field(default=None)
|
311
315
|
|
312
316
|
|
@@ -451,7 +455,7 @@ class ArgoExecutor(GenericPipelineExecutor):
|
|
451
455
|
"""
|
452
456
|
|
453
457
|
service_name: str = "argo"
|
454
|
-
|
458
|
+
_should_setup_run_log_at_traversal: bool = PrivateAttr(default=False)
|
455
459
|
mock: bool = False
|
456
460
|
|
457
461
|
model_config = ConfigDict(
|
@@ -533,13 +537,13 @@ class ArgoExecutor(GenericPipelineExecutor):
|
|
533
537
|
parameters: Optional[list[Parameter]],
|
534
538
|
task_name: str,
|
535
539
|
):
|
536
|
-
map_variable:
|
540
|
+
map_variable: MapVariableType = {}
|
537
541
|
for parameter in parameters or []:
|
538
542
|
map_variable[parameter.name] = ( # type: ignore
|
539
543
|
"{{inputs.parameters." + str(parameter.name) + "}}"
|
540
544
|
)
|
541
545
|
|
542
|
-
fan_command =
|
546
|
+
fan_command = self._context.get_fan_command(
|
543
547
|
mode=mode,
|
544
548
|
node=node,
|
545
549
|
run_id=self._run_id_as_parameter,
|
@@ -563,6 +567,8 @@ class ArgoExecutor(GenericPipelineExecutor):
|
|
563
567
|
outputs: Optional[Outputs] = None
|
564
568
|
if mode == "out" and node.node_type == "map":
|
565
569
|
outputs = Outputs(parameters=[OutputParameter(name="iterate-on")])
|
570
|
+
if mode == "out" and node.node_type == "conditional":
|
571
|
+
outputs = Outputs(parameters=[OutputParameter(name="case")])
|
566
572
|
|
567
573
|
container_template = ContainerTemplate(
|
568
574
|
name=task_name,
|
@@ -602,17 +608,17 @@ class ArgoExecutor(GenericPipelineExecutor):
|
|
602
608
|
|
603
609
|
inputs = inputs or Inputs(parameters=[])
|
604
610
|
|
605
|
-
map_variable:
|
611
|
+
map_variable: MapVariableType = {}
|
606
612
|
for parameter in inputs.parameters or []:
|
607
613
|
map_variable[parameter.name] = ( # type: ignore
|
608
614
|
"{{inputs.parameters." + str(parameter.name) + "}}"
|
609
615
|
)
|
610
616
|
|
611
617
|
# command = "runnable execute-single-node"
|
612
|
-
command =
|
618
|
+
command = self._context.get_node_callable_command(
|
613
619
|
node=node,
|
614
|
-
over_write_run_id=self._run_id_as_parameter,
|
615
620
|
map_variable=map_variable,
|
621
|
+
over_write_run_id=self._run_id_as_parameter,
|
616
622
|
log_level=self._log_level_as_parameter,
|
617
623
|
)
|
618
624
|
|
@@ -711,6 +717,7 @@ class ArgoExecutor(GenericPipelineExecutor):
|
|
711
717
|
assert parent_dag_template.dag
|
712
718
|
|
713
719
|
parent_dag_template.dag.tasks.append(on_failure_task)
|
720
|
+
|
714
721
|
self._gather_tasks_for_dag_template(
|
715
722
|
on_failure_dag,
|
716
723
|
dag=dag,
|
@@ -722,6 +729,7 @@ class ArgoExecutor(GenericPipelineExecutor):
|
|
722
729
|
# - We are using withParam and arguments of the map template to send that value in
|
723
730
|
# - The map template should receive that value as a parameter into the template.
|
724
731
|
# - The task then start to use it as inputs.parameters.iterate-on
|
732
|
+
# the when param should be an evaluation
|
725
733
|
|
726
734
|
def _gather_tasks_for_dag_template(
|
727
735
|
self,
|
@@ -757,7 +765,7 @@ class ArgoExecutor(GenericPipelineExecutor):
|
|
757
765
|
depends = task_name
|
758
766
|
|
759
767
|
match working_on.node_type:
|
760
|
-
case "task" | "success" | "stub":
|
768
|
+
case "task" | "success" | "stub" | "fail":
|
761
769
|
template_of_container = self._create_container_template(
|
762
770
|
working_on,
|
763
771
|
task_name=task_name,
|
@@ -767,9 +775,11 @@ class ArgoExecutor(GenericPipelineExecutor):
|
|
767
775
|
|
768
776
|
self._templates.append(template_of_container)
|
769
777
|
|
770
|
-
case "map" | "parallel":
|
771
|
-
assert
|
772
|
-
working_on,
|
778
|
+
case "map" | "parallel" | "conditional":
|
779
|
+
assert (
|
780
|
+
isinstance(working_on, MapNode)
|
781
|
+
or isinstance(working_on, ParallelNode)
|
782
|
+
or isinstance(working_on, ConditionalNode)
|
773
783
|
)
|
774
784
|
node_type = working_on.node_type
|
775
785
|
|
@@ -792,7 +802,8 @@ class ArgoExecutor(GenericPipelineExecutor):
|
|
792
802
|
)
|
793
803
|
|
794
804
|
# Add the composite task
|
795
|
-
with_param = None
|
805
|
+
with_param: Optional[str] = None
|
806
|
+
when_param: Optional[str] = None
|
796
807
|
added_parameters = parameters or []
|
797
808
|
branches = {}
|
798
809
|
if node_type == "map":
|
@@ -807,22 +818,34 @@ class ArgoExecutor(GenericPipelineExecutor):
|
|
807
818
|
elif node_type == "parallel":
|
808
819
|
assert isinstance(working_on, ParallelNode)
|
809
820
|
branches = working_on.branches
|
821
|
+
elif node_type == "conditional":
|
822
|
+
assert isinstance(working_on, ConditionalNode)
|
823
|
+
branches = working_on.branches
|
824
|
+
when_param = (
|
825
|
+
f"{{{{tasks.{task_name}-fan-out.outputs.parameters.case}}}}"
|
826
|
+
)
|
810
827
|
else:
|
811
828
|
raise ValueError("Invalid node type")
|
812
829
|
|
813
830
|
fan_in_depends = ""
|
814
831
|
|
815
832
|
for name, branch in branches.items():
|
833
|
+
match_when = branch.internal_branch_name.split(".")[-1]
|
816
834
|
name = (
|
817
835
|
name.replace(" ", "-").replace(".", "-").replace("_", "-")
|
818
836
|
)
|
819
837
|
|
838
|
+
if node_type == "conditional":
|
839
|
+
assert isinstance(working_on, ConditionalNode)
|
840
|
+
when_param = f"'{match_when}' == {{{{tasks.{task_name}-fan-out.outputs.parameters.case}}}}"
|
841
|
+
|
820
842
|
branch_task = DagTask(
|
821
843
|
name=f"{task_name}-{name}",
|
822
844
|
template=f"{task_name}-{name}",
|
823
845
|
depends=f"{task_name}-fan-out.Succeeded",
|
824
846
|
arguments=Arguments(parameters=added_parameters),
|
825
847
|
with_param=with_param,
|
848
|
+
when_param=when_param,
|
826
849
|
)
|
827
850
|
composite_template.dag.tasks.append(branch_task)
|
828
851
|
|
@@ -836,6 +859,8 @@ class ArgoExecutor(GenericPipelineExecutor):
|
|
836
859
|
),
|
837
860
|
)
|
838
861
|
|
862
|
+
assert isinstance(branch, Graph)
|
863
|
+
|
839
864
|
self._gather_tasks_for_dag_template(
|
840
865
|
dag_template=branch_template,
|
841
866
|
dag=branch,
|
@@ -862,28 +887,6 @@ class ArgoExecutor(GenericPipelineExecutor):
|
|
862
887
|
|
863
888
|
self._templates.append(composite_template)
|
864
889
|
|
865
|
-
case "torch":
|
866
|
-
from extensions.nodes.torch import TorchNode
|
867
|
-
|
868
|
-
assert isinstance(working_on, TorchNode)
|
869
|
-
# TODO: Need to add multi-node functionality
|
870
|
-
# Check notes on the torch node
|
871
|
-
|
872
|
-
template_of_container = self._create_container_template(
|
873
|
-
working_on,
|
874
|
-
task_name=task_name,
|
875
|
-
inputs=Inputs(parameters=parameters),
|
876
|
-
)
|
877
|
-
assert template_of_container.container is not None
|
878
|
-
|
879
|
-
if working_on.node_type == "task":
|
880
|
-
self._expose_secrets_to_task(
|
881
|
-
working_on,
|
882
|
-
container_template=template_of_container.container,
|
883
|
-
)
|
884
|
-
|
885
|
-
self._templates.append(template_of_container)
|
886
|
-
|
887
890
|
self._handle_failures(
|
888
891
|
working_on,
|
889
892
|
dag,
|
@@ -958,7 +961,7 @@ class ArgoExecutor(GenericPipelineExecutor):
|
|
958
961
|
f,
|
959
962
|
)
|
960
963
|
|
961
|
-
def _implicitly_fail(self, node: BaseNode, map_variable:
|
964
|
+
def _implicitly_fail(self, node: BaseNode, map_variable: MapVariableType):
|
962
965
|
assert self._context.dag
|
963
966
|
_, current_branch = search_node_by_internal_name(
|
964
967
|
dag=self._context.dag, internal_name=node.internal_name
|
@@ -1005,7 +1008,7 @@ class ArgoExecutor(GenericPipelineExecutor):
|
|
1005
1008
|
|
1006
1009
|
self._implicitly_fail(node, map_variable)
|
1007
1010
|
|
1008
|
-
def fan_out(self, node: BaseNode, map_variable:
|
1011
|
+
def fan_out(self, node: BaseNode, map_variable: MapVariableType = None):
|
1009
1012
|
# This could be the first step of the graph
|
1010
1013
|
self._use_volumes()
|
1011
1014
|
|
@@ -1025,7 +1028,13 @@ class ArgoExecutor(GenericPipelineExecutor):
|
|
1025
1028
|
with open("/tmp/output.txt", mode="w", encoding="utf-8") as myfile:
|
1026
1029
|
json.dump(iterate_on.get_value(), myfile, indent=4)
|
1027
1030
|
|
1028
|
-
|
1031
|
+
if node.node_type == "conditional":
|
1032
|
+
assert isinstance(node, ConditionalNode)
|
1033
|
+
|
1034
|
+
with open("/tmp/output.txt", mode="w", encoding="utf-8") as myfile:
|
1035
|
+
json.dump(node.get_parameter_value(), myfile, indent=4)
|
1036
|
+
|
1037
|
+
def fan_in(self, node: BaseNode, map_variable: MapVariableType = None):
|
1029
1038
|
self._use_volumes()
|
1030
1039
|
super().fan_in(node, map_variable)
|
1031
1040
|
|
@@ -1036,9 +1045,9 @@ class ArgoExecutor(GenericPipelineExecutor):
|
|
1036
1045
|
case "chunked-fs":
|
1037
1046
|
self._context.run_log_store.log_folder = self._container_log_location
|
1038
1047
|
|
1039
|
-
match self._context.
|
1048
|
+
match self._context.catalog.service_name:
|
1040
1049
|
case "file-system":
|
1041
|
-
self._context.
|
1050
|
+
self._context.catalog.catalog_location = (
|
1042
1051
|
self._container_catalog_location
|
1043
1052
|
)
|
1044
1053
|
|
@@ -0,0 +1,112 @@
|
|
1
|
+
import logging
|
2
|
+
import shlex
|
3
|
+
import subprocess
|
4
|
+
import sys
|
5
|
+
|
6
|
+
from pydantic import PrivateAttr
|
7
|
+
|
8
|
+
from extensions.pipeline_executor import GenericPipelineExecutor
|
9
|
+
from runnable import defaults
|
10
|
+
from runnable.defaults import MapVariableType
|
11
|
+
from runnable.nodes import BaseNode
|
12
|
+
|
13
|
+
logger = logging.getLogger(defaults.LOGGER_NAME)
|
14
|
+
|
15
|
+
|
16
|
+
class Emulator(GenericPipelineExecutor):
|
17
|
+
"""
|
18
|
+
In the mode of local execution, we run everything on the local computer.
|
19
|
+
|
20
|
+
This has some serious implications on the amount of time it would take to complete the run.
|
21
|
+
Also ensure that the local compute is good enough for the compute to happen of all the steps.
|
22
|
+
|
23
|
+
Example config:
|
24
|
+
|
25
|
+
```yaml
|
26
|
+
pipeline-executor:
|
27
|
+
type: local
|
28
|
+
```
|
29
|
+
|
30
|
+
"""
|
31
|
+
|
32
|
+
service_name: str = "emulator"
|
33
|
+
|
34
|
+
_should_setup_run_log_at_traversal: bool = PrivateAttr(default=True)
|
35
|
+
|
36
|
+
def trigger_node_execution(
|
37
|
+
self, node: BaseNode, map_variable: MapVariableType = None
|
38
|
+
):
|
39
|
+
"""
|
40
|
+
In this mode of execution, we prepare for the node execution and execute the node
|
41
|
+
|
42
|
+
Args:
|
43
|
+
node (BaseNode): [description]
|
44
|
+
map_variable (str, optional): [description]. Defaults to ''.
|
45
|
+
"""
|
46
|
+
command = self._context.get_node_callable_command(
|
47
|
+
node, map_variable=map_variable
|
48
|
+
)
|
49
|
+
|
50
|
+
self.run_click_command(command)
|
51
|
+
# execute the command in a forked process
|
52
|
+
|
53
|
+
step_log = self._context.run_log_store.get_step_log(
|
54
|
+
node._get_step_log_name(map_variable), self._context.run_id
|
55
|
+
)
|
56
|
+
if step_log.status != defaults.SUCCESS:
|
57
|
+
msg = "Node execution inside the emulate failed. Please check the logs.\n"
|
58
|
+
logger.error(msg)
|
59
|
+
step_log.status = defaults.FAIL
|
60
|
+
self._context.run_log_store.add_step_log(step_log, self._context.run_id)
|
61
|
+
|
62
|
+
def execute_node(self, node: BaseNode, map_variable: MapVariableType = None):
|
63
|
+
"""
|
64
|
+
For local execution, we just execute the node.
|
65
|
+
|
66
|
+
Args:
|
67
|
+
node (BaseNode): _description_
|
68
|
+
map_variable (dict[str, str], optional): _description_. Defaults to None.
|
69
|
+
"""
|
70
|
+
self._execute_node(node=node, map_variable=map_variable)
|
71
|
+
|
72
|
+
def run_click_command(self, command: str) -> str:
|
73
|
+
"""
|
74
|
+
Execute a Click-based CLI command in the current virtual environment.
|
75
|
+
|
76
|
+
Args:
|
77
|
+
args: List of Click command arguments (including subcommands and options)
|
78
|
+
|
79
|
+
Returns:
|
80
|
+
Combined stdout/stderr output as string
|
81
|
+
"""
|
82
|
+
# For Click commands installed via setup.py entry_points
|
83
|
+
# command = [sys.executable, '-m', 'your_package.cli'] + args
|
84
|
+
|
85
|
+
# For direct module execution
|
86
|
+
sub_command = [sys.executable, "-m", "runnable.cli"] + shlex.split(command)[1:]
|
87
|
+
|
88
|
+
process = subprocess.Popen(
|
89
|
+
sub_command,
|
90
|
+
stdout=subprocess.PIPE,
|
91
|
+
stderr=subprocess.STDOUT,
|
92
|
+
universal_newlines=True,
|
93
|
+
bufsize=1,
|
94
|
+
)
|
95
|
+
|
96
|
+
output = []
|
97
|
+
try:
|
98
|
+
while True:
|
99
|
+
line = process.stdout.readline() # type: ignore
|
100
|
+
if not line and process.poll() is not None:
|
101
|
+
break
|
102
|
+
print(line, end="")
|
103
|
+
output.append(line)
|
104
|
+
finally:
|
105
|
+
process.stdout.close() # type: ignore
|
106
|
+
|
107
|
+
if process.returncode != 0:
|
108
|
+
raise subprocess.CalledProcessError(
|
109
|
+
process.returncode, command, "".join(output)
|
110
|
+
)
|
111
|
+
|
112
|
+
return "".join(output)
|
@@ -4,7 +4,7 @@ from pydantic import Field, PrivateAttr
|
|
4
4
|
|
5
5
|
from extensions.pipeline_executor import GenericPipelineExecutor
|
6
6
|
from runnable import defaults
|
7
|
-
from runnable.defaults import
|
7
|
+
from runnable.defaults import MapVariableType
|
8
8
|
from runnable.nodes import BaseNode
|
9
9
|
|
10
10
|
logger = logging.getLogger(defaults.LOGGER_NAME)
|
@@ -32,14 +32,14 @@ class LocalExecutor(GenericPipelineExecutor):
|
|
32
32
|
|
33
33
|
_is_local: bool = PrivateAttr(default=True)
|
34
34
|
|
35
|
-
def execute_from_graph(self, node: BaseNode, map_variable:
|
35
|
+
def execute_from_graph(self, node: BaseNode, map_variable: MapVariableType = None):
|
36
36
|
if not self.object_serialisation:
|
37
37
|
self._context.object_serialisation = False
|
38
38
|
|
39
39
|
super().execute_from_graph(node=node, map_variable=map_variable)
|
40
40
|
|
41
41
|
def trigger_node_execution(
|
42
|
-
self, node: BaseNode, map_variable:
|
42
|
+
self, node: BaseNode, map_variable: MapVariableType = None
|
43
43
|
):
|
44
44
|
"""
|
45
45
|
In this mode of execution, we prepare for the node execution and execute the node
|
@@ -50,7 +50,7 @@ class LocalExecutor(GenericPipelineExecutor):
|
|
50
50
|
"""
|
51
51
|
self.execute_node(node=node, map_variable=map_variable)
|
52
52
|
|
53
|
-
def execute_node(self, node: BaseNode, map_variable:
|
53
|
+
def execute_node(self, node: BaseNode, map_variable: MapVariableType = None):
|
54
54
|
"""
|
55
55
|
For local execution, we just execute the node.
|
56
56
|
|