runnable 0.19.1__py3-none-any.whl → 0.20.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- extensions/pipeline_executor/argo.py +691 -1077
- extensions/pipeline_executor/local_container.py +0 -33
- runnable/cli.py +11 -2
- runnable/entrypoints.py +6 -2
- runnable/sdk.py +5 -0
- runnable/tasks.py +14 -39
- runnable/utils.py +5 -5
- {runnable-0.19.1.dist-info → runnable-0.20.0.dist-info}/METADATA +1 -1
- {runnable-0.19.1.dist-info → runnable-0.20.0.dist-info}/RECORD +12 -13
- extensions/pipeline_executor/argo_specification.yaml +0 -51
- {runnable-0.19.1.dist-info → runnable-0.20.0.dist-info}/WHEEL +0 -0
- {runnable-0.19.1.dist-info → runnable-0.20.0.dist-info}/entry_points.txt +0 -0
- {runnable-0.19.1.dist-info → runnable-0.20.0.dist-info}/licenses/LICENSE +0 -0
@@ -155,39 +155,6 @@ class LocalContainerExecutor(GenericPipelineExecutor):
|
|
155
155
|
)
|
156
156
|
self.trigger_node_execution(node=node, map_variable=map_variable, **kwargs)
|
157
157
|
|
158
|
-
# def execute_job(self, node: TaskNode):
|
159
|
-
# """
|
160
|
-
# Set up the step log and call the execute node
|
161
|
-
|
162
|
-
# Args:
|
163
|
-
# node (BaseNode): _description_
|
164
|
-
# """
|
165
|
-
|
166
|
-
# step_log = self._context.run_log_store.create_step_log(
|
167
|
-
# node.name, node._get_step_log_name(map_variable=None)
|
168
|
-
# )
|
169
|
-
|
170
|
-
# self.add_code_identities(node=node, step_log=step_log)
|
171
|
-
|
172
|
-
# step_log.step_type = node.node_type
|
173
|
-
# step_log.status = defaults.PROCESSING
|
174
|
-
# self._context.run_log_store.add_step_log(step_log, self._context.run_id)
|
175
|
-
|
176
|
-
# command = utils.get_job_execution_command(node)
|
177
|
-
# self._spin_container(node=node, command=command)
|
178
|
-
|
179
|
-
# # Check the step log status and warn if necessary. Docker errors are generally suppressed.
|
180
|
-
# step_log = self._context.run_log_store.get_step_log(
|
181
|
-
# node._get_step_log_name(map_variable=None), self._context.run_id
|
182
|
-
# )
|
183
|
-
# if step_log.status != defaults.SUCCESS:
|
184
|
-
# msg = (
|
185
|
-
# "Node execution inside the container failed. Please check the logs.\n"
|
186
|
-
# "Note: If you do not see any docker issue from your side and the code works properly on local execution"
|
187
|
-
# "please raise a bug report."
|
188
|
-
# )
|
189
|
-
# logger.warning(msg)
|
190
|
-
|
191
158
|
def trigger_node_execution(
|
192
159
|
self, node: BaseNode, map_variable: TypeMapVariable = None, **kwargs
|
193
160
|
):
|
runnable/cli.py
CHANGED
@@ -164,7 +164,7 @@ def fan(
|
|
164
164
|
python_or_yaml_file: Annotated[
|
165
165
|
str, typer.Argument(help="The pipeline definition file")
|
166
166
|
],
|
167
|
-
|
167
|
+
in_or_out: Annotated[str, typer.Argument(help="The fan mode")],
|
168
168
|
map_variable: Annotated[
|
169
169
|
str,
|
170
170
|
typer.Option(
|
@@ -172,7 +172,7 @@ def fan(
|
|
172
172
|
help="The map variable dictionary in str",
|
173
173
|
show_default=True,
|
174
174
|
),
|
175
|
-
],
|
175
|
+
] = "",
|
176
176
|
config_file: Annotated[
|
177
177
|
str,
|
178
178
|
typer.Option(
|
@@ -197,6 +197,14 @@ def fan(
|
|
197
197
|
),
|
198
198
|
] = LogLevel.INFO,
|
199
199
|
tag: Annotated[str, typer.Option(help="A tag attached to the run")] = "",
|
200
|
+
mode: Annotated[
|
201
|
+
ExecutionMode,
|
202
|
+
typer.Option(
|
203
|
+
"--mode",
|
204
|
+
"-m",
|
205
|
+
help="spec in yaml or python sdk",
|
206
|
+
),
|
207
|
+
] = ExecutionMode.YAML,
|
200
208
|
):
|
201
209
|
logger.setLevel(log_level.value)
|
202
210
|
|
@@ -206,6 +214,7 @@ def fan(
|
|
206
214
|
pipeline_file=python_or_yaml_file,
|
207
215
|
step_name=step_name,
|
208
216
|
mode=mode,
|
217
|
+
in_or_out=in_or_out,
|
209
218
|
map_variable=map_variable,
|
210
219
|
run_id=run_id,
|
211
220
|
tag=tag,
|
runnable/entrypoints.py
CHANGED
@@ -332,6 +332,7 @@ def execute_single_node(
|
|
332
332
|
run_id=run_id, configuration_file=configuration_file, tag=tag
|
333
333
|
)
|
334
334
|
|
335
|
+
# TODO: Is it useful to make it get from an environment variable
|
335
336
|
map_variable_dict = utils.json_to_ordered_dict(map_variable)
|
336
337
|
|
337
338
|
step_internal_name = nodes.BaseNode._get_internal_name_from_command_name(step_name)
|
@@ -354,6 +355,8 @@ def execute_single_node(
|
|
354
355
|
run_context.catalog_handler.put(name=log_file_name, run_id=run_context.run_id)
|
355
356
|
os.remove(log_file_name)
|
356
357
|
|
358
|
+
executor.send_return_code()
|
359
|
+
|
357
360
|
|
358
361
|
def execute_job_yaml_spec(
|
359
362
|
job_definition_file: str,
|
@@ -492,6 +495,7 @@ def fan(
|
|
492
495
|
pipeline_file: str,
|
493
496
|
step_name: str,
|
494
497
|
mode: str,
|
498
|
+
in_or_out: str,
|
495
499
|
map_variable: str,
|
496
500
|
run_id: str,
|
497
501
|
tag: str = "",
|
@@ -551,10 +555,10 @@ def fan(
|
|
551
555
|
|
552
556
|
map_variable_dict = utils.json_to_ordered_dict(map_variable)
|
553
557
|
|
554
|
-
if
|
558
|
+
if in_or_out == "in":
|
555
559
|
logger.info("Fanning in for : %s", node_to_execute)
|
556
560
|
executor.fan_in(node=node_to_execute, map_variable=map_variable_dict)
|
557
|
-
elif
|
561
|
+
elif in_or_out == "out":
|
558
562
|
logger.info("Fanning out for : %s", node_to_execute)
|
559
563
|
executor.fan_out(node=node_to_execute, map_variable=map_variable_dict)
|
560
564
|
else:
|
runnable/sdk.py
CHANGED
@@ -419,6 +419,11 @@ class ShellTask(BaseTask):
|
|
419
419
|
def command_type(self) -> str:
|
420
420
|
return "shell"
|
421
421
|
|
422
|
+
def create_job(self) -> RunnableTask:
|
423
|
+
self.terminate_with_success = True
|
424
|
+
node = self.create_node()
|
425
|
+
return node.executable
|
426
|
+
|
422
427
|
|
423
428
|
class Stub(BaseTraversal):
|
424
429
|
"""
|
runnable/tasks.py
CHANGED
@@ -11,7 +11,7 @@ from datetime import datetime
|
|
11
11
|
from pathlib import Path
|
12
12
|
from pickle import PicklingError
|
13
13
|
from string import Template
|
14
|
-
from typing import Any, Dict, List, Literal
|
14
|
+
from typing import Any, Dict, List, Literal
|
15
15
|
|
16
16
|
from pydantic import BaseModel, ConfigDict, Field, field_validator
|
17
17
|
from stevedore import driver
|
@@ -59,20 +59,6 @@ class BaseTaskType(BaseModel):
|
|
59
59
|
def _context(self):
|
60
60
|
return context.run_context
|
61
61
|
|
62
|
-
def get_cli_options(self) -> Tuple[str, dict]:
|
63
|
-
"""
|
64
|
-
Key is the name of the cli option and value is the value of the cli option.
|
65
|
-
This should always be in sync with the cli options defined in execute_*.
|
66
|
-
|
67
|
-
Returns:
|
68
|
-
str: The name of the cli option.
|
69
|
-
dict: The dict of cli options for the task.
|
70
|
-
|
71
|
-
Raises:
|
72
|
-
NotImplementedError: Base class, not implemented
|
73
|
-
"""
|
74
|
-
raise NotImplementedError()
|
75
|
-
|
76
62
|
def set_secrets_as_env_variables(self):
|
77
63
|
# Preparing the environment for the task execution
|
78
64
|
for key in self.secrets:
|
@@ -262,14 +248,6 @@ class PythonTaskType(BaseTaskType): # pylint: disable=too-few-public-methods
|
|
262
248
|
task_type: str = Field(default="python", serialization_alias="command_type")
|
263
249
|
command: str
|
264
250
|
|
265
|
-
def get_cli_options(self) -> Tuple[str, dict]:
|
266
|
-
"""Return the cli options for the task.
|
267
|
-
|
268
|
-
Returns:
|
269
|
-
dict: The cli options for the task
|
270
|
-
"""
|
271
|
-
return "function", {"command": self.command}
|
272
|
-
|
273
251
|
def execute_command(
|
274
252
|
self,
|
275
253
|
map_variable: TypeMapVariable = None,
|
@@ -425,26 +403,22 @@ class NotebookTaskType(BaseTaskType):
|
|
425
403
|
|
426
404
|
return command
|
427
405
|
|
428
|
-
|
429
|
-
|
430
|
-
|
406
|
+
def get_notebook_output_path(self, map_variable: TypeMapVariable = None) -> str:
|
407
|
+
tag = ""
|
408
|
+
map_variable = map_variable or {}
|
409
|
+
for key, value in map_variable.items():
|
410
|
+
tag += f"{key}_{value}_"
|
411
|
+
|
431
412
|
if self._context.executor._context_node:
|
432
|
-
|
433
|
-
|
434
|
-
|
435
|
-
sane_name = ""
|
413
|
+
tag += self._context.executor._context_node.name
|
414
|
+
|
415
|
+
tag = "".join(x for x in tag if x.isalnum())
|
436
416
|
|
437
417
|
output_path = Path(".", self.command)
|
438
|
-
file_name = output_path.parent / (output_path.stem + f"{
|
418
|
+
file_name = output_path.parent / (output_path.stem + f"-{tag}_out.ipynb")
|
439
419
|
|
440
420
|
return str(file_name)
|
441
421
|
|
442
|
-
def get_cli_options(self) -> Tuple[str, dict]:
|
443
|
-
return "notebook", {
|
444
|
-
"command": self.command,
|
445
|
-
"notebook-output-path": self.notebook_output_path,
|
446
|
-
}
|
447
|
-
|
448
422
|
def execute_command(
|
449
423
|
self,
|
450
424
|
map_variable: TypeMapVariable = None,
|
@@ -464,7 +438,9 @@ class NotebookTaskType(BaseTaskType):
|
|
464
438
|
import ploomber_engine as pm
|
465
439
|
from ploomber_engine.ipython import PloomberClient
|
466
440
|
|
467
|
-
notebook_output_path = self.
|
441
|
+
notebook_output_path = self.get_notebook_output_path(
|
442
|
+
map_variable=map_variable
|
443
|
+
)
|
468
444
|
|
469
445
|
with (
|
470
446
|
self.execution_context(
|
@@ -476,7 +452,6 @@ class NotebookTaskType(BaseTaskType):
|
|
476
452
|
|
477
453
|
if map_variable:
|
478
454
|
for key, value in map_variable.items():
|
479
|
-
notebook_output_path += "_" + str(value)
|
480
455
|
copy_params[key] = JsonParameter(kind="json", value=value)
|
481
456
|
|
482
457
|
# Remove any {v}_unreduced parameters from the parameters
|
runnable/utils.py
CHANGED
@@ -455,9 +455,9 @@ def get_fan_command(
|
|
455
455
|
log_level = log_level or logging.getLevelName(logger.getEffectiveLevel())
|
456
456
|
action = (
|
457
457
|
f"runnable fan {run_id} "
|
458
|
-
f"{node._command_friendly_name()} "
|
459
|
-
f"{context.run_context.pipeline_file} "
|
460
|
-
f"
|
458
|
+
f"{node._command_friendly_name()} " # step name
|
459
|
+
f"{context.run_context.pipeline_file} " # yaml or python
|
460
|
+
f"{mode} " # in or out
|
461
461
|
f"--log-level {log_level} "
|
462
462
|
)
|
463
463
|
if context.run_context.configuration_file:
|
@@ -469,8 +469,8 @@ def get_fan_command(
|
|
469
469
|
if map_variable:
|
470
470
|
action = action + f" --map-variable '{json.dumps(map_variable)}'"
|
471
471
|
|
472
|
-
if context.run_context.
|
473
|
-
action = action +
|
472
|
+
if context.run_context.from_sdk: # execution mode
|
473
|
+
action = action + " --mode python "
|
474
474
|
|
475
475
|
return action
|
476
476
|
|
@@ -15,10 +15,9 @@ extensions/nodes/nodes.py,sha256=ib68QE737ihGLIVp3V2wea13u7lmMZdRvK80bgUkRtA,346
|
|
15
15
|
extensions/nodes/pyproject.toml,sha256=YTu-ETN3JNFSkMzzWeOwn4m-O2nbRH-PmiPBALDCUw4,278
|
16
16
|
extensions/pipeline_executor/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
17
17
|
extensions/pipeline_executor/__init__.py,sha256=YnKILiy-SxfnG3rYUoinjh1lfkuAF5QXpPePtn6VxBY,25174
|
18
|
-
extensions/pipeline_executor/argo.py,sha256=
|
19
|
-
extensions/pipeline_executor/argo_specification.yaml,sha256=wXQcm2gOQYqy-IOQIhucohS32ZrHKCfGA5zZ0RraPYc,1276
|
18
|
+
extensions/pipeline_executor/argo.py,sha256=zA-9zeo93Rvyn-7--bLNwRybREUrtmFRC0QL8oXwNio,32154
|
20
19
|
extensions/pipeline_executor/local.py,sha256=H8s6AdML_9_f-vdGG_6k0y9FbLqAqvA1S_7xMNyARzY,1946
|
21
|
-
extensions/pipeline_executor/local_container.py,sha256=
|
20
|
+
extensions/pipeline_executor/local_container.py,sha256=HOT9I-cPDCvgy6_bzNEtl4jPhTyeYSn1GK7lplH3vDA,12515
|
22
21
|
extensions/pipeline_executor/mocked.py,sha256=SuObJ6Myt7p8duW8sylIp1cYIAnFutsJW1avWaOUY3c,5798
|
23
22
|
extensions/pipeline_executor/pyproject.toml,sha256=ykTX7srR10PBYb8LsIwEj8vIPPIEZQ5V_R7VYbZ-ido,291
|
24
23
|
extensions/pipeline_executor/retry.py,sha256=KGenhWrLLmOQgzMvqloXHDRJyoNs91t05rRW8aLW6FA,6969
|
@@ -35,11 +34,11 @@ extensions/secrets/dotenv.py,sha256=FbYYd_pVuJuVuIDIvXbzKuSSQ9GPq7xJXTDbJMTQbhM,
|
|
35
34
|
extensions/secrets/pyproject.toml,sha256=mLJNImNcBlbLKHh-0ugVWT9V83R4RibyyYDtBCSqVF4,282
|
36
35
|
runnable/__init__.py,sha256=fYkOrbsb-E1rGkrof7kOJ3KboTFH-HriGa-8npn4-50,625
|
37
36
|
runnable/catalog.py,sha256=b9N40kTv1IBidzlWjkHcBGyYhq6qIDHZfBuFenzjsMI,4924
|
38
|
-
runnable/cli.py,sha256=
|
37
|
+
runnable/cli.py,sha256=xHb2VSC16z9IT45SnhUYHuXUrzPqF_pK-jyWiTnz5sM,8670
|
39
38
|
runnable/context.py,sha256=by5uepmuCP0dmM9BmsliXihSes5QEFejwAsmekcqylE,1388
|
40
39
|
runnable/datastore.py,sha256=9y5enzn6AXLHLdwvgkdjGPrBkVlrcjfbaAHsst-lJzg,32466
|
41
40
|
runnable/defaults.py,sha256=3o9IVGryyCE6PoQTOoaIaHHTbJGEzmdXMcwzOhwAYoI,3518
|
42
|
-
runnable/entrypoints.py,sha256=
|
41
|
+
runnable/entrypoints.py,sha256=DxboaCJ4ADZjIHl_oSPsC74ChUCKYic3i8dQqd6U43w,19009
|
43
42
|
runnable/exceptions.py,sha256=LFbp0-Qxg2PAMLEVt7w2whhBxSG-5pzUEv5qN-Rc4_c,3003
|
44
43
|
runnable/executor.py,sha256=ZPpfKwjDJnta03M2cWIINXcwke2ZDVc_QrIw7kwpHDQ,15547
|
45
44
|
runnable/graph.py,sha256=jVjikRLR-so3b2ufmNKpEQ_Ny68qN4bcGDAdXBRKiCY,16574
|
@@ -47,12 +46,12 @@ runnable/names.py,sha256=vn92Kv9ANROYSZX6Z4z1v_WA3WiEdIYmG6KEStBFZug,8134
|
|
47
46
|
runnable/nodes.py,sha256=YU9u7r1ESzui1uVtJ1dgwdv1ozyJnF2k-MCFieT8CLI,17519
|
48
47
|
runnable/parameters.py,sha256=LyQb1d0SaFeI4PJ_yDYt9wArm9ThSPASWb36TwIdDUs,5213
|
49
48
|
runnable/pickler.py,sha256=ydJ_eti_U1F4l-YacFp7BWm6g5vTn04UXye25S1HVok,2684
|
50
|
-
runnable/sdk.py,sha256=
|
49
|
+
runnable/sdk.py,sha256=thffRpZgkz1XW4DUY6EIJPyF8NLaTE1puh43dZKW9MU,31595
|
51
50
|
runnable/secrets.py,sha256=PXcEJw-4WPzeWRLfsatcPPyr1zkqgHzdRWRcS9vvpvM,2354
|
52
|
-
runnable/tasks.py,sha256=
|
53
|
-
runnable/utils.py,sha256=
|
54
|
-
runnable-0.
|
55
|
-
runnable-0.
|
56
|
-
runnable-0.
|
57
|
-
runnable-0.
|
58
|
-
runnable-0.
|
51
|
+
runnable/tasks.py,sha256=WdXFUL9mT3zmUc5YoOrTmwfjss02MwNKanf11pn8py4,28497
|
52
|
+
runnable/utils.py,sha256=h-E6ZmUDcKwrJLjKlu0XXAK4uzfnD0K0EEPpZmeSIfM,19542
|
53
|
+
runnable-0.20.0.dist-info/METADATA,sha256=njNTjovB_HmjC52tNvGJkj5Ae0aZV4nLeNdjwynXnKE,9945
|
54
|
+
runnable-0.20.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
55
|
+
runnable-0.20.0.dist-info/entry_points.txt,sha256=seek5WVGvwYALm8lZ0TfPXwG5NaCeUKjU8urF8k3gvY,1621
|
56
|
+
runnable-0.20.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
57
|
+
runnable-0.20.0.dist-info/RECORD,,
|
@@ -1,51 +0,0 @@
|
|
1
|
-
apiVersion: argoproj.io/v1alpha1
|
2
|
-
kind: Workflow
|
3
|
-
metadata:
|
4
|
-
generateName: runnable-dag
|
5
|
-
spec:
|
6
|
-
activeDeadlineSeconds: int # max run time of the workflow
|
7
|
-
entrypoint: str
|
8
|
-
nodeSelector: Dict[str, str] # global node selector
|
9
|
-
parallelism: # global level
|
10
|
-
podGC: OnPodCompletion
|
11
|
-
resources: # Should be converted to podSpecPath
|
12
|
-
limits:
|
13
|
-
requests:
|
14
|
-
podSpecPatch: json str representation of resources for defaults
|
15
|
-
retryStrategy: # global level for all templates
|
16
|
-
limit: int
|
17
|
-
retryPolicy: # global level for all templates
|
18
|
-
backoff:
|
19
|
-
duration: str
|
20
|
-
factor: int
|
21
|
-
maxDuration: str
|
22
|
-
serviceAccountName: str # Optionally required
|
23
|
-
templateDefaults:
|
24
|
-
activeDeadlineSeconds: int, for a template
|
25
|
-
timeout: str # max time including the wait time
|
26
|
-
failFast: true
|
27
|
-
volumes:
|
28
|
-
templates:
|
29
|
-
activeDeadlineSeconds: # override
|
30
|
-
nodeSelector: # override
|
31
|
-
retryStrategy: # override
|
32
|
-
tolerations: # override
|
33
|
-
container:
|
34
|
-
command:
|
35
|
-
env:
|
36
|
-
image:
|
37
|
-
imagePullPolicy:
|
38
|
-
volumeMounts:
|
39
|
-
resources:
|
40
|
-
limits:
|
41
|
-
requests:
|
42
|
-
dag:
|
43
|
-
tasks:
|
44
|
-
depends:
|
45
|
-
continueOn:
|
46
|
-
tolerations: # global level for all templates
|
47
|
-
effect: str
|
48
|
-
key: str
|
49
|
-
operator: str
|
50
|
-
value: str
|
51
|
-
volumes:
|
File without changes
|
File without changes
|
File without changes
|