ddeutil-workflow 0.0.52__py3-none-any.whl → 0.0.54__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ddeutil/workflow/__about__.py +1 -1
- ddeutil/workflow/api/api.py +1 -1
- ddeutil/workflow/api/routes/job.py +2 -2
- ddeutil/workflow/job.py +112 -160
- ddeutil/workflow/stages.py +664 -388
- ddeutil/workflow/utils.py +5 -4
- ddeutil/workflow/workflow.py +105 -230
- {ddeutil_workflow-0.0.52.dist-info → ddeutil_workflow-0.0.54.dist-info}/METADATA +1 -7
- {ddeutil_workflow-0.0.52.dist-info → ddeutil_workflow-0.0.54.dist-info}/RECORD +14 -14
- /ddeutil/workflow/api/{log.py → logs.py} +0 -0
- /ddeutil/workflow/api/{repeat.py → utils.py} +0 -0
- {ddeutil_workflow-0.0.52.dist-info → ddeutil_workflow-0.0.54.dist-info}/WHEEL +0 -0
- {ddeutil_workflow-0.0.52.dist-info → ddeutil_workflow-0.0.54.dist-info}/licenses/LICENSE +0 -0
- {ddeutil_workflow-0.0.52.dist-info → ddeutil_workflow-0.0.54.dist-info}/top_level.txt +0 -0
ddeutil/workflow/stages.py
CHANGED
@@ -4,23 +4,27 @@
|
|
4
4
|
# license information.
|
5
5
|
# ------------------------------------------------------------------------------
|
6
6
|
# [x] Use dynamic config
|
7
|
-
"""
|
8
|
-
|
9
|
-
(same thread at its job owner) that mean it is the lowest executor
|
10
|
-
|
7
|
+
"""Stages module include all stage model that use be the minimum execution layer
|
8
|
+
of this workflow engine. The stage handle the minimize task that run in some
|
9
|
+
thread (same thread at its job owner) that mean it is the lowest executor that
|
10
|
+
you can track logs.
|
11
11
|
|
12
|
-
The output of stage execution only return
|
13
|
-
handle stage error on this stage
|
14
|
-
use-case, and it does not worry
|
12
|
+
The output of stage execution only return SUCCESS or CANCEL status because
|
13
|
+
I do not want to handle stage error on this stage execution. I think stage model
|
14
|
+
have a lot of use-case, and it should does not worry about it error output.
|
15
15
|
|
16
|
-
|
16
|
+
So, I will create `handler_execute` for any exception class that raise from
|
17
|
+
the stage execution method.
|
17
18
|
|
18
|
-
|
19
|
-
|
19
|
+
Execution --> Ok ---( handler )--> Result with `SUCCESS` or `CANCEL`
|
20
|
+
|
21
|
+
--> Error ┬--( handler )-> Result with `FAILED` (Set `raise_error` flag)
|
22
|
+
|
|
23
|
+
╰--( handler )-> Raise StageException(...)
|
20
24
|
|
21
25
|
On the context I/O that pass to a stage object at execute process. The
|
22
|
-
execute method receives a `params={"params": {...}}` value for
|
23
|
-
|
26
|
+
execute method receives a `params={"params": {...}}` value for passing template
|
27
|
+
searching.
|
24
28
|
"""
|
25
29
|
from __future__ import annotations
|
26
30
|
|
@@ -28,12 +32,13 @@ import asyncio
|
|
28
32
|
import contextlib
|
29
33
|
import copy
|
30
34
|
import inspect
|
35
|
+
import json
|
31
36
|
import subprocess
|
32
37
|
import sys
|
33
38
|
import time
|
34
39
|
import uuid
|
35
40
|
from abc import ABC, abstractmethod
|
36
|
-
from collections.abc import Iterator
|
41
|
+
from collections.abc import AsyncIterator, Iterator
|
37
42
|
from concurrent.futures import (
|
38
43
|
FIRST_EXCEPTION,
|
39
44
|
Future,
|
@@ -42,7 +47,7 @@ from concurrent.futures import (
|
|
42
47
|
wait,
|
43
48
|
)
|
44
49
|
from datetime import datetime
|
45
|
-
from inspect import Parameter
|
50
|
+
from inspect import Parameter, isclass, isfunction, ismodule
|
46
51
|
from pathlib import Path
|
47
52
|
from subprocess import CompletedProcess
|
48
53
|
from textwrap import dedent
|
@@ -59,6 +64,7 @@ from .exceptions import StageException, UtilException, to_dict
|
|
59
64
|
from .result import CANCEL, FAILED, SUCCESS, WAIT, Result, Status
|
60
65
|
from .reusables import TagFunc, extract_call, not_in_template, param2template
|
61
66
|
from .utils import (
|
67
|
+
NEWLINE,
|
62
68
|
delay,
|
63
69
|
filter_func,
|
64
70
|
gen_id,
|
@@ -66,20 +72,22 @@ from .utils import (
|
|
66
72
|
)
|
67
73
|
|
68
74
|
T = TypeVar("T")
|
75
|
+
StrOrInt = Union[str, int]
|
69
76
|
|
70
77
|
|
71
78
|
class BaseStage(BaseModel, ABC):
|
72
|
-
"""Base Stage Model that keep only
|
73
|
-
metadata. If you want to implement any custom
|
74
|
-
|
79
|
+
"""Base Stage Model that keep only necessary fields like `id`, `name` or
|
80
|
+
`condition` for the stage metadata. If you want to implement any custom
|
81
|
+
stage, you can inherit this class and implement `self.execute()` method
|
82
|
+
only.
|
75
83
|
|
76
|
-
This class is the abstraction class for any stage model that
|
77
|
-
|
84
|
+
This class is the abstraction class for any inherit stage model that
|
85
|
+
want to implement on this workflow package.
|
78
86
|
"""
|
79
87
|
|
80
88
|
extras: DictData = Field(
|
81
89
|
default_factory=dict,
|
82
|
-
description="An extra override config values.",
|
90
|
+
description="An extra parameter that override core config values.",
|
83
91
|
)
|
84
92
|
id: Optional[str] = Field(
|
85
93
|
default=None,
|
@@ -93,14 +101,17 @@ class BaseStage(BaseModel, ABC):
|
|
93
101
|
)
|
94
102
|
condition: Optional[str] = Field(
|
95
103
|
default=None,
|
96
|
-
description=
|
104
|
+
description=(
|
105
|
+
"A stage condition statement to allow stage executable. This field "
|
106
|
+
"alise with `if` field."
|
107
|
+
),
|
97
108
|
alias="if",
|
98
109
|
)
|
99
110
|
|
100
111
|
@property
|
101
112
|
def iden(self) -> str:
|
102
|
-
"""Return
|
103
|
-
|
113
|
+
"""Return this stage identity that return the `id` field first and if
|
114
|
+
this `id` field does not set, it will use the `name` field instead.
|
104
115
|
|
105
116
|
:rtype: str
|
106
117
|
"""
|
@@ -156,8 +167,8 @@ class BaseStage(BaseModel, ABC):
|
|
156
167
|
run_id: str | None = None,
|
157
168
|
parent_run_id: str | None = None,
|
158
169
|
result: Result | None = None,
|
159
|
-
raise_error: bool | None = None,
|
160
170
|
event: Event | None = None,
|
171
|
+
raise_error: bool | None = None,
|
161
172
|
) -> Result | DictData:
|
162
173
|
"""Handler stage execution result from the stage `execute` method.
|
163
174
|
|
@@ -170,7 +181,7 @@ class BaseStage(BaseModel, ABC):
|
|
170
181
|
╰-context:
|
171
182
|
╰-outputs: ...
|
172
183
|
|
173
|
-
--> Error --> Result (if
|
184
|
+
--> Error --> Result (if `raise_error` was set)
|
174
185
|
|-status: FAILED
|
175
186
|
╰-errors:
|
176
187
|
|-class: ...
|
@@ -179,18 +190,17 @@ class BaseStage(BaseModel, ABC):
|
|
179
190
|
|
180
191
|
--> Error --> Raise StageException(...)
|
181
192
|
|
182
|
-
On the last step, it will set the running ID on a return result
|
183
|
-
from current stage ID before release the final result.
|
193
|
+
On the last step, it will set the running ID on a return result
|
194
|
+
object from the current stage ID before release the final result.
|
184
195
|
|
185
|
-
:param params: (DictData) A
|
186
|
-
stage execution.
|
196
|
+
:param params: (DictData) A parameter data.
|
187
197
|
:param run_id: (str) A running stage ID for this execution.
|
188
198
|
:param parent_run_id: (str) A parent workflow running ID for this
|
189
199
|
execution.
|
190
200
|
:param result: (Result) A result object for keeping context and status
|
191
201
|
data before execution.
|
192
|
-
:param raise_error: (bool) A flag that all this method raise error
|
193
202
|
:param event: (Event) An event manager that pass to the stage execution.
|
203
|
+
:param raise_error: (bool) A flag that all this method raise error
|
194
204
|
|
195
205
|
:rtype: Result
|
196
206
|
"""
|
@@ -206,8 +216,9 @@ class BaseStage(BaseModel, ABC):
|
|
206
216
|
rs: Result = self.execute(params, result=result, event=event)
|
207
217
|
return rs
|
208
218
|
except Exception as e:
|
209
|
-
result.trace.error(
|
210
|
-
|
219
|
+
result.trace.error(
|
220
|
+
f"[STAGE]: Handler:{NEWLINE}{e.__class__.__name__}: {e}"
|
221
|
+
)
|
211
222
|
if dynamic("stage_raise_error", f=raise_error, extras=self.extras):
|
212
223
|
if isinstance(e, StageException):
|
213
224
|
raise
|
@@ -217,37 +228,39 @@ class BaseStage(BaseModel, ABC):
|
|
217
228
|
f"{e.__class__.__name__}: {e}"
|
218
229
|
) from e
|
219
230
|
|
220
|
-
|
221
|
-
return result.catch(status=FAILED, context=errors)
|
231
|
+
return result.catch(status=FAILED, context={"errors": to_dict(e)})
|
222
232
|
|
223
233
|
def set_outputs(self, output: DictData, to: DictData) -> DictData:
|
224
|
-
"""Set an outputs from execution
|
225
|
-
|
234
|
+
"""Set an outputs from execution result context to the received context
|
235
|
+
with a `to` input parameter. The result context from stage execution
|
236
|
+
will be set with `outputs` key in this stage ID key.
|
226
237
|
|
227
238
|
For example of setting output method, If you receive execute output
|
228
239
|
and want to set on the `to` like;
|
229
240
|
|
230
|
-
... (i) output: {'foo': bar}
|
241
|
+
... (i) output: {'foo': 'bar', 'skipped': True}
|
231
242
|
... (ii) to: {'stages': {}}
|
232
243
|
|
233
|
-
The
|
244
|
+
The received context in the `to` argument will be;
|
234
245
|
|
235
246
|
... (iii) to: {
|
236
247
|
'stages': {
|
237
248
|
'<stage-id>': {
|
238
249
|
'outputs': {'foo': 'bar'},
|
239
|
-
'skipped':
|
250
|
+
'skipped': True,
|
240
251
|
}
|
241
252
|
}
|
242
253
|
}
|
243
254
|
|
244
255
|
Important:
|
256
|
+
|
245
257
|
This method is use for reconstruct the result context and transfer
|
246
|
-
to the `to` argument.
|
258
|
+
to the `to` argument. The result context was soft copied before set
|
259
|
+
output step.
|
247
260
|
|
248
|
-
:param output: (DictData)
|
249
|
-
|
250
|
-
:param to: (DictData) A context data
|
261
|
+
:param output: (DictData) A result data context that want to extract
|
262
|
+
and transfer to the `outputs` key in receive context.
|
263
|
+
:param to: (DictData) A received context data.
|
251
264
|
|
252
265
|
:rtype: DictData
|
253
266
|
"""
|
@@ -266,7 +279,7 @@ class BaseStage(BaseModel, ABC):
|
|
266
279
|
param2template(self.name, params=to, extras=self.extras)
|
267
280
|
)
|
268
281
|
)
|
269
|
-
|
282
|
+
output: DictData = output.copy()
|
270
283
|
errors: DictData = (
|
271
284
|
{"errors": output.pop("errors", {})} if "errors" in output else {}
|
272
285
|
)
|
@@ -282,11 +295,12 @@ class BaseStage(BaseModel, ABC):
|
|
282
295
|
}
|
283
296
|
return to
|
284
297
|
|
285
|
-
def get_outputs(self,
|
298
|
+
def get_outputs(self, output: DictData) -> DictData:
|
286
299
|
"""Get the outputs from stages data. It will get this stage ID from
|
287
300
|
the stage outputs mapping.
|
288
301
|
|
289
|
-
:param
|
302
|
+
:param output: (DictData) A stage output context that want to get this
|
303
|
+
stage ID `outputs` key.
|
290
304
|
|
291
305
|
:rtype: DictData
|
292
306
|
"""
|
@@ -296,33 +310,31 @@ class BaseStage(BaseModel, ABC):
|
|
296
310
|
return {}
|
297
311
|
|
298
312
|
_id: str = (
|
299
|
-
param2template(self.id, params=
|
313
|
+
param2template(self.id, params=output, extras=self.extras)
|
300
314
|
if self.id
|
301
315
|
else gen_id(
|
302
|
-
param2template(self.name, params=
|
316
|
+
param2template(self.name, params=output, extras=self.extras)
|
303
317
|
)
|
304
318
|
)
|
305
|
-
return
|
319
|
+
return output.get("stages", {}).get(_id, {}).get("outputs", {})
|
306
320
|
|
307
|
-
def is_skipped(self, params: DictData
|
321
|
+
def is_skipped(self, params: DictData) -> bool:
|
308
322
|
"""Return true if condition of this stage do not correct. This process
|
309
323
|
use build-in eval function to execute the if-condition.
|
310
324
|
|
325
|
+
:param params: (DictData) A parameters that want to pass to condition
|
326
|
+
template.
|
327
|
+
|
311
328
|
:raise StageException: When it has any error raise from the eval
|
312
329
|
condition statement.
|
313
330
|
:raise StageException: When return type of the eval condition statement
|
314
331
|
does not return with boolean type.
|
315
332
|
|
316
|
-
:param params: (DictData) A parameters that want to pass to condition
|
317
|
-
template.
|
318
|
-
|
319
333
|
:rtype: bool
|
320
334
|
"""
|
321
335
|
if self.condition is None:
|
322
336
|
return False
|
323
337
|
|
324
|
-
params: DictData = {} if params is None else params
|
325
|
-
|
326
338
|
try:
|
327
339
|
# WARNING: The eval build-in function is very dangerous. So, it
|
328
340
|
# should use the `re` module to validate eval-string before
|
@@ -340,7 +352,14 @@ class BaseStage(BaseModel, ABC):
|
|
340
352
|
|
341
353
|
|
342
354
|
class BaseAsyncStage(BaseStage):
|
343
|
-
"""Base Async Stage model
|
355
|
+
"""Base Async Stage model to make any stage model allow async execution for
|
356
|
+
optimize CPU and Memory on the current node. If you want to implement any
|
357
|
+
custom async stage, you can inherit this class and implement
|
358
|
+
`self.axecute()` (async + execute = axecute) method only.
|
359
|
+
|
360
|
+
This class is the abstraction class for any inherit asyncable stage
|
361
|
+
model.
|
362
|
+
"""
|
344
363
|
|
345
364
|
@abstractmethod
|
346
365
|
def execute(
|
@@ -385,20 +404,18 @@ class BaseAsyncStage(BaseStage):
|
|
385
404
|
run_id: str | None = None,
|
386
405
|
parent_run_id: str | None = None,
|
387
406
|
result: Result | None = None,
|
388
|
-
raise_error: bool | None = None,
|
389
407
|
event: Event | None = None,
|
408
|
+
raise_error: bool | None = None,
|
390
409
|
) -> Result:
|
391
410
|
"""Async Handler stage execution result from the stage `execute` method.
|
392
411
|
|
393
|
-
:param params: (DictData) A
|
394
|
-
|
395
|
-
:param
|
396
|
-
:param
|
397
|
-
|
398
|
-
|
399
|
-
data before execution.
|
412
|
+
:param params: (DictData) A parameter data.
|
413
|
+
:param run_id: (str) A stage running ID.
|
414
|
+
:param parent_run_id: (str) A parent job running ID.
|
415
|
+
:param result: (Result) A Result instance for return context and status.
|
416
|
+
:param event: (Event) An Event manager instance that use to cancel this
|
417
|
+
execution if it forces stopped by parent execution.
|
400
418
|
:param raise_error: (bool) A flag that all this method raise error
|
401
|
-
:param event: (Event) An event manager that pass to the stage execution.
|
402
419
|
|
403
420
|
:rtype: Result
|
404
421
|
"""
|
@@ -414,7 +431,9 @@ class BaseAsyncStage(BaseStage):
|
|
414
431
|
rs: Result = await self.axecute(params, result=result, event=event)
|
415
432
|
return rs
|
416
433
|
except Exception as e: # pragma: no cov
|
417
|
-
await result.trace.aerror(
|
434
|
+
await result.trace.aerror(
|
435
|
+
f"[STAGE]: Handler {e.__class__.__name__}: {e}"
|
436
|
+
)
|
418
437
|
|
419
438
|
if dynamic("stage_raise_error", f=raise_error, extras=self.extras):
|
420
439
|
if isinstance(e, StageException):
|
@@ -425,13 +444,16 @@ class BaseAsyncStage(BaseStage):
|
|
425
444
|
f"{e.__class__.__name__}: {e}"
|
426
445
|
) from None
|
427
446
|
|
428
|
-
|
429
|
-
return result.catch(status=FAILED, context=errors)
|
447
|
+
return result.catch(status=FAILED, context={"errors": to_dict(e)})
|
430
448
|
|
431
449
|
|
432
450
|
class EmptyStage(BaseAsyncStage):
|
433
|
-
"""Empty stage that do nothing
|
434
|
-
|
451
|
+
"""Empty stage executor that do nothing and log the `message` field to
|
452
|
+
stdout only. It can use for tracking a template parameter on the workflow or
|
453
|
+
debug step.
|
454
|
+
|
455
|
+
You can pass a sleep value in second unit to this stage for waiting
|
456
|
+
after log message.
|
435
457
|
|
436
458
|
Data Validate:
|
437
459
|
>>> stage = {
|
@@ -443,11 +465,14 @@ class EmptyStage(BaseAsyncStage):
|
|
443
465
|
|
444
466
|
echo: Optional[str] = Field(
|
445
467
|
default=None,
|
446
|
-
description="A
|
468
|
+
description="A message that want to show on the stdout.",
|
447
469
|
)
|
448
470
|
sleep: float = Field(
|
449
471
|
default=0,
|
450
|
-
description=
|
472
|
+
description=(
|
473
|
+
"A second value to sleep before start execution. This value should "
|
474
|
+
"gather or equal 0, and less than 1800 seconds."
|
475
|
+
),
|
451
476
|
ge=0,
|
452
477
|
lt=1800,
|
453
478
|
)
|
@@ -460,18 +485,15 @@ class EmptyStage(BaseAsyncStage):
|
|
460
485
|
event: Event | None = None,
|
461
486
|
) -> Result:
|
462
487
|
"""Execution method for the Empty stage that do only logging out to
|
463
|
-
stdout.
|
464
|
-
it does not get any error from logging function.
|
488
|
+
stdout.
|
465
489
|
|
466
490
|
The result context should be empty and do not process anything
|
467
491
|
without calling logging function.
|
468
492
|
|
469
|
-
:param params: (DictData) A
|
470
|
-
|
471
|
-
:param
|
472
|
-
|
473
|
-
:param event: (Event) An event manager that use to track parent execute
|
474
|
-
was not force stopped.
|
493
|
+
:param params: (DictData) A parameter data.
|
494
|
+
:param result: (Result) A Result instance for return context and status.
|
495
|
+
:param event: (Event) An Event manager instance that use to cancel this
|
496
|
+
execution if it forces stopped by parent execution.
|
475
497
|
|
476
498
|
:rtype: Result
|
477
499
|
"""
|
@@ -484,21 +506,18 @@ class EmptyStage(BaseAsyncStage):
|
|
484
506
|
message: str = "..."
|
485
507
|
else:
|
486
508
|
message: str = param2template(
|
487
|
-
dedent(self.echo), params, extras=self.extras
|
509
|
+
dedent(self.echo.strip("\n")), params, extras=self.extras
|
488
510
|
)
|
489
|
-
if "\n" in
|
490
|
-
message: str =
|
491
|
-
"\n"
|
492
|
-
)
|
511
|
+
if "\n" in message:
|
512
|
+
message: str = NEWLINE + message.replace("\n", NEWLINE)
|
493
513
|
|
494
514
|
result.trace.info(
|
495
515
|
f"[STAGE]: Empty-Execute: {self.name!r}: ( {message} )"
|
496
516
|
)
|
497
517
|
if self.sleep > 0:
|
498
518
|
if self.sleep > 5:
|
499
|
-
result.trace.info(f"[STAGE]: ... sleep ({self.sleep}
|
519
|
+
result.trace.info(f"[STAGE]: ... sleep ({self.sleep} sec)")
|
500
520
|
time.sleep(self.sleep)
|
501
|
-
|
502
521
|
return result.catch(status=SUCCESS)
|
503
522
|
|
504
523
|
async def axecute(
|
@@ -511,12 +530,10 @@ class EmptyStage(BaseAsyncStage):
|
|
511
530
|
"""Async execution method for this Empty stage that only logging out to
|
512
531
|
stdout.
|
513
532
|
|
514
|
-
:param params: (DictData) A
|
515
|
-
|
516
|
-
:param
|
517
|
-
|
518
|
-
:param event: (Event) An event manager that use to track parent execute
|
519
|
-
was not force stopped.
|
533
|
+
:param params: (DictData) A parameter data.
|
534
|
+
:param result: (Result) A Result instance for return context and status.
|
535
|
+
:param event: (Event) An Event manager instance that use to cancel this
|
536
|
+
execution if it forces stopped by parent execution.
|
520
537
|
|
521
538
|
:rtype: Result
|
522
539
|
"""
|
@@ -526,29 +543,36 @@ class EmptyStage(BaseAsyncStage):
|
|
526
543
|
extras=self.extras,
|
527
544
|
)
|
528
545
|
|
529
|
-
|
530
|
-
|
531
|
-
|
532
|
-
|
546
|
+
if not self.echo:
|
547
|
+
message: str = "..."
|
548
|
+
else:
|
549
|
+
message: str = param2template(
|
550
|
+
dedent(self.echo.strip("\n")), params, extras=self.extras
|
551
|
+
)
|
552
|
+
if "\n" in message:
|
553
|
+
message: str = NEWLINE + message.replace("\n", NEWLINE)
|
533
554
|
|
555
|
+
result.trace.info(
|
556
|
+
f"[STAGE]: Empty-Execute: {self.name!r}: ( {message} )"
|
557
|
+
)
|
534
558
|
if self.sleep > 0:
|
535
559
|
if self.sleep > 5:
|
536
560
|
await result.trace.ainfo(
|
537
|
-
f"[STAGE]: ... sleep ({self.sleep}
|
561
|
+
f"[STAGE]: ... sleep ({self.sleep} sec)"
|
538
562
|
)
|
539
563
|
await asyncio.sleep(self.sleep)
|
540
|
-
|
541
564
|
return result.catch(status=SUCCESS)
|
542
565
|
|
543
566
|
|
544
567
|
class BashStage(BaseStage):
|
545
|
-
"""Bash
|
546
|
-
If your current OS is Windows, it will run on the bash
|
568
|
+
"""Bash stage executor that execute bash script on the current OS.
|
569
|
+
If your current OS is Windows, it will run on the bash from the current WSL.
|
570
|
+
It will use `bash` for Windows OS and use `sh` for Linux OS.
|
547
571
|
|
548
|
-
|
549
|
-
subprocess package. It does not good enough to use multiline
|
550
|
-
Thus,
|
551
|
-
issue.
|
572
|
+
This stage has some limitation when it runs shell statement with the
|
573
|
+
built-in subprocess package. It does not good enough to use multiline
|
574
|
+
statement. Thus, it will write the `.sh` file before start running bash
|
575
|
+
command for fix this issue.
|
552
576
|
|
553
577
|
Data Validate:
|
554
578
|
>>> stage = {
|
@@ -560,15 +584,46 @@ class BashStage(BaseStage):
|
|
560
584
|
... }
|
561
585
|
"""
|
562
586
|
|
563
|
-
bash: str = Field(
|
587
|
+
bash: str = Field(
|
588
|
+
description=(
|
589
|
+
"A bash statement that want to execute via Python subprocess."
|
590
|
+
)
|
591
|
+
)
|
564
592
|
env: DictStr = Field(
|
565
593
|
default_factory=dict,
|
566
594
|
description=(
|
567
|
-
"An environment variables that set before
|
568
|
-
"on the header of the `.sh` file."
|
595
|
+
"An environment variables that set before run bash command. It "
|
596
|
+
"will add on the header of the `.sh` file."
|
569
597
|
),
|
570
598
|
)
|
571
599
|
|
600
|
+
@contextlib.asynccontextmanager
|
601
|
+
async def acreate_sh_file(
|
602
|
+
self, bash: str, env: DictStr, run_id: str | None = None
|
603
|
+
) -> AsyncIterator: # pragma no cov
|
604
|
+
import aiofiles
|
605
|
+
|
606
|
+
f_name: str = f"{run_id or uuid.uuid4()}.sh"
|
607
|
+
f_shebang: str = "bash" if sys.platform.startswith("win") else "sh"
|
608
|
+
|
609
|
+
async with aiofiles.open(f"./{f_name}", mode="w", newline="\n") as f:
|
610
|
+
# NOTE: write header of `.sh` file
|
611
|
+
await f.write(f"#!/bin/{f_shebang}\n\n")
|
612
|
+
|
613
|
+
# NOTE: add setting environment variable before bash skip statement.
|
614
|
+
await f.writelines([f"{k}='{env[k]}';\n" for k in env])
|
615
|
+
|
616
|
+
# NOTE: make sure that shell script file does not have `\r` char.
|
617
|
+
await f.write("\n" + bash.replace("\r\n", "\n"))
|
618
|
+
|
619
|
+
# NOTE: Make this .sh file able to executable.
|
620
|
+
make_exec(f"./{f_name}")
|
621
|
+
|
622
|
+
yield [f_shebang, f_name]
|
623
|
+
|
624
|
+
# Note: Remove .sh file that use to run bash.
|
625
|
+
Path(f"./{f_name}").unlink()
|
626
|
+
|
572
627
|
@contextlib.contextmanager
|
573
628
|
def create_sh_file(
|
574
629
|
self, bash: str, env: DictStr, run_id: str | None = None
|
@@ -577,16 +632,14 @@ class BashStage(BaseStage):
|
|
577
632
|
step will write the `.sh` file before giving this file name to context.
|
578
633
|
After that, it will auto delete this file automatic.
|
579
634
|
|
580
|
-
:param bash: (str) A bash statement
|
581
|
-
:param env: (DictStr) An environment variable that
|
582
|
-
statement.
|
635
|
+
:param bash: (str) A bash statement.
|
636
|
+
:param env: (DictStr) An environment variable that set before run bash.
|
583
637
|
:param run_id: (str | None) A running stage ID that use for writing sh
|
584
638
|
file instead generate by UUID4.
|
585
639
|
|
586
640
|
:rtype: Iterator[TupleStr]
|
587
641
|
"""
|
588
|
-
|
589
|
-
f_name: str = f"{run_id}.sh"
|
642
|
+
f_name: str = f"{run_id or uuid.uuid4()}.sh"
|
590
643
|
f_shebang: str = "bash" if sys.platform.startswith("win") else "sh"
|
591
644
|
|
592
645
|
with open(f"./{f_name}", mode="w", newline="\n") as f:
|
@@ -614,14 +667,14 @@ class BashStage(BaseStage):
|
|
614
667
|
result: Result | None = None,
|
615
668
|
event: Event | None = None,
|
616
669
|
) -> Result:
|
617
|
-
"""Execute
|
618
|
-
|
670
|
+
"""Execute bash statement with the Python build-in `subprocess` package.
|
671
|
+
It will catch result from the `subprocess.run` returning output like
|
672
|
+
`return_code`, `stdout`, and `stderr`.
|
619
673
|
|
620
|
-
:param params: A parameter data
|
621
|
-
:param result: (Result) A
|
622
|
-
|
623
|
-
|
624
|
-
was not force stopped.
|
674
|
+
:param params: (DictData) A parameter data.
|
675
|
+
:param result: (Result) A Result instance for return context and status.
|
676
|
+
:param event: (Event) An Event manager instance that use to cancel this
|
677
|
+
execution if it forces stopped by parent execution.
|
625
678
|
|
626
679
|
:rtype: Result
|
627
680
|
"""
|
@@ -631,11 +684,12 @@ class BashStage(BaseStage):
|
|
631
684
|
extras=self.extras,
|
632
685
|
)
|
633
686
|
|
687
|
+
result.trace.info(f"[STAGE]: Shell-Execute: {self.name}")
|
688
|
+
|
634
689
|
bash: str = param2template(
|
635
|
-
dedent(self.bash), params, extras=self.extras
|
690
|
+
dedent(self.bash.strip("\n")), params, extras=self.extras
|
636
691
|
)
|
637
692
|
|
638
|
-
result.trace.info(f"[STAGE]: Shell-Execute: {self.name}")
|
639
693
|
with self.create_sh_file(
|
640
694
|
bash=bash,
|
641
695
|
env=param2template(self.env, params, extras=self.extras),
|
@@ -654,7 +708,7 @@ class BashStage(BaseStage):
|
|
654
708
|
else rs.stderr
|
655
709
|
).removesuffix("\n")
|
656
710
|
raise StageException(
|
657
|
-
f"Subprocess: {e}\
|
711
|
+
f"Subprocess: {e}\n---( statement )---\n"
|
658
712
|
f"```bash\n{bash}\n```"
|
659
713
|
)
|
660
714
|
return result.catch(
|
@@ -668,18 +722,24 @@ class BashStage(BaseStage):
|
|
668
722
|
|
669
723
|
|
670
724
|
class PyStage(BaseStage):
|
671
|
-
"""Python
|
672
|
-
|
725
|
+
"""Python stage that running the Python statement with the current globals
|
726
|
+
and passing an input additional variables via `exec` built-in function.
|
673
727
|
|
674
728
|
This stage allow you to use any Python object that exists on the globals
|
675
729
|
such as import your installed package.
|
676
730
|
|
731
|
+
Warning:
|
732
|
+
|
733
|
+
The exec build-in function is very dangerous. So, it should use the `re`
|
734
|
+
module to validate exec-string before running or exclude the `os` package
|
735
|
+
from the current globals variable.
|
736
|
+
|
677
737
|
Data Validate:
|
678
738
|
>>> stage = {
|
679
739
|
... "name": "Python stage execution",
|
680
|
-
... "run": 'print("Hello {
|
740
|
+
... "run": 'print(f"Hello {VARIABLE}")',
|
681
741
|
... "vars": {
|
682
|
-
... "
|
742
|
+
... "VARIABLE": "WORLD",
|
683
743
|
... },
|
684
744
|
... }
|
685
745
|
"""
|
@@ -704,12 +764,11 @@ class PyStage(BaseStage):
|
|
704
764
|
|
705
765
|
:rtype: Iterator[str]
|
706
766
|
"""
|
707
|
-
from inspect import isclass, ismodule
|
708
|
-
|
709
767
|
for value in values:
|
710
768
|
|
711
769
|
if (
|
712
770
|
value == "__annotations__"
|
771
|
+
or (value.startswith("__") and value.endswith("__"))
|
713
772
|
or ismodule(values[value])
|
714
773
|
or isclass(values[value])
|
715
774
|
):
|
@@ -727,11 +786,10 @@ class PyStage(BaseStage):
|
|
727
786
|
|
728
787
|
:rtype: DictData
|
729
788
|
"""
|
789
|
+
output: DictData = output.copy()
|
730
790
|
lc: DictData = output.pop("locals", {})
|
731
791
|
gb: DictData = output.pop("globals", {})
|
732
|
-
super().set_outputs(
|
733
|
-
{k: lc[k] for k in self.filter_locals(lc)} | output, to=to
|
734
|
-
)
|
792
|
+
super().set_outputs(lc | output, to=to)
|
735
793
|
to.update({k: gb[k] for k in to if k in gb})
|
736
794
|
return to
|
737
795
|
|
@@ -743,9 +801,9 @@ class PyStage(BaseStage):
|
|
743
801
|
event: Event | None = None,
|
744
802
|
) -> Result:
|
745
803
|
"""Execute the Python statement that pass all globals and input params
|
746
|
-
to globals argument on
|
804
|
+
to globals argument on `exec` build-in function.
|
747
805
|
|
748
|
-
:param params: A parameter
|
806
|
+
:param params: (DictData) A parameter data.
|
749
807
|
:param result: (Result) A result object for keeping context and status
|
750
808
|
data.
|
751
809
|
:param event: (Event) An event manager that use to track parent execute
|
@@ -762,42 +820,68 @@ class PyStage(BaseStage):
|
|
762
820
|
lc: DictData = {}
|
763
821
|
gb: DictData = (
|
764
822
|
globals()
|
765
|
-
| params
|
766
823
|
| param2template(self.vars, params, extras=self.extras)
|
767
824
|
| {"result": result}
|
768
825
|
)
|
769
826
|
|
770
|
-
# NOTE: Start exec the run statement.
|
771
827
|
result.trace.info(f"[STAGE]: Py-Execute: {self.name}")
|
772
|
-
|
773
|
-
# "[STAGE]: This stage allow use `eval` function, so, please "
|
774
|
-
# "check your statement be safe before execute."
|
775
|
-
# )
|
776
|
-
#
|
828
|
+
|
777
829
|
# WARNING: The exec build-in function is very dangerous. So, it
|
778
830
|
# should use the re module to validate exec-string before running.
|
779
831
|
exec(
|
780
|
-
param2template(dedent(self.run), params, extras=self.extras),
|
832
|
+
param2template(dedent(self.run), params, extras=self.extras),
|
833
|
+
gb,
|
834
|
+
lc,
|
781
835
|
)
|
782
836
|
|
783
837
|
return result.catch(
|
784
|
-
status=SUCCESS,
|
838
|
+
status=SUCCESS,
|
839
|
+
context={
|
840
|
+
"locals": {k: lc[k] for k in self.filter_locals(lc)},
|
841
|
+
"globals": {
|
842
|
+
k: gb[k]
|
843
|
+
for k in gb
|
844
|
+
if (
|
845
|
+
not k.startswith("__")
|
846
|
+
and k != "annotations"
|
847
|
+
and not ismodule(gb[k])
|
848
|
+
and not isclass(gb[k])
|
849
|
+
and not isfunction(gb[k])
|
850
|
+
)
|
851
|
+
},
|
852
|
+
},
|
785
853
|
)
|
786
854
|
|
855
|
+
async def axecute(
|
856
|
+
self,
|
857
|
+
):
|
858
|
+
"""Async execution method.
|
859
|
+
|
860
|
+
References:
|
861
|
+
- https://stackoverflow.com/questions/44859165/async-exec-in-python
|
862
|
+
"""
|
863
|
+
|
787
864
|
|
788
865
|
class CallStage(BaseStage):
|
789
|
-
"""Call executor that call the Python function from registry with tag
|
790
|
-
decorator function in
|
866
|
+
"""Call stage executor that call the Python function from registry with tag
|
867
|
+
decorator function in `reusables` module and run it with input arguments.
|
868
|
+
|
869
|
+
This stage is different with PyStage because the PyStage is just run
|
870
|
+
a Python statement with the `exec` function and pass the current locals and
|
871
|
+
globals before exec that statement. This stage will import the caller
|
872
|
+
function can call it with an input arguments. So, you can create your
|
873
|
+
function complexly that you can for your objective to invoked by this stage
|
874
|
+
object.
|
875
|
+
|
876
|
+
This stage is the most powerfull stage of this package for run every
|
877
|
+
use-case by a custom requirement that you want by creating the Python
|
878
|
+
function and adding it to the caller registry value by importer syntax like
|
879
|
+
`module.caller.registry` not path style like `module/caller/registry`.
|
791
880
|
|
792
|
-
|
793
|
-
a Python statement with the ``eval`` and pass that locale before eval that
|
794
|
-
statement. So, you can create your function complexly that you can for your
|
795
|
-
objective to invoked by this stage object.
|
881
|
+
Warning:
|
796
882
|
|
797
|
-
|
798
|
-
|
799
|
-
registry by importer syntax like `module.tasks.registry` not path style like
|
800
|
-
`module/tasks/registry`.
|
883
|
+
The caller registry to get a caller function should importable by the
|
884
|
+
current Python execution pointer.
|
801
885
|
|
802
886
|
Data Validate:
|
803
887
|
>>> stage = {
|
@@ -809,12 +893,16 @@ class CallStage(BaseStage):
|
|
809
893
|
|
810
894
|
uses: str = Field(
|
811
895
|
description=(
|
812
|
-
"A
|
896
|
+
"A caller function with registry importer syntax that use to load "
|
897
|
+
"function before execute step. The caller registry syntax should "
|
898
|
+
"be `<import.part>/<func-name>@<tag-name>`."
|
813
899
|
),
|
814
900
|
)
|
815
901
|
args: DictData = Field(
|
816
902
|
default_factory=dict,
|
817
|
-
description=
|
903
|
+
description=(
|
904
|
+
"An argument parameter that will pass to this caller function."
|
905
|
+
),
|
818
906
|
alias="with",
|
819
907
|
)
|
820
908
|
|
@@ -825,24 +913,17 @@ class CallStage(BaseStage):
|
|
825
913
|
result: Result | None = None,
|
826
914
|
event: Event | None = None,
|
827
915
|
) -> Result:
|
828
|
-
"""Execute
|
916
|
+
"""Execute this caller function with its argument parameter.
|
829
917
|
|
830
|
-
:
|
831
|
-
|
832
|
-
:
|
833
|
-
|
834
|
-
|
835
|
-
:param params: (DictData) A parameter that want to pass before run any
|
836
|
-
statement.
|
837
|
-
:param result: (Result) A result object for keeping context and status
|
838
|
-
data.
|
839
|
-
:param event: (Event) An event manager that use to track parent execute
|
840
|
-
was not force stopped.
|
918
|
+
:param params: (DictData) A parameter data.
|
919
|
+
:param result: (Result) A Result instance for return context and status.
|
920
|
+
:param event: (Event) An Event manager instance that use to cancel this
|
921
|
+
execution if it forces stopped by parent execution.
|
841
922
|
|
842
923
|
:raise ValueError: If necessary arguments does not pass from the `args`
|
843
924
|
field.
|
844
|
-
:raise TypeError: If the result from the caller function does not
|
845
|
-
a `dict` type.
|
925
|
+
:raise TypeError: If the result from the caller function does not match
|
926
|
+
with a `dict` type.
|
846
927
|
|
847
928
|
:rtype: Result
|
848
929
|
"""
|
@@ -852,12 +933,15 @@ class CallStage(BaseStage):
|
|
852
933
|
extras=self.extras,
|
853
934
|
)
|
854
935
|
|
855
|
-
has_keyword: bool = False
|
856
936
|
call_func: TagFunc = extract_call(
|
857
937
|
param2template(self.uses, params, extras=self.extras),
|
858
938
|
registries=self.extras.get("registry_caller"),
|
859
939
|
)()
|
860
940
|
|
941
|
+
result.trace.info(
|
942
|
+
f"[STAGE]: Call-Execute: {call_func.name}@{call_func.tag}"
|
943
|
+
)
|
944
|
+
|
861
945
|
# VALIDATE: check input task caller parameters that exists before
|
862
946
|
# calling.
|
863
947
|
args: DictData = {"result": result} | param2template(
|
@@ -865,6 +949,7 @@ class CallStage(BaseStage):
|
|
865
949
|
)
|
866
950
|
ips = inspect.signature(call_func)
|
867
951
|
necessary_params: list[str] = []
|
952
|
+
has_keyword: bool = False
|
868
953
|
for k in ips.parameters:
|
869
954
|
if (
|
870
955
|
v := ips.parameters[k]
|
@@ -888,10 +973,6 @@ class CallStage(BaseStage):
|
|
888
973
|
if "result" not in ips.parameters and not has_keyword:
|
889
974
|
args.pop("result")
|
890
975
|
|
891
|
-
result.trace.info(
|
892
|
-
f"[STAGE]: Call-Execute: {call_func.name}@{call_func.tag}"
|
893
|
-
)
|
894
|
-
|
895
976
|
args = self.parse_model_args(call_func, args, result)
|
896
977
|
|
897
978
|
if inspect.iscoroutinefunction(call_func):
|
@@ -962,9 +1043,9 @@ class CallStage(BaseStage):
|
|
962
1043
|
|
963
1044
|
|
964
1045
|
class TriggerStage(BaseStage):
|
965
|
-
"""Trigger
|
966
|
-
the
|
967
|
-
|
1046
|
+
"""Trigger workflow executor stage that run an input trigger Workflow
|
1047
|
+
execute method. This is the stage that allow you to create the reusable
|
1048
|
+
Workflow template with dynamic parameters.
|
968
1049
|
|
969
1050
|
Data Validate:
|
970
1051
|
>>> stage = {
|
@@ -976,12 +1057,13 @@ class TriggerStage(BaseStage):
|
|
976
1057
|
|
977
1058
|
trigger: str = Field(
|
978
1059
|
description=(
|
979
|
-
"A trigger workflow name
|
1060
|
+
"A trigger workflow name. This workflow name should exist on the "
|
1061
|
+
"config path because it will load by the `load_conf` method."
|
980
1062
|
),
|
981
1063
|
)
|
982
1064
|
params: DictData = Field(
|
983
1065
|
default_factory=dict,
|
984
|
-
description="A parameter that
|
1066
|
+
description="A parameter that will pass to workflow execution method.",
|
985
1067
|
)
|
986
1068
|
|
987
1069
|
def execute(
|
@@ -994,7 +1076,7 @@ class TriggerStage(BaseStage):
|
|
994
1076
|
"""Trigger another workflow execution. It will wait the trigger
|
995
1077
|
workflow running complete before catching its result.
|
996
1078
|
|
997
|
-
:param params: A parameter data
|
1079
|
+
:param params: (DictData) A parameter data.
|
998
1080
|
:param result: (Result) A result object for keeping context and status
|
999
1081
|
data.
|
1000
1082
|
:param event: (Event) An event manager that use to track parent execute
|
@@ -1029,14 +1111,18 @@ class TriggerStage(BaseStage):
|
|
1029
1111
|
err_msg: str | None = (
|
1030
1112
|
f" with:\n{msg}"
|
1031
1113
|
if (msg := rs.context.get("errors", {}).get("message"))
|
1032
|
-
else ""
|
1114
|
+
else "."
|
1115
|
+
)
|
1116
|
+
raise StageException(
|
1117
|
+
f"Trigger workflow return failed status{err_msg}"
|
1033
1118
|
)
|
1034
|
-
raise StageException(f"Trigger workflow was failed{err_msg}.")
|
1035
1119
|
return rs
|
1036
1120
|
|
1037
1121
|
|
1038
1122
|
class ParallelStage(BaseStage): # pragma: no cov
|
1039
|
-
"""Parallel
|
1123
|
+
"""Parallel stage executor that execute branch stages with multithreading.
|
1124
|
+
This stage let you set the fix branches for running substage inside it on
|
1125
|
+
multithread pool.
|
1040
1126
|
|
1041
1127
|
This stage is not the low-level stage model because it runs muti-stages
|
1042
1128
|
in this stage execution.
|
@@ -1064,52 +1150,50 @@ class ParallelStage(BaseStage): # pragma: no cov
|
|
1064
1150
|
"""
|
1065
1151
|
|
1066
1152
|
parallel: dict[str, list[Stage]] = Field(
|
1067
|
-
description="A mapping of
|
1153
|
+
description="A mapping of branch name and its stages.",
|
1068
1154
|
)
|
1069
1155
|
max_workers: int = Field(
|
1070
1156
|
default=2,
|
1071
1157
|
ge=1,
|
1072
1158
|
lt=20,
|
1073
1159
|
description=(
|
1074
|
-
"The maximum thread pool worker size for execution parallel."
|
1160
|
+
"The maximum multi-thread pool worker size for execution parallel. "
|
1161
|
+
"This value should be gather or equal than 1, and less than 20."
|
1075
1162
|
),
|
1076
1163
|
alias="max-workers",
|
1077
1164
|
)
|
1078
1165
|
|
1079
|
-
def
|
1166
|
+
def execute_branch(
|
1080
1167
|
self,
|
1081
1168
|
branch: str,
|
1082
1169
|
params: DictData,
|
1083
1170
|
result: Result,
|
1084
1171
|
*,
|
1085
1172
|
event: Event | None = None,
|
1086
|
-
extras: DictData | None = None,
|
1087
1173
|
) -> DictData:
|
1088
|
-
"""
|
1174
|
+
"""Branch execution method for execute all stages of a specific branch
|
1175
|
+
ID.
|
1089
1176
|
|
1090
|
-
:param branch: A branch ID.
|
1091
|
-
:param params: A parameter data
|
1092
|
-
:param result: (Result) A
|
1093
|
-
|
1094
|
-
|
1095
|
-
was not force stopped.
|
1096
|
-
:param extras: (DictData) An extra parameters that want to override
|
1097
|
-
config values.
|
1177
|
+
:param branch: (str) A branch ID.
|
1178
|
+
:param params: (DictData) A parameter data.
|
1179
|
+
:param result: (Result) A Result instance for return context and status.
|
1180
|
+
:param event: (Event) An Event manager instance that use to cancel this
|
1181
|
+
execution if it forces stopped by parent execution.
|
1098
1182
|
|
1099
1183
|
:rtype: DictData
|
1100
1184
|
"""
|
1101
|
-
result.trace.debug(f"
|
1102
|
-
|
1103
|
-
|
1104
|
-
|
1185
|
+
result.trace.debug(f"[STAGE]: Execute Branch: {branch!r}")
|
1186
|
+
context: DictData = copy.deepcopy(params)
|
1187
|
+
context.update({"branch": branch})
|
1188
|
+
output: DictData = {"branch": branch, "stages": {}}
|
1105
1189
|
for stage in self.parallel[branch]:
|
1106
1190
|
|
1107
|
-
if extras:
|
1108
|
-
stage.extras = extras
|
1191
|
+
if self.extras:
|
1192
|
+
stage.extras = self.extras
|
1109
1193
|
|
1110
|
-
if stage.is_skipped(params=
|
1194
|
+
if stage.is_skipped(params=context):
|
1111
1195
|
result.trace.info(f"... Skip stage: {stage.iden!r}")
|
1112
|
-
stage.set_outputs(output={"skipped": True}, to=
|
1196
|
+
stage.set_outputs(output={"skipped": True}, to=output)
|
1113
1197
|
continue
|
1114
1198
|
|
1115
1199
|
if event and event.is_set():
|
@@ -1122,7 +1206,7 @@ class ParallelStage(BaseStage): # pragma: no cov
|
|
1122
1206
|
parallel={
|
1123
1207
|
branch: {
|
1124
1208
|
"branch": branch,
|
1125
|
-
"stages": filter_func(
|
1209
|
+
"stages": filter_func(output.pop("stages", {})),
|
1126
1210
|
"errors": StageException(error_msg).to_dict(),
|
1127
1211
|
}
|
1128
1212
|
},
|
@@ -1130,14 +1214,15 @@ class ParallelStage(BaseStage): # pragma: no cov
|
|
1130
1214
|
|
1131
1215
|
try:
|
1132
1216
|
rs: Result = stage.handler_execute(
|
1133
|
-
params=
|
1217
|
+
params=context,
|
1134
1218
|
run_id=result.run_id,
|
1135
1219
|
parent_run_id=result.parent_run_id,
|
1136
1220
|
raise_error=True,
|
1137
1221
|
event=event,
|
1138
1222
|
)
|
1139
|
-
stage.set_outputs(rs.context, to=
|
1140
|
-
|
1223
|
+
stage.set_outputs(rs.context, to=output)
|
1224
|
+
stage.set_outputs(stage.get_outputs(output), to=context)
|
1225
|
+
except (StageException, UtilException) as e: # pragma: no cov
|
1141
1226
|
result.trace.error(f"[STAGE]: {e.__class__.__name__}: {e}")
|
1142
1227
|
raise StageException(
|
1143
1228
|
f"Sub-Stage execution error: {e.__class__.__name__}: {e}"
|
@@ -1145,7 +1230,7 @@ class ParallelStage(BaseStage): # pragma: no cov
|
|
1145
1230
|
|
1146
1231
|
if rs.status == FAILED:
|
1147
1232
|
error_msg: str = (
|
1148
|
-
f"
|
1233
|
+
f"Branch-Stage was break because it has a sub stage, "
|
1149
1234
|
f"{stage.iden}, failed without raise error."
|
1150
1235
|
)
|
1151
1236
|
return result.catch(
|
@@ -1153,7 +1238,7 @@ class ParallelStage(BaseStage): # pragma: no cov
|
|
1153
1238
|
parallel={
|
1154
1239
|
branch: {
|
1155
1240
|
"branch": branch,
|
1156
|
-
"stages": filter_func(
|
1241
|
+
"stages": filter_func(output.pop("stages", {})),
|
1157
1242
|
"errors": StageException(error_msg).to_dict(),
|
1158
1243
|
},
|
1159
1244
|
},
|
@@ -1164,7 +1249,7 @@ class ParallelStage(BaseStage): # pragma: no cov
|
|
1164
1249
|
parallel={
|
1165
1250
|
branch: {
|
1166
1251
|
"branch": branch,
|
1167
|
-
"stages": filter_func(
|
1252
|
+
"stages": filter_func(output.pop("stages", {})),
|
1168
1253
|
},
|
1169
1254
|
},
|
1170
1255
|
)
|
@@ -1176,14 +1261,12 @@ class ParallelStage(BaseStage): # pragma: no cov
|
|
1176
1261
|
result: Result | None = None,
|
1177
1262
|
event: Event | None = None,
|
1178
1263
|
) -> Result:
|
1179
|
-
"""Execute
|
1180
|
-
or async mode by changing `async_mode` flag.
|
1264
|
+
"""Execute parallel each branch via multi-threading pool.
|
1181
1265
|
|
1182
|
-
:param params: A parameter
|
1183
|
-
:param result: (Result) A
|
1184
|
-
|
1185
|
-
|
1186
|
-
was not force stopped.
|
1266
|
+
:param params: (DictData) A parameter data.
|
1267
|
+
:param result: (Result) A Result instance for return context and status.
|
1268
|
+
:param event: (Event) An Event manager instance that use to cancel this
|
1269
|
+
execution if it forces stopped by parent execution.
|
1187
1270
|
|
1188
1271
|
:rtype: Result
|
1189
1272
|
"""
|
@@ -1207,12 +1290,11 @@ class ParallelStage(BaseStage): # pragma: no cov
|
|
1207
1290
|
|
1208
1291
|
futures: list[Future] = (
|
1209
1292
|
executor.submit(
|
1210
|
-
self.
|
1293
|
+
self.execute_branch,
|
1211
1294
|
branch=branch,
|
1212
1295
|
params=params,
|
1213
1296
|
result=result,
|
1214
1297
|
event=event,
|
1215
|
-
extras=self.extras,
|
1216
1298
|
)
|
1217
1299
|
for branch in self.parallel
|
1218
1300
|
)
|
@@ -1232,11 +1314,11 @@ class ParallelStage(BaseStage): # pragma: no cov
|
|
1232
1314
|
|
1233
1315
|
|
1234
1316
|
class ForEachStage(BaseStage):
|
1235
|
-
"""For-Each
|
1236
|
-
|
1237
|
-
muti-stages in this stage execution.
|
1317
|
+
"""For-Each stage executor that execute all stages with each item in the
|
1318
|
+
foreach list.
|
1238
1319
|
|
1239
|
-
|
1320
|
+
This stage is not the low-level stage model because it runs
|
1321
|
+
muti-stages in this stage execution.
|
1240
1322
|
|
1241
1323
|
Data Validate:
|
1242
1324
|
>>> stage = {
|
@@ -1245,7 +1327,7 @@ class ForEachStage(BaseStage):
|
|
1245
1327
|
... "stages": [
|
1246
1328
|
... {
|
1247
1329
|
... "name": "Echo stage",
|
1248
|
-
... "echo": "Start run with item {{ item }}"
|
1330
|
+
... "echo": "Start run with item ${{ item }}"
|
1249
1331
|
... },
|
1250
1332
|
... ],
|
1251
1333
|
... }
|
@@ -1253,13 +1335,14 @@ class ForEachStage(BaseStage):
|
|
1253
1335
|
|
1254
1336
|
foreach: Union[list[str], list[int], str] = Field(
|
1255
1337
|
description=(
|
1256
|
-
"A items for passing to
|
1338
|
+
"A items for passing to stages via ${{ item }} template parameter."
|
1257
1339
|
),
|
1258
1340
|
)
|
1259
1341
|
stages: list[Stage] = Field(
|
1260
1342
|
default_factory=list,
|
1261
1343
|
description=(
|
1262
|
-
"A list of stage that will run with each item in the foreach
|
1344
|
+
"A list of stage that will run with each item in the `foreach` "
|
1345
|
+
"field."
|
1263
1346
|
),
|
1264
1347
|
)
|
1265
1348
|
concurrent: int = Field(
|
@@ -1274,7 +1357,7 @@ class ForEachStage(BaseStage):
|
|
1274
1357
|
|
1275
1358
|
def execute_item(
|
1276
1359
|
self,
|
1277
|
-
item:
|
1360
|
+
item: StrOrInt,
|
1278
1361
|
params: DictData,
|
1279
1362
|
result: Result,
|
1280
1363
|
*,
|
@@ -1283,29 +1366,27 @@ class ForEachStage(BaseStage):
|
|
1283
1366
|
"""Execute foreach item from list of item.
|
1284
1367
|
|
1285
1368
|
:param item: (str | int) An item that want to execution.
|
1286
|
-
:param params: (DictData) A parameter
|
1287
|
-
|
1288
|
-
:param
|
1289
|
-
|
1290
|
-
:param event: (Event) An event manager that use to track parent execute
|
1291
|
-
was not force stopped.
|
1369
|
+
:param params: (DictData) A parameter data.
|
1370
|
+
:param result: (Result) A Result instance for return context and status.
|
1371
|
+
:param event: (Event) An Event manager instance that use to cancel this
|
1372
|
+
execution if it forces stopped by parent execution.
|
1292
1373
|
|
1293
1374
|
:raise StageException: If the stage execution raise errors.
|
1294
1375
|
|
1295
1376
|
:rtype: Result
|
1296
1377
|
"""
|
1297
|
-
result.trace.debug(f"
|
1298
|
-
|
1299
|
-
|
1300
|
-
|
1378
|
+
result.trace.debug(f"[STAGE]: Execute Item: {item!r}")
|
1379
|
+
context: DictData = copy.deepcopy(params)
|
1380
|
+
context.update({"item": item})
|
1381
|
+
output: DictData = {"item": item, "stages": {}}
|
1301
1382
|
for stage in self.stages:
|
1302
1383
|
|
1303
1384
|
if self.extras:
|
1304
1385
|
stage.extras = self.extras
|
1305
1386
|
|
1306
|
-
if stage.is_skipped(params=
|
1387
|
+
if stage.is_skipped(params=context):
|
1307
1388
|
result.trace.info(f"... Skip stage: {stage.iden!r}")
|
1308
|
-
stage.set_outputs(output={"skipped": True}, to=
|
1389
|
+
stage.set_outputs(output={"skipped": True}, to=output)
|
1309
1390
|
continue
|
1310
1391
|
|
1311
1392
|
if event and event.is_set(): # pragma: no cov
|
@@ -1318,7 +1399,7 @@ class ForEachStage(BaseStage):
|
|
1318
1399
|
foreach={
|
1319
1400
|
item: {
|
1320
1401
|
"item": item,
|
1321
|
-
"stages": filter_func(
|
1402
|
+
"stages": filter_func(output.pop("stages", {})),
|
1322
1403
|
"errors": StageException(error_msg).to_dict(),
|
1323
1404
|
}
|
1324
1405
|
},
|
@@ -1326,17 +1407,28 @@ class ForEachStage(BaseStage):
|
|
1326
1407
|
|
1327
1408
|
try:
|
1328
1409
|
rs: Result = stage.handler_execute(
|
1329
|
-
params=
|
1410
|
+
params=context,
|
1330
1411
|
run_id=result.run_id,
|
1331
1412
|
parent_run_id=result.parent_run_id,
|
1332
1413
|
raise_error=True,
|
1333
1414
|
event=event,
|
1334
1415
|
)
|
1335
|
-
stage.set_outputs(rs.context, to=
|
1416
|
+
stage.set_outputs(rs.context, to=output)
|
1417
|
+
stage.set_outputs(stage.get_outputs(output), to=context)
|
1336
1418
|
except (StageException, UtilException) as e:
|
1337
1419
|
result.trace.error(f"[STAGE]: {e.__class__.__name__}: {e}")
|
1420
|
+
result.catch(
|
1421
|
+
status=FAILED,
|
1422
|
+
foreach={
|
1423
|
+
item: {
|
1424
|
+
"item": item,
|
1425
|
+
"stages": filter_func(output.pop("stages", {})),
|
1426
|
+
"errors": e.to_dict(),
|
1427
|
+
},
|
1428
|
+
},
|
1429
|
+
)
|
1338
1430
|
raise StageException(
|
1339
|
-
f"Sub-Stage
|
1431
|
+
f"Sub-Stage raise: {e.__class__.__name__}: {e}"
|
1340
1432
|
) from None
|
1341
1433
|
|
1342
1434
|
if rs.status == FAILED:
|
@@ -1349,18 +1441,17 @@ class ForEachStage(BaseStage):
|
|
1349
1441
|
foreach={
|
1350
1442
|
item: {
|
1351
1443
|
"item": item,
|
1352
|
-
"stages": filter_func(
|
1444
|
+
"stages": filter_func(output.pop("stages", {})),
|
1353
1445
|
"errors": StageException(error_msg).to_dict(),
|
1354
1446
|
},
|
1355
1447
|
},
|
1356
1448
|
)
|
1357
|
-
|
1358
1449
|
return result.catch(
|
1359
1450
|
status=SUCCESS,
|
1360
1451
|
foreach={
|
1361
1452
|
item: {
|
1362
1453
|
"item": item,
|
1363
|
-
"stages": filter_func(
|
1454
|
+
"stages": filter_func(output.pop("stages", {})),
|
1364
1455
|
},
|
1365
1456
|
},
|
1366
1457
|
)
|
@@ -1374,11 +1465,10 @@ class ForEachStage(BaseStage):
|
|
1374
1465
|
) -> Result:
|
1375
1466
|
"""Execute the stages that pass each item form the foreach field.
|
1376
1467
|
|
1377
|
-
:param params: A parameter
|
1378
|
-
:param result: (Result) A
|
1379
|
-
|
1380
|
-
|
1381
|
-
was not force stopped.
|
1468
|
+
:param params: (DictData) A parameter data.
|
1469
|
+
:param result: (Result) A Result instance for return context and status.
|
1470
|
+
:param event: (Event) An Event manager instance that use to cancel this
|
1471
|
+
execution if it forces stopped by parent execution.
|
1382
1472
|
|
1383
1473
|
:rtype: Result
|
1384
1474
|
"""
|
@@ -1394,9 +1484,7 @@ class ForEachStage(BaseStage):
|
|
1394
1484
|
else self.foreach
|
1395
1485
|
)
|
1396
1486
|
if not isinstance(foreach, list):
|
1397
|
-
raise StageException(
|
1398
|
-
f"Foreach does not support foreach value: {foreach!r}"
|
1399
|
-
)
|
1487
|
+
raise StageException(f"Does not support foreach: {foreach!r}")
|
1400
1488
|
|
1401
1489
|
result.trace.info(f"[STAGE]: Foreach-Execute: {foreach!r}.")
|
1402
1490
|
result.catch(status=WAIT, context={"items": foreach, "foreach": {}})
|
@@ -1428,33 +1516,33 @@ class ForEachStage(BaseStage):
|
|
1428
1516
|
context: DictData = {}
|
1429
1517
|
status: Status = SUCCESS
|
1430
1518
|
|
1431
|
-
done, not_done = wait(
|
1432
|
-
futures, timeout=1800, return_when=FIRST_EXCEPTION
|
1433
|
-
)
|
1434
|
-
|
1519
|
+
done, not_done = wait(futures, return_when=FIRST_EXCEPTION)
|
1435
1520
|
if len(done) != len(futures):
|
1436
1521
|
result.trace.warning(
|
1437
|
-
"[STAGE]: Set
|
1522
|
+
"[STAGE]: Set event for stop pending stage future."
|
1438
1523
|
)
|
1439
1524
|
event.set()
|
1440
1525
|
for future in not_done:
|
1441
1526
|
future.cancel()
|
1442
1527
|
|
1528
|
+
nd: str = f", item not run: {not_done}" if not_done else ""
|
1529
|
+
result.trace.debug(f"... Foreach set Fail-Fast{nd}")
|
1530
|
+
|
1443
1531
|
for future in done:
|
1444
1532
|
try:
|
1445
1533
|
future.result()
|
1446
|
-
except StageException as e:
|
1534
|
+
except (StageException, UtilException) as e:
|
1447
1535
|
status = FAILED
|
1448
1536
|
result.trace.error(
|
1449
|
-
f"[STAGE]: {e.__class__.__name__}
|
1537
|
+
f"[STAGE]: {e.__class__.__name__}:{NEWLINE}{e}"
|
1450
1538
|
)
|
1451
1539
|
context.update({"errors": e.to_dict()})
|
1452
|
-
|
1453
1540
|
return result.catch(status=status, context=context)
|
1454
1541
|
|
1455
1542
|
|
1456
|
-
class UntilStage(BaseStage):
|
1457
|
-
"""Until
|
1543
|
+
class UntilStage(BaseStage):
|
1544
|
+
"""Until stage executor that will run stages in each loop until it valid
|
1545
|
+
with stop loop condition.
|
1458
1546
|
|
1459
1547
|
Data Validate:
|
1460
1548
|
>>> stage = {
|
@@ -1476,19 +1564,21 @@ class UntilStage(BaseStage): # pragma: no cov
|
|
1476
1564
|
"An initial value that can be any value in str, int, or bool type."
|
1477
1565
|
),
|
1478
1566
|
)
|
1479
|
-
until: str = Field(description="A until condition.")
|
1567
|
+
until: str = Field(description="A until condition for stop the while loop.")
|
1480
1568
|
stages: list[Stage] = Field(
|
1481
1569
|
default_factory=list,
|
1482
1570
|
description=(
|
1483
|
-
"A list of stage that will run with each item until
|
1484
|
-
"correct."
|
1571
|
+
"A list of stage that will run with each item in until loop."
|
1485
1572
|
),
|
1486
1573
|
)
|
1487
1574
|
max_loop: int = Field(
|
1488
1575
|
default=10,
|
1489
1576
|
ge=1,
|
1490
1577
|
lt=100,
|
1491
|
-
description=
|
1578
|
+
description=(
|
1579
|
+
"The maximum value of loop for this until stage. This value should "
|
1580
|
+
"be gather or equal than 1, and less than 100."
|
1581
|
+
),
|
1492
1582
|
alias="max-loop",
|
1493
1583
|
)
|
1494
1584
|
|
@@ -1500,33 +1590,31 @@ class UntilStage(BaseStage): # pragma: no cov
|
|
1500
1590
|
result: Result,
|
1501
1591
|
event: Event | None = None,
|
1502
1592
|
) -> tuple[Result, T]:
|
1503
|
-
"""Execute
|
1593
|
+
"""Execute loop item that was set from some stage or set by default loop
|
1504
1594
|
variable.
|
1505
1595
|
|
1506
1596
|
:param item: (T) An item that want to execution.
|
1507
1597
|
:param loop: (int) A number of loop.
|
1508
|
-
:param params: (DictData) A parameter
|
1509
|
-
|
1510
|
-
:param
|
1511
|
-
|
1512
|
-
:param event: (Event) An event manager that use to track parent execute
|
1513
|
-
was not force stopped.
|
1598
|
+
:param params: (DictData) A parameter data.
|
1599
|
+
:param result: (Result) A Result instance for return context and status.
|
1600
|
+
:param event: (Event) An Event manager instance that use to cancel this
|
1601
|
+
execution if it forces stopped by parent execution.
|
1514
1602
|
|
1515
1603
|
:rtype: tuple[Result, T]
|
1516
1604
|
"""
|
1517
1605
|
result.trace.debug(f"... Execute until item: {item!r}")
|
1518
|
-
|
1519
|
-
|
1520
|
-
|
1606
|
+
context: DictData = copy.deepcopy(params)
|
1607
|
+
context.update({"item": item})
|
1608
|
+
output: DictData = {"loop": loop, "item": item, "stages": {}}
|
1521
1609
|
next_item: T = None
|
1522
1610
|
for stage in self.stages:
|
1523
1611
|
|
1524
1612
|
if self.extras:
|
1525
1613
|
stage.extras = self.extras
|
1526
1614
|
|
1527
|
-
if stage.is_skipped(params=
|
1615
|
+
if stage.is_skipped(params=context):
|
1528
1616
|
result.trace.info(f"... Skip stage: {stage.iden!r}")
|
1529
|
-
stage.set_outputs(output={"skipped": True}, to=
|
1617
|
+
stage.set_outputs(output={"skipped": True}, to=output)
|
1530
1618
|
continue
|
1531
1619
|
|
1532
1620
|
if event and event.is_set():
|
@@ -1541,9 +1629,7 @@ class UntilStage(BaseStage): # pragma: no cov
|
|
1541
1629
|
loop: {
|
1542
1630
|
"loop": loop,
|
1543
1631
|
"item": item,
|
1544
|
-
"stages": filter_func(
|
1545
|
-
context.pop("stages", {})
|
1546
|
-
),
|
1632
|
+
"stages": filter_func(output.pop("stages", {})),
|
1547
1633
|
"errors": StageException(error_msg).to_dict(),
|
1548
1634
|
}
|
1549
1635
|
},
|
@@ -1553,17 +1639,18 @@ class UntilStage(BaseStage): # pragma: no cov
|
|
1553
1639
|
|
1554
1640
|
try:
|
1555
1641
|
rs: Result = stage.handler_execute(
|
1556
|
-
params=
|
1642
|
+
params=context,
|
1557
1643
|
run_id=result.run_id,
|
1558
1644
|
parent_run_id=result.parent_run_id,
|
1559
1645
|
raise_error=True,
|
1560
1646
|
event=event,
|
1561
1647
|
)
|
1562
|
-
stage.set_outputs(rs.context, to=
|
1563
|
-
|
1564
|
-
|
1565
|
-
|
1566
|
-
|
1648
|
+
stage.set_outputs(rs.context, to=output)
|
1649
|
+
|
1650
|
+
if "item" in (_output := stage.get_outputs(output)):
|
1651
|
+
next_item = _output["item"]
|
1652
|
+
|
1653
|
+
stage.set_outputs(_output, to=context)
|
1567
1654
|
except (StageException, UtilException) as e:
|
1568
1655
|
result.trace.error(f"[STAGE]: {e.__class__.__name__}: {e}")
|
1569
1656
|
raise StageException(
|
@@ -1577,7 +1664,7 @@ class UntilStage(BaseStage): # pragma: no cov
|
|
1577
1664
|
loop: {
|
1578
1665
|
"loop": loop,
|
1579
1666
|
"item": item,
|
1580
|
-
"stages": filter_func(
|
1667
|
+
"stages": filter_func(output.pop("stages", {})),
|
1581
1668
|
}
|
1582
1669
|
},
|
1583
1670
|
),
|
@@ -1594,11 +1681,10 @@ class UntilStage(BaseStage): # pragma: no cov
|
|
1594
1681
|
"""Execute the stages that pass item from until condition field and
|
1595
1682
|
setter step.
|
1596
1683
|
|
1597
|
-
:param params: A parameter
|
1598
|
-
:param result: (Result) A
|
1599
|
-
|
1600
|
-
|
1601
|
-
was not force stopped.
|
1684
|
+
:param params: (DictData) A parameter data.
|
1685
|
+
:param result: (Result) A Result instance for return context and status.
|
1686
|
+
:param event: (Event) An Event manager instance that use to cancel this
|
1687
|
+
execution if it forces stopped by parent execution.
|
1602
1688
|
|
1603
1689
|
:rtype: Result
|
1604
1690
|
"""
|
@@ -1671,12 +1757,14 @@ class UntilStage(BaseStage): # pragma: no cov
|
|
1671
1757
|
class Match(BaseModel):
|
1672
1758
|
"""Match model for the Case Stage."""
|
1673
1759
|
|
1674
|
-
case:
|
1675
|
-
|
1760
|
+
case: StrOrInt = Field(description="A match case.")
|
1761
|
+
stages: list[Stage] = Field(
|
1762
|
+
description="A list of stage to execution for this case."
|
1763
|
+
)
|
1676
1764
|
|
1677
1765
|
|
1678
1766
|
class CaseStage(BaseStage):
|
1679
|
-
"""Case
|
1767
|
+
"""Case stage executor that execute all stages if the condition was matched.
|
1680
1768
|
|
1681
1769
|
Data Validate:
|
1682
1770
|
>>> stage = {
|
@@ -1685,24 +1773,21 @@ class CaseStage(BaseStage):
|
|
1685
1773
|
... "match": [
|
1686
1774
|
... {
|
1687
1775
|
... "case": "1",
|
1688
|
-
... "
|
1689
|
-
...
|
1690
|
-
...
|
1691
|
-
...
|
1692
|
-
...
|
1693
|
-
...
|
1694
|
-
... "case": "2",
|
1695
|
-
... "stage": {
|
1696
|
-
... "name": "Stage case 2",
|
1697
|
-
... "eche": "Hello case 2",
|
1698
|
-
... },
|
1776
|
+
... "stages": [
|
1777
|
+
... {
|
1778
|
+
... "name": "Stage case 1",
|
1779
|
+
... "eche": "Hello case 1",
|
1780
|
+
... },
|
1781
|
+
... ],
|
1699
1782
|
... },
|
1700
1783
|
... {
|
1701
1784
|
... "case": "_",
|
1702
|
-
... "
|
1703
|
-
...
|
1704
|
-
...
|
1705
|
-
...
|
1785
|
+
... "stages": [
|
1786
|
+
... {
|
1787
|
+
... "name": "Stage else",
|
1788
|
+
... "eche": "Hello case else",
|
1789
|
+
... },
|
1790
|
+
... ],
|
1706
1791
|
... },
|
1707
1792
|
... ],
|
1708
1793
|
... }
|
@@ -1722,6 +1807,96 @@ class CaseStage(BaseStage):
|
|
1722
1807
|
alias="skip-not-match",
|
1723
1808
|
)
|
1724
1809
|
|
1810
|
+
def execute_case(
|
1811
|
+
self,
|
1812
|
+
case: str,
|
1813
|
+
stages: list[Stage],
|
1814
|
+
params: DictData,
|
1815
|
+
result: Result,
|
1816
|
+
*,
|
1817
|
+
event: Event | None = None,
|
1818
|
+
) -> Result:
|
1819
|
+
"""Execute case.
|
1820
|
+
|
1821
|
+
:param case: (str) A case that want to execution.
|
1822
|
+
:param stages: (list[Stage]) A list of stage.
|
1823
|
+
:param params: (DictData) A parameter data.
|
1824
|
+
:param result: (Result) A Result instance for return context and status.
|
1825
|
+
:param event: (Event) An Event manager instance that use to cancel this
|
1826
|
+
execution if it forces stopped by parent execution.
|
1827
|
+
|
1828
|
+
:rtype: Result
|
1829
|
+
"""
|
1830
|
+
context: DictData = copy.deepcopy(params)
|
1831
|
+
context.update({"case": case})
|
1832
|
+
output: DictData = {"case": case, "stages": {}}
|
1833
|
+
|
1834
|
+
for stage in stages:
|
1835
|
+
|
1836
|
+
if self.extras:
|
1837
|
+
stage.extras = self.extras
|
1838
|
+
|
1839
|
+
if stage.is_skipped(params=context):
|
1840
|
+
result.trace.info(f"... Skip stage: {stage.iden!r}")
|
1841
|
+
stage.set_outputs(output={"skipped": True}, to=output)
|
1842
|
+
continue
|
1843
|
+
|
1844
|
+
if event and event.is_set(): # pragma: no cov
|
1845
|
+
error_msg: str = (
|
1846
|
+
"Case-Stage was canceled from event that had set before "
|
1847
|
+
"stage case execution."
|
1848
|
+
)
|
1849
|
+
return result.catch(
|
1850
|
+
status=CANCEL,
|
1851
|
+
context={
|
1852
|
+
"case": case,
|
1853
|
+
"stages": filter_func(output.pop("stages", {})),
|
1854
|
+
"errors": StageException(error_msg).to_dict(),
|
1855
|
+
},
|
1856
|
+
)
|
1857
|
+
|
1858
|
+
try:
|
1859
|
+
rs: Result = stage.handler_execute(
|
1860
|
+
params=context,
|
1861
|
+
run_id=result.run_id,
|
1862
|
+
parent_run_id=result.parent_run_id,
|
1863
|
+
raise_error=True,
|
1864
|
+
event=event,
|
1865
|
+
)
|
1866
|
+
stage.set_outputs(rs.context, to=output)
|
1867
|
+
stage.set_outputs(stage.get_outputs(output), to=context)
|
1868
|
+
except (StageException, UtilException) as e: # pragma: no cov
|
1869
|
+
result.trace.error(f"[STAGE]: {e.__class__.__name__}: {e}")
|
1870
|
+
return result.catch(
|
1871
|
+
status=FAILED,
|
1872
|
+
context={
|
1873
|
+
"case": case,
|
1874
|
+
"stages": filter_func(output.pop("stages", {})),
|
1875
|
+
"errors": e.to_dict(),
|
1876
|
+
},
|
1877
|
+
)
|
1878
|
+
|
1879
|
+
if rs.status == FAILED:
|
1880
|
+
error_msg: str = (
|
1881
|
+
f"Case-Stage was break because it has a sub stage, "
|
1882
|
+
f"{stage.iden}, failed without raise error."
|
1883
|
+
)
|
1884
|
+
return result.catch(
|
1885
|
+
status=FAILED,
|
1886
|
+
context={
|
1887
|
+
"case": case,
|
1888
|
+
"stages": filter_func(output.pop("stages", {})),
|
1889
|
+
"errors": StageException(error_msg).to_dict(),
|
1890
|
+
},
|
1891
|
+
)
|
1892
|
+
return result.catch(
|
1893
|
+
status=SUCCESS,
|
1894
|
+
context={
|
1895
|
+
"case": case,
|
1896
|
+
"stages": filter_func(output.pop("stages", {})),
|
1897
|
+
},
|
1898
|
+
)
|
1899
|
+
|
1725
1900
|
def execute(
|
1726
1901
|
self,
|
1727
1902
|
params: DictData,
|
@@ -1731,11 +1906,10 @@ class CaseStage(BaseStage):
|
|
1731
1906
|
) -> Result:
|
1732
1907
|
"""Execute case-match condition that pass to the case field.
|
1733
1908
|
|
1734
|
-
:param params: A parameter
|
1735
|
-
:param result: (Result) A
|
1736
|
-
|
1737
|
-
|
1738
|
-
was not force stopped.
|
1909
|
+
:param params: (DictData) A parameter data.
|
1910
|
+
:param result: (Result) A Result instance for return context and status.
|
1911
|
+
:param event: (Event) An Event manager instance that use to cancel this
|
1912
|
+
execution if it forces stopped by parent execution.
|
1739
1913
|
|
1740
1914
|
:rtype: Result
|
1741
1915
|
"""
|
@@ -1751,17 +1925,17 @@ class CaseStage(BaseStage):
|
|
1751
1925
|
|
1752
1926
|
result.trace.info(f"[STAGE]: Case-Execute: {_case!r}.")
|
1753
1927
|
_else: Optional[Match] = None
|
1754
|
-
|
1928
|
+
stages: Optional[list[Stage]] = None
|
1755
1929
|
for match in self.match:
|
1756
1930
|
if (c := match.case) == "_":
|
1757
1931
|
_else: Match = match
|
1758
1932
|
continue
|
1759
1933
|
|
1760
1934
|
_condition: str = param2template(c, params, extras=self.extras)
|
1761
|
-
if
|
1762
|
-
|
1935
|
+
if stages is None and _case == _condition:
|
1936
|
+
stages: list[Stage] = match.stages
|
1763
1937
|
|
1764
|
-
if
|
1938
|
+
if stages is None:
|
1765
1939
|
if _else is None:
|
1766
1940
|
if not self.skip_not_match:
|
1767
1941
|
raise StageException(
|
@@ -1779,10 +1953,8 @@ class CaseStage(BaseStage):
|
|
1779
1953
|
status=CANCEL,
|
1780
1954
|
context={"errors": StageException(error_msg).to_dict()},
|
1781
1955
|
)
|
1782
|
-
|
1783
|
-
|
1784
|
-
if self.extras:
|
1785
|
-
stage.extras = self.extras
|
1956
|
+
_case: str = "_"
|
1957
|
+
stages: list[Stage] = _else.stages
|
1786
1958
|
|
1787
1959
|
if event and event.is_set(): # pragma: no cov
|
1788
1960
|
return result.catch(
|
@@ -1795,19 +1967,9 @@ class CaseStage(BaseStage):
|
|
1795
1967
|
},
|
1796
1968
|
)
|
1797
1969
|
|
1798
|
-
|
1799
|
-
|
1800
|
-
|
1801
|
-
context=stage.handler_execute(
|
1802
|
-
params=params,
|
1803
|
-
run_id=result.run_id,
|
1804
|
-
parent_run_id=result.parent_run_id,
|
1805
|
-
event=event,
|
1806
|
-
).context,
|
1807
|
-
)
|
1808
|
-
except StageException as e: # pragma: no cov
|
1809
|
-
result.trace.error(f"[STAGE]: {e.__class__.__name__}:" f"\n\t{e}")
|
1810
|
-
return result.catch(status=FAILED, context={"errors": e.to_dict()})
|
1970
|
+
return self.execute_case(
|
1971
|
+
case=_case, stages=stages, params=params, result=result, event=event
|
1972
|
+
)
|
1811
1973
|
|
1812
1974
|
|
1813
1975
|
class RaiseStage(BaseStage): # pragma: no cov
|
@@ -1824,7 +1986,7 @@ class RaiseStage(BaseStage): # pragma: no cov
|
|
1824
1986
|
|
1825
1987
|
message: str = Field(
|
1826
1988
|
description=(
|
1827
|
-
"An error message that want to raise with StageException class"
|
1989
|
+
"An error message that want to raise with `StageException` class"
|
1828
1990
|
),
|
1829
1991
|
alias="raise",
|
1830
1992
|
)
|
@@ -1838,11 +2000,10 @@ class RaiseStage(BaseStage): # pragma: no cov
|
|
1838
2000
|
) -> Result:
|
1839
2001
|
"""Raise the StageException object with the message field execution.
|
1840
2002
|
|
1841
|
-
:param params: A parameter
|
1842
|
-
:param result: (Result) A
|
1843
|
-
|
1844
|
-
|
1845
|
-
was not force stopped.
|
2003
|
+
:param params: (DictData) A parameter data.
|
2004
|
+
:param result: (Result) A Result instance for return context and status.
|
2005
|
+
:param event: (Event) An Event manager instance that use to cancel this
|
2006
|
+
execution if it forces stopped by parent execution.
|
1846
2007
|
"""
|
1847
2008
|
if result is None: # pragma: no cov
|
1848
2009
|
result: Result = Result(
|
@@ -1854,27 +2015,13 @@ class RaiseStage(BaseStage): # pragma: no cov
|
|
1854
2015
|
raise StageException(message)
|
1855
2016
|
|
1856
2017
|
|
1857
|
-
# TODO: Not implement this stages yet
|
1858
|
-
class HookStage(BaseStage): # pragma: no cov
|
1859
|
-
"""Hook stage execution."""
|
1860
|
-
|
1861
|
-
hook: str
|
1862
|
-
args: DictData = Field(default_factory=dict)
|
1863
|
-
callback: str
|
1864
|
-
|
1865
|
-
def execute(
|
1866
|
-
self,
|
1867
|
-
params: DictData,
|
1868
|
-
*,
|
1869
|
-
result: Result | None = None,
|
1870
|
-
event: Event | None = None,
|
1871
|
-
) -> Result:
|
1872
|
-
raise NotImplementedError("Hook Stage does not implement yet.")
|
1873
|
-
|
1874
|
-
|
1875
|
-
# TODO: Not implement this stages yet
|
1876
2018
|
class DockerStage(BaseStage): # pragma: no cov
|
1877
|
-
"""Docker container stage execution
|
2019
|
+
"""Docker container stage execution that will pull the specific Docker image
|
2020
|
+
with custom authentication and run this image by passing environment
|
2021
|
+
variables and mounting local volume to this Docker container.
|
2022
|
+
|
2023
|
+
The volume path that mount to this Docker container will limit. That is
|
2024
|
+
this stage does not allow you to mount any path to this container.
|
1878
2025
|
|
1879
2026
|
Data Validate:
|
1880
2027
|
>>> stage = {
|
@@ -1882,10 +2029,7 @@ class DockerStage(BaseStage): # pragma: no cov
|
|
1882
2029
|
... "image": "image-name.pkg.com",
|
1883
2030
|
... "env": {
|
1884
2031
|
... "ENV": "dev",
|
1885
|
-
... "
|
1886
|
-
... },
|
1887
|
-
... "volume": {
|
1888
|
-
... "secrets": "/secrets",
|
2032
|
+
... "SECRET": "${SPECIFIC_SECRET}",
|
1889
2033
|
... },
|
1890
2034
|
... "auth": {
|
1891
2035
|
... "username": "__json_key",
|
@@ -1898,8 +2042,16 @@ class DockerStage(BaseStage): # pragma: no cov
|
|
1898
2042
|
description="A Docker image url with tag that want to run.",
|
1899
2043
|
)
|
1900
2044
|
tag: str = Field(default="latest", description="An Docker image tag.")
|
1901
|
-
env: DictData = Field(
|
1902
|
-
|
2045
|
+
env: DictData = Field(
|
2046
|
+
default_factory=dict,
|
2047
|
+
description=(
|
2048
|
+
"An environment variable that want pass to Docker container.",
|
2049
|
+
),
|
2050
|
+
)
|
2051
|
+
volume: DictData = Field(
|
2052
|
+
default_factory=dict,
|
2053
|
+
description="A mapping of local and target mounting path.",
|
2054
|
+
)
|
1903
2055
|
auth: DictData = Field(
|
1904
2056
|
default_factory=dict,
|
1905
2057
|
description=(
|
@@ -1911,7 +2063,17 @@ class DockerStage(BaseStage): # pragma: no cov
|
|
1911
2063
|
self,
|
1912
2064
|
params: DictData,
|
1913
2065
|
result: Result,
|
1914
|
-
|
2066
|
+
event: Event | None = None,
|
2067
|
+
) -> Result:
|
2068
|
+
"""Execute Docker container task.
|
2069
|
+
|
2070
|
+
:param params: (DictData) A parameter data.
|
2071
|
+
:param result: (Result) A Result instance for return context and status.
|
2072
|
+
:param event: (Event) An Event manager instance that use to cancel this
|
2073
|
+
execution if it forces stopped by parent execution.
|
2074
|
+
|
2075
|
+
:rtype: Result
|
2076
|
+
"""
|
1915
2077
|
from docker import DockerClient
|
1916
2078
|
from docker.errors import ContainerError
|
1917
2079
|
|
@@ -1929,6 +2091,16 @@ class DockerStage(BaseStage): # pragma: no cov
|
|
1929
2091
|
for line in resp:
|
1930
2092
|
result.trace.info(f"... {line}")
|
1931
2093
|
|
2094
|
+
if event and event.is_set():
|
2095
|
+
error_msg: str = (
|
2096
|
+
"Docker-Stage was canceled from event that had set before "
|
2097
|
+
"run the Docker container."
|
2098
|
+
)
|
2099
|
+
return result.catch(
|
2100
|
+
status=CANCEL,
|
2101
|
+
context={"errors": StageException(error_msg).to_dict()},
|
2102
|
+
)
|
2103
|
+
|
1932
2104
|
unique_image_name: str = f"{self.image}_{datetime.now():%Y%m%d%H%M%S%f}"
|
1933
2105
|
container = client.containers.run(
|
1934
2106
|
image=f"{self.image}:{self.tag}",
|
@@ -1967,6 +2139,13 @@ class DockerStage(BaseStage): # pragma: no cov
|
|
1967
2139
|
f"{self.image}:{self.tag}",
|
1968
2140
|
out,
|
1969
2141
|
)
|
2142
|
+
output_file: Path = Path(f".docker.{result.run_id}.logs/outputs.json")
|
2143
|
+
if not output_file.exists():
|
2144
|
+
return result.catch(status=SUCCESS)
|
2145
|
+
|
2146
|
+
with output_file.open(mode="rt") as f:
|
2147
|
+
data = json.load(f)
|
2148
|
+
return result.catch(status=SUCCESS, context=data)
|
1970
2149
|
|
1971
2150
|
def execute(
|
1972
2151
|
self,
|
@@ -1975,13 +2154,34 @@ class DockerStage(BaseStage): # pragma: no cov
|
|
1975
2154
|
result: Result | None = None,
|
1976
2155
|
event: Event | None = None,
|
1977
2156
|
) -> Result:
|
2157
|
+
"""Execute the Docker image via Python API.
|
2158
|
+
|
2159
|
+
:param params: (DictData) A parameter data.
|
2160
|
+
:param result: (Result) A Result instance for return context and status.
|
2161
|
+
:param event: (Event) An Event manager instance that use to cancel this
|
2162
|
+
execution if it forces stopped by parent execution.
|
2163
|
+
|
2164
|
+
:rtype: Result
|
2165
|
+
"""
|
2166
|
+
if result is None: # pragma: no cov
|
2167
|
+
result: Result = Result(
|
2168
|
+
run_id=gen_id(self.name + (self.id or ""), unique=True)
|
2169
|
+
)
|
2170
|
+
|
2171
|
+
result.trace.info(f"[STAGE]: Docker-Execute: {self.image}:{self.tag}")
|
2172
|
+
|
1978
2173
|
raise NotImplementedError("Docker Stage does not implement yet.")
|
1979
2174
|
|
1980
2175
|
|
1981
|
-
# TODO: Not implement this stages yet
|
1982
2176
|
class VirtualPyStage(PyStage): # pragma: no cov
|
1983
|
-
"""
|
2177
|
+
"""Virtual Python stage executor that run Python statement on the dependent
|
2178
|
+
Python virtual environment via the `uv` package.
|
2179
|
+
"""
|
1984
2180
|
|
2181
|
+
version: str = Field(
|
2182
|
+
default="3.9",
|
2183
|
+
description="A Python version that want to run.",
|
2184
|
+
)
|
1985
2185
|
deps: list[str] = Field(
|
1986
2186
|
description=(
|
1987
2187
|
"list of Python dependency that want to install before execution "
|
@@ -1989,7 +2189,54 @@ class VirtualPyStage(PyStage): # pragma: no cov
|
|
1989
2189
|
),
|
1990
2190
|
)
|
1991
2191
|
|
1992
|
-
|
2192
|
+
@contextlib.contextmanager
|
2193
|
+
def create_py_file(
|
2194
|
+
self,
|
2195
|
+
py: str,
|
2196
|
+
values: DictData,
|
2197
|
+
deps: list[str],
|
2198
|
+
run_id: str | None = None,
|
2199
|
+
) -> Iterator[str]:
|
2200
|
+
"""Create the .py file with an input Python string statement.
|
2201
|
+
|
2202
|
+
:param py: A Python string statement.
|
2203
|
+
:param values: A variable that want to set before running this
|
2204
|
+
:param deps: An additional Python dependencies that want install before
|
2205
|
+
run this python stage.
|
2206
|
+
:param run_id: (str | None) A running ID of this stage execution.
|
2207
|
+
"""
|
2208
|
+
run_id: str = run_id or uuid.uuid4()
|
2209
|
+
f_name: str = f"{run_id}.py"
|
2210
|
+
with open(f"./{f_name}", mode="w", newline="\n") as f:
|
2211
|
+
# NOTE: Create variable mapping that write before running statement.
|
2212
|
+
vars_str: str = "\n ".join(
|
2213
|
+
f"{var} = {value!r}" for var, value in values.items()
|
2214
|
+
)
|
2215
|
+
|
2216
|
+
# NOTE: uv supports PEP 723 — inline TOML metadata.
|
2217
|
+
f.write(
|
2218
|
+
dedent(
|
2219
|
+
f"""
|
2220
|
+
# /// script
|
2221
|
+
# dependencies = [{', '.join(f'"{dep}"' for dep in deps)}]
|
2222
|
+
# ///
|
2223
|
+
{vars_str}
|
2224
|
+
""".strip(
|
2225
|
+
"\n"
|
2226
|
+
)
|
2227
|
+
)
|
2228
|
+
)
|
2229
|
+
|
2230
|
+
# NOTE: make sure that py script file does not have `\r` char.
|
2231
|
+
f.write("\n" + py.replace("\r\n", "\n"))
|
2232
|
+
|
2233
|
+
# NOTE: Make this .py file able to executable.
|
2234
|
+
make_exec(f"./{f_name}")
|
2235
|
+
|
2236
|
+
yield f_name
|
2237
|
+
|
2238
|
+
# Note: Remove .py file that use to run Python.
|
2239
|
+
Path(f"./{f_name}").unlink()
|
1993
2240
|
|
1994
2241
|
def execute(
|
1995
2242
|
self,
|
@@ -2001,15 +2248,13 @@ class VirtualPyStage(PyStage): # pragma: no cov
|
|
2001
2248
|
"""Execute the Python statement via Python virtual environment.
|
2002
2249
|
|
2003
2250
|
Steps:
|
2004
|
-
- Create python file.
|
2005
|
-
-
|
2006
|
-
- Execution python file with uv and specific `.venv`.
|
2251
|
+
- Create python file with the `uv` syntax.
|
2252
|
+
- Execution python file with `uv run` via Python subprocess module.
|
2007
2253
|
|
2008
|
-
:param params: A parameter
|
2009
|
-
:param result: (Result) A
|
2010
|
-
|
2011
|
-
|
2012
|
-
was not force stopped.
|
2254
|
+
:param params: (DictData) A parameter data.
|
2255
|
+
:param result: (Result) A Result instance for return context and status.
|
2256
|
+
:param event: (Event) An Event manager instance that use to cancel this
|
2257
|
+
execution if it forces stopped by parent execution.
|
2013
2258
|
|
2014
2259
|
:rtype: Result
|
2015
2260
|
"""
|
@@ -2019,13 +2264,45 @@ class VirtualPyStage(PyStage): # pragma: no cov
|
|
2019
2264
|
)
|
2020
2265
|
|
2021
2266
|
result.trace.info(f"[STAGE]: Py-Virtual-Execute: {self.name}")
|
2022
|
-
|
2023
|
-
|
2267
|
+
run: str = param2template(dedent(self.run), params, extras=self.extras)
|
2268
|
+
with self.create_py_file(
|
2269
|
+
py=run,
|
2270
|
+
values=param2template(self.vars, params, extras=self.extras),
|
2271
|
+
deps=param2template(self.deps, params, extras=self.extras),
|
2272
|
+
run_id=result.run_id,
|
2273
|
+
) as py:
|
2274
|
+
result.trace.debug(f"... Create `{py}` file.")
|
2275
|
+
rs: CompletedProcess = subprocess.run(
|
2276
|
+
["uv", "run", py, "--no-cache"],
|
2277
|
+
# ["uv", "run", "--python", "3.9", py],
|
2278
|
+
shell=False,
|
2279
|
+
capture_output=True,
|
2280
|
+
text=True,
|
2281
|
+
)
|
2282
|
+
|
2283
|
+
if rs.returncode > 0:
|
2284
|
+
# NOTE: Prepare stderr message that returning from subprocess.
|
2285
|
+
e: str = (
|
2286
|
+
rs.stderr.encode("utf-8").decode("utf-16")
|
2287
|
+
if "\\x00" in rs.stderr
|
2288
|
+
else rs.stderr
|
2289
|
+
).removesuffix("\n")
|
2290
|
+
raise StageException(
|
2291
|
+
f"Subprocess: {e}\nRunning Statement:\n---\n"
|
2292
|
+
f"```python\n{run}\n```"
|
2293
|
+
)
|
2294
|
+
return result.catch(
|
2295
|
+
status=SUCCESS,
|
2296
|
+
context={
|
2297
|
+
"return_code": rs.returncode,
|
2298
|
+
"stdout": None if (out := rs.stdout.strip("\n")) == "" else out,
|
2299
|
+
"stderr": None if (out := rs.stderr.strip("\n")) == "" else out,
|
2300
|
+
},
|
2024
2301
|
)
|
2025
2302
|
|
2026
2303
|
|
2027
2304
|
# NOTE:
|
2028
|
-
# An order of parsing stage model on the Job model with
|
2305
|
+
# An order of parsing stage model on the Job model with `stages` field.
|
2029
2306
|
# From the current build-in stages, they do not have stage that have the same
|
2030
2307
|
# fields that because of parsing on the Job's stages key.
|
2031
2308
|
#
|
@@ -2034,7 +2311,6 @@ Stage = Annotated[
|
|
2034
2311
|
DockerStage,
|
2035
2312
|
BashStage,
|
2036
2313
|
CallStage,
|
2037
|
-
HookStage,
|
2038
2314
|
TriggerStage,
|
2039
2315
|
ForEachStage,
|
2040
2316
|
UntilStage,
|