ddeutil-workflow 0.0.53__py3-none-any.whl → 0.0.55__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ddeutil/workflow/__about__.py +1 -1
- ddeutil/workflow/api/__init__.py +170 -1
- ddeutil/workflow/api/routes/job.py +23 -22
- ddeutil/workflow/api/routes/schedules.py +0 -2
- ddeutil/workflow/api/routes/workflows.py +3 -4
- ddeutil/workflow/job.py +125 -170
- ddeutil/workflow/result.py +1 -0
- ddeutil/workflow/scheduler.py +1 -3
- ddeutil/workflow/stages.py +641 -399
- ddeutil/workflow/utils.py +5 -4
- ddeutil/workflow/workflow.py +118 -258
- {ddeutil_workflow-0.0.53.dist-info → ddeutil_workflow-0.0.55.dist-info}/METADATA +5 -13
- ddeutil_workflow-0.0.55.dist-info/RECORD +30 -0
- ddeutil/workflow/api/api.py +0 -170
- ddeutil_workflow-0.0.53.dist-info/RECORD +0 -31
- /ddeutil/workflow/api/{log.py → logs.py} +0 -0
- /ddeutil/workflow/api/{repeat.py → utils.py} +0 -0
- {ddeutil_workflow-0.0.53.dist-info → ddeutil_workflow-0.0.55.dist-info}/WHEEL +0 -0
- {ddeutil_workflow-0.0.53.dist-info → ddeutil_workflow-0.0.55.dist-info}/licenses/LICENSE +0 -0
- {ddeutil_workflow-0.0.53.dist-info → ddeutil_workflow-0.0.55.dist-info}/top_level.txt +0 -0
ddeutil/workflow/stages.py
CHANGED
@@ -4,23 +4,27 @@
|
|
4
4
|
# license information.
|
5
5
|
# ------------------------------------------------------------------------------
|
6
6
|
# [x] Use dynamic config
|
7
|
-
"""
|
8
|
-
|
9
|
-
(same thread at its job owner) that mean it is the lowest executor
|
10
|
-
|
7
|
+
"""Stages module include all stage model that use be the minimum execution layer
|
8
|
+
of this workflow engine. The stage handle the minimize task that run in some
|
9
|
+
thread (same thread at its job owner) that mean it is the lowest executor that
|
10
|
+
you can track logs.
|
11
11
|
|
12
|
-
The output of stage execution only return
|
13
|
-
handle stage error on this stage
|
14
|
-
use-case, and it does not worry
|
12
|
+
The output of stage execution only return SUCCESS or CANCEL status because
|
13
|
+
I do not want to handle stage error on this stage execution. I think stage model
|
14
|
+
have a lot of use-case, and it should does not worry about it error output.
|
15
15
|
|
16
|
-
|
16
|
+
So, I will create `handler_execute` for any exception class that raise from
|
17
|
+
the stage execution method.
|
17
18
|
|
18
|
-
|
19
|
-
|
19
|
+
Execution --> Ok ---( handler )--> Result with `SUCCESS` or `CANCEL`
|
20
|
+
|
21
|
+
--> Error ┬--( handler )-> Result with `FAILED` (Set `raise_error` flag)
|
22
|
+
|
|
23
|
+
╰--( handler )-> Raise StageException(...)
|
20
24
|
|
21
25
|
On the context I/O that pass to a stage object at execute process. The
|
22
|
-
execute method receives a `params={"params": {...}}` value for
|
23
|
-
|
26
|
+
execute method receives a `params={"params": {...}}` value for passing template
|
27
|
+
searching.
|
24
28
|
"""
|
25
29
|
from __future__ import annotations
|
26
30
|
|
@@ -28,12 +32,13 @@ import asyncio
|
|
28
32
|
import contextlib
|
29
33
|
import copy
|
30
34
|
import inspect
|
35
|
+
import json
|
31
36
|
import subprocess
|
32
37
|
import sys
|
33
38
|
import time
|
34
39
|
import uuid
|
35
40
|
from abc import ABC, abstractmethod
|
36
|
-
from collections.abc import Iterator
|
41
|
+
from collections.abc import AsyncIterator, Iterator
|
37
42
|
from concurrent.futures import (
|
38
43
|
FIRST_EXCEPTION,
|
39
44
|
Future,
|
@@ -59,6 +64,7 @@ from .exceptions import StageException, UtilException, to_dict
|
|
59
64
|
from .result import CANCEL, FAILED, SUCCESS, WAIT, Result, Status
|
60
65
|
from .reusables import TagFunc, extract_call, not_in_template, param2template
|
61
66
|
from .utils import (
|
67
|
+
NEWLINE,
|
62
68
|
delay,
|
63
69
|
filter_func,
|
64
70
|
gen_id,
|
@@ -66,20 +72,22 @@ from .utils import (
|
|
66
72
|
)
|
67
73
|
|
68
74
|
T = TypeVar("T")
|
75
|
+
StrOrInt = Union[str, int]
|
69
76
|
|
70
77
|
|
71
78
|
class BaseStage(BaseModel, ABC):
|
72
|
-
"""Base Stage Model that keep only
|
73
|
-
metadata. If you want to implement any custom
|
74
|
-
|
79
|
+
"""Base Stage Model that keep only necessary fields like `id`, `name` or
|
80
|
+
`condition` for the stage metadata. If you want to implement any custom
|
81
|
+
stage, you can inherit this class and implement `self.execute()` method
|
82
|
+
only.
|
75
83
|
|
76
|
-
This class is the abstraction class for any stage model that
|
77
|
-
|
84
|
+
This class is the abstraction class for any inherit stage model that
|
85
|
+
want to implement on this workflow package.
|
78
86
|
"""
|
79
87
|
|
80
88
|
extras: DictData = Field(
|
81
89
|
default_factory=dict,
|
82
|
-
description="An extra override config values.",
|
90
|
+
description="An extra parameter that override core config values.",
|
83
91
|
)
|
84
92
|
id: Optional[str] = Field(
|
85
93
|
default=None,
|
@@ -93,14 +101,17 @@ class BaseStage(BaseModel, ABC):
|
|
93
101
|
)
|
94
102
|
condition: Optional[str] = Field(
|
95
103
|
default=None,
|
96
|
-
description=
|
104
|
+
description=(
|
105
|
+
"A stage condition statement to allow stage executable. This field "
|
106
|
+
"alise with `if` field."
|
107
|
+
),
|
97
108
|
alias="if",
|
98
109
|
)
|
99
110
|
|
100
111
|
@property
|
101
112
|
def iden(self) -> str:
|
102
|
-
"""Return
|
103
|
-
|
113
|
+
"""Return this stage identity that return the `id` field first and if
|
114
|
+
this `id` field does not set, it will use the `name` field instead.
|
104
115
|
|
105
116
|
:rtype: str
|
106
117
|
"""
|
@@ -156,8 +167,8 @@ class BaseStage(BaseModel, ABC):
|
|
156
167
|
run_id: str | None = None,
|
157
168
|
parent_run_id: str | None = None,
|
158
169
|
result: Result | None = None,
|
159
|
-
raise_error: bool | None = None,
|
160
170
|
event: Event | None = None,
|
171
|
+
raise_error: bool | None = None,
|
161
172
|
) -> Result | DictData:
|
162
173
|
"""Handler stage execution result from the stage `execute` method.
|
163
174
|
|
@@ -170,7 +181,7 @@ class BaseStage(BaseModel, ABC):
|
|
170
181
|
╰-context:
|
171
182
|
╰-outputs: ...
|
172
183
|
|
173
|
-
--> Error --> Result (if
|
184
|
+
--> Error --> Result (if `raise_error` was set)
|
174
185
|
|-status: FAILED
|
175
186
|
╰-errors:
|
176
187
|
|-class: ...
|
@@ -179,18 +190,16 @@ class BaseStage(BaseModel, ABC):
|
|
179
190
|
|
180
191
|
--> Error --> Raise StageException(...)
|
181
192
|
|
182
|
-
On the last step, it will set the running ID on a return result
|
183
|
-
from current stage ID before release the final result.
|
193
|
+
On the last step, it will set the running ID on a return result
|
194
|
+
object from the current stage ID before release the final result.
|
184
195
|
|
185
|
-
:param params: (DictData) A
|
186
|
-
|
187
|
-
:param
|
188
|
-
:param parent_run_id: (str) A parent workflow running ID for this
|
189
|
-
execution.
|
196
|
+
:param params: (DictData) A parameter data.
|
197
|
+
:param run_id: (str) A running stage ID.
|
198
|
+
:param parent_run_id: (str) A parent running ID.
|
190
199
|
:param result: (Result) A result object for keeping context and status
|
191
200
|
data before execution.
|
192
|
-
:param raise_error: (bool) A flag that all this method raise error
|
193
201
|
:param event: (Event) An event manager that pass to the stage execution.
|
202
|
+
:param raise_error: (bool) A flag that all this method raise error
|
194
203
|
|
195
204
|
:rtype: Result
|
196
205
|
"""
|
@@ -203,51 +212,56 @@ class BaseStage(BaseModel, ABC):
|
|
203
212
|
)
|
204
213
|
|
205
214
|
try:
|
206
|
-
|
207
|
-
return rs
|
215
|
+
return self.execute(params, result=result, event=event)
|
208
216
|
except Exception as e:
|
209
|
-
|
210
|
-
|
217
|
+
e_name: str = e.__class__.__name__
|
218
|
+
result.trace.error(f"[STAGE]: Handler:{NEWLINE}{e_name}: {e}")
|
211
219
|
if dynamic("stage_raise_error", f=raise_error, extras=self.extras):
|
212
220
|
if isinstance(e, StageException):
|
213
221
|
raise
|
214
222
|
|
215
223
|
raise StageException(
|
216
|
-
f"{self.__class__.__name__}:
|
217
|
-
f"{e.__class__.__name__}: {e}"
|
224
|
+
f"{self.__class__.__name__}: {NEWLINE}{e_name}: {e}"
|
218
225
|
) from e
|
219
226
|
|
220
|
-
|
221
|
-
return result.catch(status=FAILED, context=errors)
|
227
|
+
return result.catch(status=FAILED, context={"errors": to_dict(e)})
|
222
228
|
|
223
229
|
def set_outputs(self, output: DictData, to: DictData) -> DictData:
|
224
|
-
"""Set an outputs from execution
|
225
|
-
|
230
|
+
"""Set an outputs from execution result context to the received context
|
231
|
+
with a `to` input parameter. The result context from stage execution
|
232
|
+
will be set with `outputs` key in this stage ID key.
|
226
233
|
|
227
234
|
For example of setting output method, If you receive execute output
|
228
235
|
and want to set on the `to` like;
|
229
236
|
|
230
|
-
... (i) output: {'foo': bar}
|
237
|
+
... (i) output: {'foo': 'bar', 'skipped': True}
|
231
238
|
... (ii) to: {'stages': {}}
|
232
239
|
|
233
|
-
The
|
240
|
+
The received context in the `to` argument will be;
|
234
241
|
|
235
242
|
... (iii) to: {
|
236
243
|
'stages': {
|
237
244
|
'<stage-id>': {
|
238
245
|
'outputs': {'foo': 'bar'},
|
239
|
-
'skipped':
|
246
|
+
'skipped': True,
|
240
247
|
}
|
241
248
|
}
|
242
249
|
}
|
243
250
|
|
251
|
+
The keys that will set to the received context is `outputs`,
|
252
|
+
`errors`, and `skipped` keys. The `errors` and `skipped` keys will
|
253
|
+
extract from the result context if it exists. If it does not found, it
|
254
|
+
will not set on the received context.
|
255
|
+
|
244
256
|
Important:
|
257
|
+
|
245
258
|
This method is use for reconstruct the result context and transfer
|
246
|
-
to the `to` argument.
|
259
|
+
to the `to` argument. The result context was soft copied before set
|
260
|
+
output step.
|
247
261
|
|
248
|
-
:param output: (DictData)
|
249
|
-
|
250
|
-
:param to: (DictData) A context data
|
262
|
+
:param output: (DictData) A result data context that want to extract
|
263
|
+
and transfer to the `outputs` key in receive context.
|
264
|
+
:param to: (DictData) A received context data.
|
251
265
|
|
252
266
|
:rtype: DictData
|
253
267
|
"""
|
@@ -282,11 +296,12 @@ class BaseStage(BaseModel, ABC):
|
|
282
296
|
}
|
283
297
|
return to
|
284
298
|
|
285
|
-
def get_outputs(self,
|
299
|
+
def get_outputs(self, output: DictData) -> DictData:
|
286
300
|
"""Get the outputs from stages data. It will get this stage ID from
|
287
301
|
the stage outputs mapping.
|
288
302
|
|
289
|
-
:param
|
303
|
+
:param output: (DictData) A stage output context that want to get this
|
304
|
+
stage ID `outputs` key.
|
290
305
|
|
291
306
|
:rtype: DictData
|
292
307
|
"""
|
@@ -296,33 +311,31 @@ class BaseStage(BaseModel, ABC):
|
|
296
311
|
return {}
|
297
312
|
|
298
313
|
_id: str = (
|
299
|
-
param2template(self.id, params=
|
314
|
+
param2template(self.id, params=output, extras=self.extras)
|
300
315
|
if self.id
|
301
316
|
else gen_id(
|
302
|
-
param2template(self.name, params=
|
317
|
+
param2template(self.name, params=output, extras=self.extras)
|
303
318
|
)
|
304
319
|
)
|
305
|
-
return
|
320
|
+
return output.get("stages", {}).get(_id, {}).get("outputs", {})
|
306
321
|
|
307
|
-
def is_skipped(self, params: DictData
|
322
|
+
def is_skipped(self, params: DictData) -> bool:
|
308
323
|
"""Return true if condition of this stage do not correct. This process
|
309
324
|
use build-in eval function to execute the if-condition.
|
310
325
|
|
326
|
+
:param params: (DictData) A parameters that want to pass to condition
|
327
|
+
template.
|
328
|
+
|
311
329
|
:raise StageException: When it has any error raise from the eval
|
312
330
|
condition statement.
|
313
331
|
:raise StageException: When return type of the eval condition statement
|
314
332
|
does not return with boolean type.
|
315
333
|
|
316
|
-
:param params: (DictData) A parameters that want to pass to condition
|
317
|
-
template.
|
318
|
-
|
319
334
|
:rtype: bool
|
320
335
|
"""
|
321
336
|
if self.condition is None:
|
322
337
|
return False
|
323
338
|
|
324
|
-
params: DictData = {} if params is None else params
|
325
|
-
|
326
339
|
try:
|
327
340
|
# WARNING: The eval build-in function is very dangerous. So, it
|
328
341
|
# should use the `re` module to validate eval-string before
|
@@ -340,7 +353,14 @@ class BaseStage(BaseModel, ABC):
|
|
340
353
|
|
341
354
|
|
342
355
|
class BaseAsyncStage(BaseStage):
|
343
|
-
"""Base Async Stage model
|
356
|
+
"""Base Async Stage model to make any stage model allow async execution for
|
357
|
+
optimize CPU and Memory on the current node. If you want to implement any
|
358
|
+
custom async stage, you can inherit this class and implement
|
359
|
+
`self.axecute()` (async + execute = axecute) method only.
|
360
|
+
|
361
|
+
This class is the abstraction class for any inherit asyncable stage
|
362
|
+
model.
|
363
|
+
"""
|
344
364
|
|
345
365
|
@abstractmethod
|
346
366
|
def execute(
|
@@ -385,20 +405,18 @@ class BaseAsyncStage(BaseStage):
|
|
385
405
|
run_id: str | None = None,
|
386
406
|
parent_run_id: str | None = None,
|
387
407
|
result: Result | None = None,
|
388
|
-
raise_error: bool | None = None,
|
389
408
|
event: Event | None = None,
|
409
|
+
raise_error: bool | None = None,
|
390
410
|
) -> Result:
|
391
411
|
"""Async Handler stage execution result from the stage `execute` method.
|
392
412
|
|
393
|
-
:param params: (DictData) A
|
394
|
-
|
395
|
-
:param
|
396
|
-
:param
|
397
|
-
|
398
|
-
|
399
|
-
data before execution.
|
413
|
+
:param params: (DictData) A parameter data.
|
414
|
+
:param run_id: (str) A stage running ID.
|
415
|
+
:param parent_run_id: (str) A parent job running ID.
|
416
|
+
:param result: (Result) A Result instance for return context and status.
|
417
|
+
:param event: (Event) An Event manager instance that use to cancel this
|
418
|
+
execution if it forces stopped by parent execution.
|
400
419
|
:param raise_error: (bool) A flag that all this method raise error
|
401
|
-
:param event: (Event) An event manager that pass to the stage execution.
|
402
420
|
|
403
421
|
:rtype: Result
|
404
422
|
"""
|
@@ -414,24 +432,26 @@ class BaseAsyncStage(BaseStage):
|
|
414
432
|
rs: Result = await self.axecute(params, result=result, event=event)
|
415
433
|
return rs
|
416
434
|
except Exception as e: # pragma: no cov
|
417
|
-
|
418
|
-
|
435
|
+
e_name: str = e.__class__.__name__
|
436
|
+
await result.trace.aerror(f"[STAGE]: Handler {e_name}: {e}")
|
419
437
|
if dynamic("stage_raise_error", f=raise_error, extras=self.extras):
|
420
438
|
if isinstance(e, StageException):
|
421
439
|
raise
|
422
440
|
|
423
441
|
raise StageException(
|
424
|
-
f"{self.__class__.__name__}:
|
425
|
-
f"{e.__class__.__name__}: {e}"
|
442
|
+
f"{self.__class__.__name__}: {NEWLINE}{e_name}: {e}"
|
426
443
|
) from None
|
427
444
|
|
428
|
-
|
429
|
-
return result.catch(status=FAILED, context=errors)
|
445
|
+
return result.catch(status=FAILED, context={"errors": to_dict(e)})
|
430
446
|
|
431
447
|
|
432
448
|
class EmptyStage(BaseAsyncStage):
|
433
|
-
"""Empty stage that do nothing
|
434
|
-
|
449
|
+
"""Empty stage executor that do nothing and log the `message` field to
|
450
|
+
stdout only. It can use for tracking a template parameter on the workflow or
|
451
|
+
debug step.
|
452
|
+
|
453
|
+
You can pass a sleep value in second unit to this stage for waiting
|
454
|
+
after log message.
|
435
455
|
|
436
456
|
Data Validate:
|
437
457
|
>>> stage = {
|
@@ -443,11 +463,14 @@ class EmptyStage(BaseAsyncStage):
|
|
443
463
|
|
444
464
|
echo: Optional[str] = Field(
|
445
465
|
default=None,
|
446
|
-
description="A
|
466
|
+
description="A message that want to show on the stdout.",
|
447
467
|
)
|
448
468
|
sleep: float = Field(
|
449
469
|
default=0,
|
450
|
-
description=
|
470
|
+
description=(
|
471
|
+
"A second value to sleep before start execution. This value should "
|
472
|
+
"gather or equal 0, and less than 1800 seconds."
|
473
|
+
),
|
451
474
|
ge=0,
|
452
475
|
lt=1800,
|
453
476
|
)
|
@@ -460,18 +483,15 @@ class EmptyStage(BaseAsyncStage):
|
|
460
483
|
event: Event | None = None,
|
461
484
|
) -> Result:
|
462
485
|
"""Execution method for the Empty stage that do only logging out to
|
463
|
-
stdout.
|
464
|
-
it does not get any error from logging function.
|
486
|
+
stdout.
|
465
487
|
|
466
488
|
The result context should be empty and do not process anything
|
467
489
|
without calling logging function.
|
468
490
|
|
469
|
-
:param params: (DictData) A
|
470
|
-
|
471
|
-
:param
|
472
|
-
|
473
|
-
:param event: (Event) An event manager that use to track parent execute
|
474
|
-
was not force stopped.
|
491
|
+
:param params: (DictData) A parameter data.
|
492
|
+
:param result: (Result) A Result instance for return context and status.
|
493
|
+
:param event: (Event) An Event manager instance that use to cancel this
|
494
|
+
execution if it forces stopped by parent execution.
|
475
495
|
|
476
496
|
:rtype: Result
|
477
497
|
"""
|
@@ -484,21 +504,18 @@ class EmptyStage(BaseAsyncStage):
|
|
484
504
|
message: str = "..."
|
485
505
|
else:
|
486
506
|
message: str = param2template(
|
487
|
-
dedent(self.echo), params, extras=self.extras
|
507
|
+
dedent(self.echo.strip("\n")), params, extras=self.extras
|
488
508
|
)
|
489
|
-
if "\n" in
|
490
|
-
message: str =
|
491
|
-
"\n"
|
492
|
-
)
|
509
|
+
if "\n" in message:
|
510
|
+
message: str = NEWLINE + message.replace("\n", NEWLINE)
|
493
511
|
|
494
512
|
result.trace.info(
|
495
513
|
f"[STAGE]: Empty-Execute: {self.name!r}: ( {message} )"
|
496
514
|
)
|
497
515
|
if self.sleep > 0:
|
498
516
|
if self.sleep > 5:
|
499
|
-
result.trace.info(f"[STAGE]: ... sleep ({self.sleep}
|
517
|
+
result.trace.info(f"[STAGE]: ... sleep ({self.sleep} sec)")
|
500
518
|
time.sleep(self.sleep)
|
501
|
-
|
502
519
|
return result.catch(status=SUCCESS)
|
503
520
|
|
504
521
|
async def axecute(
|
@@ -511,44 +528,48 @@ class EmptyStage(BaseAsyncStage):
|
|
511
528
|
"""Async execution method for this Empty stage that only logging out to
|
512
529
|
stdout.
|
513
530
|
|
514
|
-
:param params: (DictData) A
|
515
|
-
|
516
|
-
:param
|
517
|
-
|
518
|
-
:param event: (Event) An event manager that use to track parent execute
|
519
|
-
was not force stopped.
|
531
|
+
:param params: (DictData) A parameter data.
|
532
|
+
:param result: (Result) A Result instance for return context and status.
|
533
|
+
:param event: (Event) An Event manager instance that use to cancel this
|
534
|
+
execution if it forces stopped by parent execution.
|
520
535
|
|
521
536
|
:rtype: Result
|
522
537
|
"""
|
523
|
-
|
524
|
-
|
525
|
-
|
526
|
-
|
538
|
+
result: Result = result or Result(
|
539
|
+
run_id=gen_id(self.name + (self.id or ""), unique=True),
|
540
|
+
extras=self.extras,
|
541
|
+
)
|
542
|
+
|
543
|
+
if not self.echo:
|
544
|
+
message: str = "..."
|
545
|
+
else:
|
546
|
+
message: str = param2template(
|
547
|
+
dedent(self.echo.strip("\n")), params, extras=self.extras
|
527
548
|
)
|
549
|
+
if "\n" in message:
|
550
|
+
message: str = NEWLINE + message.replace("\n", NEWLINE)
|
528
551
|
|
529
|
-
|
530
|
-
f"[STAGE]: Empty-Execute: {self.name!r}: "
|
531
|
-
f"( {param2template(self.echo, params, extras=self.extras) or '...'} )"
|
552
|
+
result.trace.info(
|
553
|
+
f"[STAGE]: Empty-Execute: {self.name!r}: ( {message} )"
|
532
554
|
)
|
533
|
-
|
534
555
|
if self.sleep > 0:
|
535
556
|
if self.sleep > 5:
|
536
557
|
await result.trace.ainfo(
|
537
|
-
f"[STAGE]: ... sleep ({self.sleep}
|
558
|
+
f"[STAGE]: ... sleep ({self.sleep} sec)"
|
538
559
|
)
|
539
560
|
await asyncio.sleep(self.sleep)
|
540
|
-
|
541
561
|
return result.catch(status=SUCCESS)
|
542
562
|
|
543
563
|
|
544
564
|
class BashStage(BaseStage):
|
545
|
-
"""Bash
|
546
|
-
If your current OS is Windows, it will run on the bash
|
565
|
+
"""Bash stage executor that execute bash script on the current OS.
|
566
|
+
If your current OS is Windows, it will run on the bash from the current WSL.
|
567
|
+
It will use `bash` for Windows OS and use `sh` for Linux OS.
|
547
568
|
|
548
|
-
|
549
|
-
subprocess package. It does not good enough to use multiline
|
550
|
-
Thus,
|
551
|
-
issue.
|
569
|
+
This stage has some limitation when it runs shell statement with the
|
570
|
+
built-in subprocess package. It does not good enough to use multiline
|
571
|
+
statement. Thus, it will write the `.sh` file before start running bash
|
572
|
+
command for fix this issue.
|
552
573
|
|
553
574
|
Data Validate:
|
554
575
|
>>> stage = {
|
@@ -560,15 +581,46 @@ class BashStage(BaseStage):
|
|
560
581
|
... }
|
561
582
|
"""
|
562
583
|
|
563
|
-
bash: str = Field(
|
584
|
+
bash: str = Field(
|
585
|
+
description=(
|
586
|
+
"A bash statement that want to execute via Python subprocess."
|
587
|
+
)
|
588
|
+
)
|
564
589
|
env: DictStr = Field(
|
565
590
|
default_factory=dict,
|
566
591
|
description=(
|
567
|
-
"An environment variables that set before
|
568
|
-
"on the header of the `.sh` file."
|
592
|
+
"An environment variables that set before run bash command. It "
|
593
|
+
"will add on the header of the `.sh` file."
|
569
594
|
),
|
570
595
|
)
|
571
596
|
|
597
|
+
@contextlib.asynccontextmanager
|
598
|
+
async def acreate_sh_file(
|
599
|
+
self, bash: str, env: DictStr, run_id: str | None = None
|
600
|
+
) -> AsyncIterator: # pragma no cov
|
601
|
+
import aiofiles
|
602
|
+
|
603
|
+
f_name: str = f"{run_id or uuid.uuid4()}.sh"
|
604
|
+
f_shebang: str = "bash" if sys.platform.startswith("win") else "sh"
|
605
|
+
|
606
|
+
async with aiofiles.open(f"./{f_name}", mode="w", newline="\n") as f:
|
607
|
+
# NOTE: write header of `.sh` file
|
608
|
+
await f.write(f"#!/bin/{f_shebang}\n\n")
|
609
|
+
|
610
|
+
# NOTE: add setting environment variable before bash skip statement.
|
611
|
+
await f.writelines([f"{k}='{env[k]}';\n" for k in env])
|
612
|
+
|
613
|
+
# NOTE: make sure that shell script file does not have `\r` char.
|
614
|
+
await f.write("\n" + bash.replace("\r\n", "\n"))
|
615
|
+
|
616
|
+
# NOTE: Make this .sh file able to executable.
|
617
|
+
make_exec(f"./{f_name}")
|
618
|
+
|
619
|
+
yield [f_shebang, f_name]
|
620
|
+
|
621
|
+
# Note: Remove .sh file that use to run bash.
|
622
|
+
Path(f"./{f_name}").unlink()
|
623
|
+
|
572
624
|
@contextlib.contextmanager
|
573
625
|
def create_sh_file(
|
574
626
|
self, bash: str, env: DictStr, run_id: str | None = None
|
@@ -577,16 +629,14 @@ class BashStage(BaseStage):
|
|
577
629
|
step will write the `.sh` file before giving this file name to context.
|
578
630
|
After that, it will auto delete this file automatic.
|
579
631
|
|
580
|
-
:param bash: (str) A bash statement
|
581
|
-
:param env: (DictStr) An environment variable that
|
582
|
-
statement.
|
632
|
+
:param bash: (str) A bash statement.
|
633
|
+
:param env: (DictStr) An environment variable that set before run bash.
|
583
634
|
:param run_id: (str | None) A running stage ID that use for writing sh
|
584
635
|
file instead generate by UUID4.
|
585
636
|
|
586
637
|
:rtype: Iterator[TupleStr]
|
587
638
|
"""
|
588
|
-
|
589
|
-
f_name: str = f"{run_id}.sh"
|
639
|
+
f_name: str = f"{run_id or uuid.uuid4()}.sh"
|
590
640
|
f_shebang: str = "bash" if sys.platform.startswith("win") else "sh"
|
591
641
|
|
592
642
|
with open(f"./{f_name}", mode="w", newline="\n") as f:
|
@@ -614,28 +664,28 @@ class BashStage(BaseStage):
|
|
614
664
|
result: Result | None = None,
|
615
665
|
event: Event | None = None,
|
616
666
|
) -> Result:
|
617
|
-
"""Execute
|
618
|
-
|
667
|
+
"""Execute bash statement with the Python build-in `subprocess` package.
|
668
|
+
It will catch result from the `subprocess.run` returning output like
|
669
|
+
`return_code`, `stdout`, and `stderr`.
|
619
670
|
|
620
|
-
:param params: A parameter data
|
621
|
-
:param result: (Result) A
|
622
|
-
|
623
|
-
|
624
|
-
was not force stopped.
|
671
|
+
:param params: (DictData) A parameter data.
|
672
|
+
:param result: (Result) A Result instance for return context and status.
|
673
|
+
:param event: (Event) An Event manager instance that use to cancel this
|
674
|
+
execution if it forces stopped by parent execution.
|
625
675
|
|
626
676
|
:rtype: Result
|
627
677
|
"""
|
628
|
-
|
629
|
-
|
630
|
-
|
631
|
-
|
632
|
-
|
678
|
+
result: Result = result or Result(
|
679
|
+
run_id=gen_id(self.name + (self.id or ""), unique=True),
|
680
|
+
extras=self.extras,
|
681
|
+
)
|
682
|
+
|
683
|
+
result.trace.info(f"[STAGE]: Shell-Execute: {self.name}")
|
633
684
|
|
634
685
|
bash: str = param2template(
|
635
|
-
dedent(self.bash), params, extras=self.extras
|
686
|
+
dedent(self.bash.strip("\n")), params, extras=self.extras
|
636
687
|
)
|
637
688
|
|
638
|
-
result.trace.info(f"[STAGE]: Shell-Execute: {self.name}")
|
639
689
|
with self.create_sh_file(
|
640
690
|
bash=bash,
|
641
691
|
env=param2template(self.env, params, extras=self.extras),
|
@@ -654,7 +704,7 @@ class BashStage(BaseStage):
|
|
654
704
|
else rs.stderr
|
655
705
|
).removesuffix("\n")
|
656
706
|
raise StageException(
|
657
|
-
f"Subprocess: {e}\
|
707
|
+
f"Subprocess: {e}\n---( statement )---\n"
|
658
708
|
f"```bash\n{bash}\n```"
|
659
709
|
)
|
660
710
|
return result.catch(
|
@@ -668,18 +718,24 @@ class BashStage(BaseStage):
|
|
668
718
|
|
669
719
|
|
670
720
|
class PyStage(BaseStage):
|
671
|
-
"""Python
|
672
|
-
|
721
|
+
"""Python stage that running the Python statement with the current globals
|
722
|
+
and passing an input additional variables via `exec` built-in function.
|
673
723
|
|
674
724
|
This stage allow you to use any Python object that exists on the globals
|
675
725
|
such as import your installed package.
|
676
726
|
|
727
|
+
Warning:
|
728
|
+
|
729
|
+
The exec build-in function is very dangerous. So, it should use the `re`
|
730
|
+
module to validate exec-string before running or exclude the `os` package
|
731
|
+
from the current globals variable.
|
732
|
+
|
677
733
|
Data Validate:
|
678
734
|
>>> stage = {
|
679
735
|
... "name": "Python stage execution",
|
680
|
-
... "run": 'print("Hello {
|
736
|
+
... "run": 'print(f"Hello {VARIABLE}")',
|
681
737
|
... "vars": {
|
682
|
-
... "
|
738
|
+
... "VARIABLE": "WORLD",
|
683
739
|
... },
|
684
740
|
... }
|
685
741
|
"""
|
@@ -741,9 +797,9 @@ class PyStage(BaseStage):
|
|
741
797
|
event: Event | None = None,
|
742
798
|
) -> Result:
|
743
799
|
"""Execute the Python statement that pass all globals and input params
|
744
|
-
to globals argument on
|
800
|
+
to globals argument on `exec` build-in function.
|
745
801
|
|
746
|
-
:param params: A parameter
|
802
|
+
:param params: (DictData) A parameter data.
|
747
803
|
:param result: (Result) A result object for keeping context and status
|
748
804
|
data.
|
749
805
|
:param event: (Event) An event manager that use to track parent execute
|
@@ -751,11 +807,10 @@ class PyStage(BaseStage):
|
|
751
807
|
|
752
808
|
:rtype: Result
|
753
809
|
"""
|
754
|
-
|
755
|
-
|
756
|
-
|
757
|
-
|
758
|
-
)
|
810
|
+
result: Result = result or Result(
|
811
|
+
run_id=gen_id(self.name + (self.id or ""), unique=True),
|
812
|
+
extras=self.extras,
|
813
|
+
)
|
759
814
|
|
760
815
|
lc: DictData = {}
|
761
816
|
gb: DictData = (
|
@@ -792,20 +847,36 @@ class PyStage(BaseStage):
|
|
792
847
|
},
|
793
848
|
)
|
794
849
|
|
850
|
+
async def axecute(
|
851
|
+
self,
|
852
|
+
):
|
853
|
+
"""Async execution method.
|
854
|
+
|
855
|
+
References:
|
856
|
+
- https://stackoverflow.com/questions/44859165/async-exec-in-python
|
857
|
+
"""
|
858
|
+
|
795
859
|
|
796
860
|
class CallStage(BaseStage):
|
797
|
-
"""Call executor that call the Python function from registry with tag
|
798
|
-
decorator function in
|
861
|
+
"""Call stage executor that call the Python function from registry with tag
|
862
|
+
decorator function in `reusables` module and run it with input arguments.
|
863
|
+
|
864
|
+
This stage is different with PyStage because the PyStage is just run
|
865
|
+
a Python statement with the `exec` function and pass the current locals and
|
866
|
+
globals before exec that statement. This stage will import the caller
|
867
|
+
function can call it with an input arguments. So, you can create your
|
868
|
+
function complexly that you can for your objective to invoked by this stage
|
869
|
+
object.
|
799
870
|
|
800
|
-
This stage is
|
801
|
-
|
802
|
-
|
803
|
-
|
871
|
+
This stage is the most powerfull stage of this package for run every
|
872
|
+
use-case by a custom requirement that you want by creating the Python
|
873
|
+
function and adding it to the caller registry value by importer syntax like
|
874
|
+
`module.caller.registry` not path style like `module/caller/registry`.
|
804
875
|
|
805
|
-
|
806
|
-
|
807
|
-
|
808
|
-
|
876
|
+
Warning:
|
877
|
+
|
878
|
+
The caller registry to get a caller function should importable by the
|
879
|
+
current Python execution pointer.
|
809
880
|
|
810
881
|
Data Validate:
|
811
882
|
>>> stage = {
|
@@ -817,12 +888,16 @@ class CallStage(BaseStage):
|
|
817
888
|
|
818
889
|
uses: str = Field(
|
819
890
|
description=(
|
820
|
-
"A
|
891
|
+
"A caller function with registry importer syntax that use to load "
|
892
|
+
"function before execute step. The caller registry syntax should "
|
893
|
+
"be `<import.part>/<func-name>@<tag-name>`."
|
821
894
|
),
|
822
895
|
)
|
823
896
|
args: DictData = Field(
|
824
897
|
default_factory=dict,
|
825
|
-
description=
|
898
|
+
description=(
|
899
|
+
"An argument parameter that will pass to this caller function."
|
900
|
+
),
|
826
901
|
alias="with",
|
827
902
|
)
|
828
903
|
|
@@ -833,19 +908,12 @@ class CallStage(BaseStage):
|
|
833
908
|
result: Result | None = None,
|
834
909
|
event: Event | None = None,
|
835
910
|
) -> Result:
|
836
|
-
"""Execute
|
911
|
+
"""Execute this caller function with its argument parameter.
|
837
912
|
|
838
|
-
:
|
839
|
-
|
840
|
-
:
|
841
|
-
|
842
|
-
|
843
|
-
:param params: (DictData) A parameter that want to pass before run any
|
844
|
-
statement.
|
845
|
-
:param result: (Result) A result object for keeping context and status
|
846
|
-
data.
|
847
|
-
:param event: (Event) An event manager that use to track parent execute
|
848
|
-
was not force stopped.
|
913
|
+
:param params: (DictData) A parameter data.
|
914
|
+
:param result: (Result) A Result instance for return context and status.
|
915
|
+
:param event: (Event) An Event manager instance that use to cancel this
|
916
|
+
execution if it forces stopped by parent execution.
|
849
917
|
|
850
918
|
:raise ValueError: If necessary arguments does not pass from the `args`
|
851
919
|
field.
|
@@ -854,18 +922,20 @@ class CallStage(BaseStage):
|
|
854
922
|
|
855
923
|
:rtype: Result
|
856
924
|
"""
|
857
|
-
|
858
|
-
|
859
|
-
|
860
|
-
|
861
|
-
)
|
925
|
+
result: Result = result or Result(
|
926
|
+
run_id=gen_id(self.name + (self.id or ""), unique=True),
|
927
|
+
extras=self.extras,
|
928
|
+
)
|
862
929
|
|
863
|
-
has_keyword: bool = False
|
864
930
|
call_func: TagFunc = extract_call(
|
865
931
|
param2template(self.uses, params, extras=self.extras),
|
866
932
|
registries=self.extras.get("registry_caller"),
|
867
933
|
)()
|
868
934
|
|
935
|
+
result.trace.info(
|
936
|
+
f"[STAGE]: Call-Execute: {call_func.name}@{call_func.tag}"
|
937
|
+
)
|
938
|
+
|
869
939
|
# VALIDATE: check input task caller parameters that exists before
|
870
940
|
# calling.
|
871
941
|
args: DictData = {"result": result} | param2template(
|
@@ -873,6 +943,7 @@ class CallStage(BaseStage):
|
|
873
943
|
)
|
874
944
|
ips = inspect.signature(call_func)
|
875
945
|
necessary_params: list[str] = []
|
946
|
+
has_keyword: bool = False
|
876
947
|
for k in ips.parameters:
|
877
948
|
if (
|
878
949
|
v := ips.parameters[k]
|
@@ -896,10 +967,6 @@ class CallStage(BaseStage):
|
|
896
967
|
if "result" not in ips.parameters and not has_keyword:
|
897
968
|
args.pop("result")
|
898
969
|
|
899
|
-
result.trace.info(
|
900
|
-
f"[STAGE]: Call-Execute: {call_func.name}@{call_func.tag}"
|
901
|
-
)
|
902
|
-
|
903
970
|
args = self.parse_model_args(call_func, args, result)
|
904
971
|
|
905
972
|
if inspect.iscoroutinefunction(call_func):
|
@@ -970,9 +1037,9 @@ class CallStage(BaseStage):
|
|
970
1037
|
|
971
1038
|
|
972
1039
|
class TriggerStage(BaseStage):
|
973
|
-
"""Trigger
|
974
|
-
the
|
975
|
-
|
1040
|
+
"""Trigger workflow executor stage that run an input trigger Workflow
|
1041
|
+
execute method. This is the stage that allow you to create the reusable
|
1042
|
+
Workflow template with dynamic parameters.
|
976
1043
|
|
977
1044
|
Data Validate:
|
978
1045
|
>>> stage = {
|
@@ -984,12 +1051,13 @@ class TriggerStage(BaseStage):
|
|
984
1051
|
|
985
1052
|
trigger: str = Field(
|
986
1053
|
description=(
|
987
|
-
"A trigger workflow name
|
1054
|
+
"A trigger workflow name. This workflow name should exist on the "
|
1055
|
+
"config path because it will load by the `load_conf` method."
|
988
1056
|
),
|
989
1057
|
)
|
990
1058
|
params: DictData = Field(
|
991
1059
|
default_factory=dict,
|
992
|
-
description="A parameter that
|
1060
|
+
description="A parameter that will pass to workflow execution method.",
|
993
1061
|
)
|
994
1062
|
|
995
1063
|
def execute(
|
@@ -1002,7 +1070,7 @@ class TriggerStage(BaseStage):
|
|
1002
1070
|
"""Trigger another workflow execution. It will wait the trigger
|
1003
1071
|
workflow running complete before catching its result.
|
1004
1072
|
|
1005
|
-
:param params: A parameter data
|
1073
|
+
:param params: (DictData) A parameter data.
|
1006
1074
|
:param result: (Result) A result object for keeping context and status
|
1007
1075
|
data.
|
1008
1076
|
:param event: (Event) An event manager that use to track parent execute
|
@@ -1013,11 +1081,10 @@ class TriggerStage(BaseStage):
|
|
1013
1081
|
from .exceptions import WorkflowException
|
1014
1082
|
from .workflow import Workflow
|
1015
1083
|
|
1016
|
-
|
1017
|
-
|
1018
|
-
|
1019
|
-
|
1020
|
-
)
|
1084
|
+
result: Result = result or Result(
|
1085
|
+
run_id=gen_id(self.name + (self.id or ""), unique=True),
|
1086
|
+
extras=self.extras,
|
1087
|
+
)
|
1021
1088
|
|
1022
1089
|
_trigger: str = param2template(self.trigger, params, extras=self.extras)
|
1023
1090
|
result.trace.info(f"[STAGE]: Trigger-Execute: {_trigger!r}")
|
@@ -1037,16 +1104,20 @@ class TriggerStage(BaseStage):
|
|
1037
1104
|
err_msg: str | None = (
|
1038
1105
|
f" with:\n{msg}"
|
1039
1106
|
if (msg := rs.context.get("errors", {}).get("message"))
|
1040
|
-
else ""
|
1107
|
+
else "."
|
1108
|
+
)
|
1109
|
+
raise StageException(
|
1110
|
+
f"Trigger workflow return failed status{err_msg}"
|
1041
1111
|
)
|
1042
|
-
raise StageException(f"Trigger workflow was failed{err_msg}.")
|
1043
1112
|
return rs
|
1044
1113
|
|
1045
1114
|
|
1046
1115
|
class ParallelStage(BaseStage): # pragma: no cov
|
1047
|
-
"""Parallel
|
1116
|
+
"""Parallel stage executor that execute branch stages with multithreading.
|
1117
|
+
This stage let you set the fix branches for running child stage inside it on
|
1118
|
+
multithread pool.
|
1048
1119
|
|
1049
|
-
This stage is not the low-level stage model because it runs
|
1120
|
+
This stage is not the low-level stage model because it runs multi-stages
|
1050
1121
|
in this stage execution.
|
1051
1122
|
|
1052
1123
|
Data Validate:
|
@@ -1072,58 +1143,55 @@ class ParallelStage(BaseStage): # pragma: no cov
|
|
1072
1143
|
"""
|
1073
1144
|
|
1074
1145
|
parallel: dict[str, list[Stage]] = Field(
|
1075
|
-
description="A mapping of
|
1146
|
+
description="A mapping of branch name and its stages.",
|
1076
1147
|
)
|
1077
1148
|
max_workers: int = Field(
|
1078
1149
|
default=2,
|
1079
1150
|
ge=1,
|
1080
1151
|
lt=20,
|
1081
1152
|
description=(
|
1082
|
-
"The maximum thread pool worker size for execution parallel."
|
1153
|
+
"The maximum multi-thread pool worker size for execution parallel. "
|
1154
|
+
"This value should be gather or equal than 1, and less than 20."
|
1083
1155
|
),
|
1084
1156
|
alias="max-workers",
|
1085
1157
|
)
|
1086
1158
|
|
1087
|
-
def
|
1159
|
+
def execute_branch(
|
1088
1160
|
self,
|
1089
1161
|
branch: str,
|
1090
1162
|
params: DictData,
|
1091
1163
|
result: Result,
|
1092
1164
|
*,
|
1093
1165
|
event: Event | None = None,
|
1094
|
-
|
1095
|
-
|
1096
|
-
"""Task execution method for passing a branch to each thread.
|
1166
|
+
) -> Result:
|
1167
|
+
"""Execute all stage with specific branch ID.
|
1097
1168
|
|
1098
|
-
:param branch: A branch ID.
|
1099
|
-
:param params: A parameter data
|
1100
|
-
:param result: (Result) A
|
1101
|
-
|
1102
|
-
|
1103
|
-
was not force stopped.
|
1104
|
-
:param extras: (DictData) An extra parameters that want to override
|
1105
|
-
config values.
|
1169
|
+
:param branch: (str) A branch ID.
|
1170
|
+
:param params: (DictData) A parameter data.
|
1171
|
+
:param result: (Result) A Result instance for return context and status.
|
1172
|
+
:param event: (Event) An Event manager instance that use to cancel this
|
1173
|
+
execution if it forces stopped by parent execution.
|
1106
1174
|
|
1107
|
-
:rtype:
|
1175
|
+
:rtype: Result
|
1108
1176
|
"""
|
1109
|
-
result.trace.debug(f"
|
1177
|
+
result.trace.debug(f"[STAGE]: Execute Branch: {branch!r}")
|
1110
1178
|
context: DictData = copy.deepcopy(params)
|
1111
1179
|
context.update({"branch": branch})
|
1112
1180
|
output: DictData = {"branch": branch, "stages": {}}
|
1113
1181
|
for stage in self.parallel[branch]:
|
1114
1182
|
|
1115
|
-
if extras:
|
1116
|
-
stage.extras = extras
|
1183
|
+
if self.extras:
|
1184
|
+
stage.extras = self.extras
|
1117
1185
|
|
1118
1186
|
if stage.is_skipped(params=context):
|
1119
|
-
result.trace.info(f"
|
1187
|
+
result.trace.info(f"[STAGE]: Skip stage: {stage.iden!r}")
|
1120
1188
|
stage.set_outputs(output={"skipped": True}, to=output)
|
1121
1189
|
continue
|
1122
1190
|
|
1123
1191
|
if event and event.is_set():
|
1124
1192
|
error_msg: str = (
|
1125
1193
|
"Branch-Stage was canceled from event that had set before "
|
1126
|
-
"stage
|
1194
|
+
"stage branch execution."
|
1127
1195
|
)
|
1128
1196
|
return result.catch(
|
1129
1197
|
status=CANCEL,
|
@@ -1147,17 +1215,29 @@ class ParallelStage(BaseStage): # pragma: no cov
|
|
1147
1215
|
stage.set_outputs(rs.context, to=output)
|
1148
1216
|
stage.set_outputs(stage.get_outputs(output), to=context)
|
1149
1217
|
except (StageException, UtilException) as e: # pragma: no cov
|
1150
|
-
result.trace.error(
|
1218
|
+
result.trace.error(
|
1219
|
+
f"[STAGE]: {e.__class__.__name__}:{NEWLINE}{e}"
|
1220
|
+
)
|
1221
|
+
result.catch(
|
1222
|
+
status=FAILED,
|
1223
|
+
parallel={
|
1224
|
+
branch: {
|
1225
|
+
"branch": branch,
|
1226
|
+
"stages": filter_func(output.pop("stages", {})),
|
1227
|
+
"errors": e.to_dict(),
|
1228
|
+
},
|
1229
|
+
},
|
1230
|
+
)
|
1151
1231
|
raise StageException(
|
1152
|
-
f"Sub-Stage
|
1232
|
+
f"Sub-Stage raise: {e.__class__.__name__}: {e}"
|
1153
1233
|
) from None
|
1154
1234
|
|
1155
1235
|
if rs.status == FAILED:
|
1156
1236
|
error_msg: str = (
|
1157
|
-
f"
|
1237
|
+
f"Branch-Stage was break because it has a sub stage, "
|
1158
1238
|
f"{stage.iden}, failed without raise error."
|
1159
1239
|
)
|
1160
|
-
|
1240
|
+
result.catch(
|
1161
1241
|
status=FAILED,
|
1162
1242
|
parallel={
|
1163
1243
|
branch: {
|
@@ -1167,6 +1247,7 @@ class ParallelStage(BaseStage): # pragma: no cov
|
|
1167
1247
|
},
|
1168
1248
|
},
|
1169
1249
|
)
|
1250
|
+
raise StageException(error_msg)
|
1170
1251
|
|
1171
1252
|
return result.catch(
|
1172
1253
|
status=SUCCESS,
|
@@ -1185,30 +1266,37 @@ class ParallelStage(BaseStage): # pragma: no cov
|
|
1185
1266
|
result: Result | None = None,
|
1186
1267
|
event: Event | None = None,
|
1187
1268
|
) -> Result:
|
1188
|
-
"""Execute
|
1189
|
-
or async mode by changing `async_mode` flag.
|
1269
|
+
"""Execute parallel each branch via multi-threading pool.
|
1190
1270
|
|
1191
|
-
:param params: A parameter
|
1192
|
-
:param result: (Result) A
|
1193
|
-
|
1194
|
-
|
1195
|
-
was not force stopped.
|
1271
|
+
:param params: (DictData) A parameter data.
|
1272
|
+
:param result: (Result) A Result instance for return context and status.
|
1273
|
+
:param event: (Event) An Event manager instance that use to cancel this
|
1274
|
+
execution if it forces stopped by parent execution.
|
1196
1275
|
|
1197
1276
|
:rtype: Result
|
1198
1277
|
"""
|
1199
|
-
|
1200
|
-
|
1201
|
-
|
1202
|
-
|
1203
|
-
)
|
1278
|
+
result: Result = result or Result(
|
1279
|
+
run_id=gen_id(self.name + (self.id or ""), unique=True),
|
1280
|
+
extras=self.extras,
|
1281
|
+
)
|
1204
1282
|
event: Event = Event() if event is None else event
|
1205
1283
|
result.trace.info(
|
1206
1284
|
f"[STAGE]: Parallel-Execute: {self.max_workers} workers."
|
1207
1285
|
)
|
1208
1286
|
result.catch(status=WAIT, context={"parallel": {}})
|
1287
|
+
if event and event.is_set(): # pragma: no cov
|
1288
|
+
return result.catch(
|
1289
|
+
status=CANCEL,
|
1290
|
+
context={
|
1291
|
+
"errors": StageException(
|
1292
|
+
"Stage was canceled from event that had set "
|
1293
|
+
"before stage parallel execution."
|
1294
|
+
).to_dict()
|
1295
|
+
},
|
1296
|
+
)
|
1297
|
+
|
1209
1298
|
with ThreadPoolExecutor(
|
1210
|
-
max_workers=self.max_workers,
|
1211
|
-
thread_name_prefix="parallel_stage_exec_",
|
1299
|
+
max_workers=self.max_workers, thread_name_prefix="stage_parallel_"
|
1212
1300
|
) as executor:
|
1213
1301
|
|
1214
1302
|
context: DictData = {}
|
@@ -1216,36 +1304,36 @@ class ParallelStage(BaseStage): # pragma: no cov
|
|
1216
1304
|
|
1217
1305
|
futures: list[Future] = (
|
1218
1306
|
executor.submit(
|
1219
|
-
self.
|
1307
|
+
self.execute_branch,
|
1220
1308
|
branch=branch,
|
1221
1309
|
params=params,
|
1222
1310
|
result=result,
|
1223
1311
|
event=event,
|
1224
|
-
extras=self.extras,
|
1225
1312
|
)
|
1226
1313
|
for branch in self.parallel
|
1227
1314
|
)
|
1228
1315
|
|
1229
|
-
|
1230
|
-
for future in done:
|
1316
|
+
for future in as_completed(futures):
|
1231
1317
|
try:
|
1232
1318
|
future.result()
|
1233
1319
|
except StageException as e:
|
1234
1320
|
status = FAILED
|
1235
1321
|
result.trace.error(
|
1236
|
-
f"[STAGE]: {e.__class__.__name__}
|
1322
|
+
f"[STAGE]: {e.__class__.__name__}:{NEWLINE}{e}"
|
1237
1323
|
)
|
1238
|
-
|
1239
|
-
|
1324
|
+
if "errors" in context:
|
1325
|
+
context["errors"].append(e.to_dict())
|
1326
|
+
else:
|
1327
|
+
context["errors"] = [e.to_dict()]
|
1240
1328
|
return result.catch(status=status, context=context)
|
1241
1329
|
|
1242
1330
|
|
1243
1331
|
class ForEachStage(BaseStage):
|
1244
|
-
"""For-Each
|
1245
|
-
|
1246
|
-
muti-stages in this stage execution.
|
1332
|
+
"""For-Each stage executor that execute all stages with each item in the
|
1333
|
+
foreach list.
|
1247
1334
|
|
1248
|
-
|
1335
|
+
This stage is not the low-level stage model because it runs
|
1336
|
+
multi-stages in this stage execution.
|
1249
1337
|
|
1250
1338
|
Data Validate:
|
1251
1339
|
>>> stage = {
|
@@ -1254,7 +1342,7 @@ class ForEachStage(BaseStage):
|
|
1254
1342
|
... "stages": [
|
1255
1343
|
... {
|
1256
1344
|
... "name": "Echo stage",
|
1257
|
-
... "echo": "Start run with item {{ item }}"
|
1345
|
+
... "echo": "Start run with item ${{ item }}"
|
1258
1346
|
... },
|
1259
1347
|
... ],
|
1260
1348
|
... }
|
@@ -1262,13 +1350,14 @@ class ForEachStage(BaseStage):
|
|
1262
1350
|
|
1263
1351
|
foreach: Union[list[str], list[int], str] = Field(
|
1264
1352
|
description=(
|
1265
|
-
"A items for passing to
|
1353
|
+
"A items for passing to stages via ${{ item }} template parameter."
|
1266
1354
|
),
|
1267
1355
|
)
|
1268
1356
|
stages: list[Stage] = Field(
|
1269
1357
|
default_factory=list,
|
1270
1358
|
description=(
|
1271
|
-
"A list of stage that will run with each item in the foreach
|
1359
|
+
"A list of stage that will run with each item in the `foreach` "
|
1360
|
+
"field."
|
1272
1361
|
),
|
1273
1362
|
)
|
1274
1363
|
concurrent: int = Field(
|
@@ -1283,27 +1372,25 @@ class ForEachStage(BaseStage):
|
|
1283
1372
|
|
1284
1373
|
def execute_item(
|
1285
1374
|
self,
|
1286
|
-
item:
|
1375
|
+
item: StrOrInt,
|
1287
1376
|
params: DictData,
|
1288
1377
|
result: Result,
|
1289
1378
|
*,
|
1290
1379
|
event: Event | None = None,
|
1291
1380
|
) -> Result:
|
1292
|
-
"""Execute
|
1381
|
+
"""Execute all stage with specific foreach item.
|
1293
1382
|
|
1294
1383
|
:param item: (str | int) An item that want to execution.
|
1295
|
-
:param params: (DictData) A parameter
|
1296
|
-
|
1297
|
-
:param
|
1298
|
-
|
1299
|
-
:param event: (Event) An event manager that use to track parent execute
|
1300
|
-
was not force stopped.
|
1384
|
+
:param params: (DictData) A parameter data.
|
1385
|
+
:param result: (Result) A Result instance for return context and status.
|
1386
|
+
:param event: (Event) An Event manager instance that use to cancel this
|
1387
|
+
execution if it forces stopped by parent execution.
|
1301
1388
|
|
1302
1389
|
:raise StageException: If the stage execution raise errors.
|
1303
1390
|
|
1304
1391
|
:rtype: Result
|
1305
1392
|
"""
|
1306
|
-
result.trace.debug(f"[STAGE]: Execute
|
1393
|
+
result.trace.debug(f"[STAGE]: Execute Item: {item!r}")
|
1307
1394
|
context: DictData = copy.deepcopy(params)
|
1308
1395
|
context.update({"item": item})
|
1309
1396
|
output: DictData = {"item": item, "stages": {}}
|
@@ -1313,7 +1400,7 @@ class ForEachStage(BaseStage):
|
|
1313
1400
|
stage.extras = self.extras
|
1314
1401
|
|
1315
1402
|
if stage.is_skipped(params=context):
|
1316
|
-
result.trace.info(f"
|
1403
|
+
result.trace.info(f"[STAGE]: Skip stage: {stage.iden!r}")
|
1317
1404
|
stage.set_outputs(output={"skipped": True}, to=output)
|
1318
1405
|
continue
|
1319
1406
|
|
@@ -1344,9 +1431,21 @@ class ForEachStage(BaseStage):
|
|
1344
1431
|
stage.set_outputs(rs.context, to=output)
|
1345
1432
|
stage.set_outputs(stage.get_outputs(output), to=context)
|
1346
1433
|
except (StageException, UtilException) as e:
|
1347
|
-
result.trace.error(
|
1434
|
+
result.trace.error(
|
1435
|
+
f"[STAGE]: {e.__class__.__name__}:{NEWLINE}{e}"
|
1436
|
+
)
|
1437
|
+
result.catch(
|
1438
|
+
status=FAILED,
|
1439
|
+
foreach={
|
1440
|
+
item: {
|
1441
|
+
"item": item,
|
1442
|
+
"stages": filter_func(output.pop("stages", {})),
|
1443
|
+
"errors": e.to_dict(),
|
1444
|
+
},
|
1445
|
+
},
|
1446
|
+
)
|
1348
1447
|
raise StageException(
|
1349
|
-
f"Sub-Stage
|
1448
|
+
f"Sub-Stage raise: {e.__class__.__name__}: {e}"
|
1350
1449
|
) from None
|
1351
1450
|
|
1352
1451
|
if rs.status == FAILED:
|
@@ -1354,7 +1453,8 @@ class ForEachStage(BaseStage):
|
|
1354
1453
|
f"Item-Stage was break because it has a sub stage, "
|
1355
1454
|
f"{stage.iden}, failed without raise error."
|
1356
1455
|
)
|
1357
|
-
|
1456
|
+
result.trace.warning(f"[STAGE]: {error_msg}")
|
1457
|
+
result.catch(
|
1358
1458
|
status=FAILED,
|
1359
1459
|
foreach={
|
1360
1460
|
item: {
|
@@ -1364,6 +1464,8 @@ class ForEachStage(BaseStage):
|
|
1364
1464
|
},
|
1365
1465
|
},
|
1366
1466
|
)
|
1467
|
+
raise StageException(error_msg)
|
1468
|
+
|
1367
1469
|
return result.catch(
|
1368
1470
|
status=SUCCESS,
|
1369
1471
|
foreach={
|
@@ -1383,29 +1485,29 @@ class ForEachStage(BaseStage):
|
|
1383
1485
|
) -> Result:
|
1384
1486
|
"""Execute the stages that pass each item form the foreach field.
|
1385
1487
|
|
1386
|
-
:param params: A parameter
|
1387
|
-
:param result: (Result) A
|
1388
|
-
|
1389
|
-
|
1390
|
-
|
1488
|
+
:param params: (DictData) A parameter data.
|
1489
|
+
:param result: (Result) A Result instance for return context and status.
|
1490
|
+
:param event: (Event) An Event manager instance that use to cancel this
|
1491
|
+
execution if it forces stopped by parent execution.
|
1492
|
+
|
1493
|
+
:raise TypeError: If the foreach does not match with type list.
|
1391
1494
|
|
1392
1495
|
:rtype: Result
|
1393
1496
|
"""
|
1394
|
-
|
1395
|
-
|
1396
|
-
|
1397
|
-
|
1398
|
-
)
|
1497
|
+
result: Result = result or Result(
|
1498
|
+
run_id=gen_id(self.name + (self.id or ""), unique=True),
|
1499
|
+
extras=self.extras,
|
1500
|
+
)
|
1399
1501
|
event: Event = Event() if event is None else event
|
1400
1502
|
foreach: Union[list[str], list[int]] = (
|
1401
1503
|
param2template(self.foreach, params, extras=self.extras)
|
1402
1504
|
if isinstance(self.foreach, str)
|
1403
1505
|
else self.foreach
|
1404
1506
|
)
|
1507
|
+
|
1508
|
+
# [VALIDATE]: Type of the foreach should be `list` type.
|
1405
1509
|
if not isinstance(foreach, list):
|
1406
|
-
raise
|
1407
|
-
f"Foreach does not support foreach value: {foreach!r}"
|
1408
|
-
)
|
1510
|
+
raise TypeError(f"Does not support foreach: {foreach!r}")
|
1409
1511
|
|
1410
1512
|
result.trace.info(f"[STAGE]: Foreach-Execute: {foreach!r}.")
|
1411
1513
|
result.catch(status=WAIT, context={"items": foreach, "foreach": {}})
|
@@ -1437,33 +1539,36 @@ class ForEachStage(BaseStage):
|
|
1437
1539
|
context: DictData = {}
|
1438
1540
|
status: Status = SUCCESS
|
1439
1541
|
|
1440
|
-
done, not_done = wait(
|
1441
|
-
futures, timeout=1800, return_when=FIRST_EXCEPTION
|
1442
|
-
)
|
1443
|
-
|
1542
|
+
done, not_done = wait(futures, return_when=FIRST_EXCEPTION)
|
1444
1543
|
if len(done) != len(futures):
|
1445
1544
|
result.trace.warning(
|
1446
|
-
"[STAGE]: Set
|
1545
|
+
"[STAGE]: Set event for stop pending stage future."
|
1447
1546
|
)
|
1448
1547
|
event.set()
|
1449
1548
|
for future in not_done:
|
1450
1549
|
future.cancel()
|
1451
1550
|
|
1551
|
+
nd: str = f", item not run: {not_done}" if not_done else ""
|
1552
|
+
result.trace.debug(f"... Foreach set Fail-Fast{nd}")
|
1553
|
+
|
1452
1554
|
for future in done:
|
1453
1555
|
try:
|
1454
1556
|
future.result()
|
1455
1557
|
except StageException as e:
|
1456
1558
|
status = FAILED
|
1457
1559
|
result.trace.error(
|
1458
|
-
f"[STAGE]: {e.__class__.__name__}
|
1560
|
+
f"[STAGE]: {e.__class__.__name__}:{NEWLINE}{e}"
|
1459
1561
|
)
|
1460
1562
|
context.update({"errors": e.to_dict()})
|
1461
|
-
|
1462
1563
|
return result.catch(status=status, context=context)
|
1463
1564
|
|
1464
1565
|
|
1465
|
-
class UntilStage(BaseStage):
|
1466
|
-
"""Until
|
1566
|
+
class UntilStage(BaseStage):
|
1567
|
+
"""Until stage executor that will run stages in each loop until it valid
|
1568
|
+
with stop loop condition.
|
1569
|
+
|
1570
|
+
This stage is not the low-level stage model because it runs
|
1571
|
+
multi-stages in this stage execution.
|
1467
1572
|
|
1468
1573
|
Data Validate:
|
1469
1574
|
>>> stage = {
|
@@ -1485,23 +1590,25 @@ class UntilStage(BaseStage): # pragma: no cov
|
|
1485
1590
|
"An initial value that can be any value in str, int, or bool type."
|
1486
1591
|
),
|
1487
1592
|
)
|
1488
|
-
until: str = Field(description="A until condition.")
|
1593
|
+
until: str = Field(description="A until condition for stop the while loop.")
|
1489
1594
|
stages: list[Stage] = Field(
|
1490
1595
|
default_factory=list,
|
1491
1596
|
description=(
|
1492
|
-
"A list of stage that will run with each item until
|
1493
|
-
"correct."
|
1597
|
+
"A list of stage that will run with each item in until loop."
|
1494
1598
|
),
|
1495
1599
|
)
|
1496
1600
|
max_loop: int = Field(
|
1497
1601
|
default=10,
|
1498
1602
|
ge=1,
|
1499
1603
|
lt=100,
|
1500
|
-
description=
|
1604
|
+
description=(
|
1605
|
+
"The maximum value of loop for this until stage. This value should "
|
1606
|
+
"be gather or equal than 1, and less than 100."
|
1607
|
+
),
|
1501
1608
|
alias="max-loop",
|
1502
1609
|
)
|
1503
1610
|
|
1504
|
-
def
|
1611
|
+
def execute_loop(
|
1505
1612
|
self,
|
1506
1613
|
item: T,
|
1507
1614
|
loop: int,
|
@@ -1509,19 +1616,17 @@ class UntilStage(BaseStage): # pragma: no cov
|
|
1509
1616
|
result: Result,
|
1510
1617
|
event: Event | None = None,
|
1511
1618
|
) -> tuple[Result, T]:
|
1512
|
-
"""Execute
|
1513
|
-
variable.
|
1619
|
+
"""Execute all stage with specific loop and item.
|
1514
1620
|
|
1515
1621
|
:param item: (T) An item that want to execution.
|
1516
1622
|
:param loop: (int) A number of loop.
|
1517
|
-
:param params: (DictData) A parameter
|
1518
|
-
|
1519
|
-
:param
|
1520
|
-
|
1521
|
-
:param event: (Event) An event manager that use to track parent execute
|
1522
|
-
was not force stopped.
|
1623
|
+
:param params: (DictData) A parameter data.
|
1624
|
+
:param result: (Result) A Result instance for return context and status.
|
1625
|
+
:param event: (Event) An Event manager instance that use to cancel this
|
1626
|
+
execution if it forces stopped by parent execution.
|
1523
1627
|
|
1524
1628
|
:rtype: tuple[Result, T]
|
1629
|
+
:return: Return a pair of Result and changed item.
|
1525
1630
|
"""
|
1526
1631
|
result.trace.debug(f"... Execute until item: {item!r}")
|
1527
1632
|
context: DictData = copy.deepcopy(params)
|
@@ -1534,14 +1639,14 @@ class UntilStage(BaseStage): # pragma: no cov
|
|
1534
1639
|
stage.extras = self.extras
|
1535
1640
|
|
1536
1641
|
if stage.is_skipped(params=context):
|
1537
|
-
result.trace.info(f"
|
1642
|
+
result.trace.info(f"[STAGE]: Skip stage: {stage.iden!r}")
|
1538
1643
|
stage.set_outputs(output={"skipped": True}, to=output)
|
1539
1644
|
continue
|
1540
1645
|
|
1541
1646
|
if event and event.is_set():
|
1542
1647
|
error_msg: str = (
|
1543
|
-
"
|
1544
|
-
"stage
|
1648
|
+
"Loop-Stage was canceled from event that had set before "
|
1649
|
+
"stage loop execution."
|
1545
1650
|
)
|
1546
1651
|
return (
|
1547
1652
|
result.catch(
|
@@ -1573,11 +1678,42 @@ class UntilStage(BaseStage): # pragma: no cov
|
|
1573
1678
|
|
1574
1679
|
stage.set_outputs(_output, to=context)
|
1575
1680
|
except (StageException, UtilException) as e:
|
1576
|
-
result.trace.error(
|
1681
|
+
result.trace.error(
|
1682
|
+
f"[STAGE]: {e.__class__.__name__}:{NEWLINE}{e}"
|
1683
|
+
)
|
1684
|
+
result.catch(
|
1685
|
+
status=FAILED,
|
1686
|
+
until={
|
1687
|
+
loop: {
|
1688
|
+
"loop": loop,
|
1689
|
+
"item": item,
|
1690
|
+
"stages": filter_func(output.pop("stages", {})),
|
1691
|
+
"errors": e.to_dict(),
|
1692
|
+
}
|
1693
|
+
},
|
1694
|
+
)
|
1577
1695
|
raise StageException(
|
1578
1696
|
f"Sub-Stage execution error: {e.__class__.__name__}: {e}"
|
1579
1697
|
) from None
|
1580
1698
|
|
1699
|
+
if rs.status == FAILED:
|
1700
|
+
error_msg: str = (
|
1701
|
+
f"Loop-Stage was break because it has a sub stage, "
|
1702
|
+
f"{stage.iden}, failed without raise error."
|
1703
|
+
)
|
1704
|
+
result.catch(
|
1705
|
+
status=FAILED,
|
1706
|
+
until={
|
1707
|
+
loop: {
|
1708
|
+
"loop": loop,
|
1709
|
+
"item": item,
|
1710
|
+
"stages": filter_func(output.pop("stages", {})),
|
1711
|
+
"errors": StageException(error_msg).to_dict(),
|
1712
|
+
}
|
1713
|
+
},
|
1714
|
+
)
|
1715
|
+
raise StageException(error_msg)
|
1716
|
+
|
1581
1717
|
return (
|
1582
1718
|
result.catch(
|
1583
1719
|
status=SUCCESS,
|
@@ -1599,21 +1735,19 @@ class UntilStage(BaseStage): # pragma: no cov
|
|
1599
1735
|
result: Result | None = None,
|
1600
1736
|
event: Event | None = None,
|
1601
1737
|
) -> Result:
|
1602
|
-
"""Execute
|
1603
|
-
setter step.
|
1738
|
+
"""Execute until loop with checking until condition.
|
1604
1739
|
|
1605
|
-
:param params: A parameter
|
1606
|
-
:param result: (Result) A
|
1607
|
-
|
1608
|
-
|
1609
|
-
was not force stopped.
|
1740
|
+
:param params: (DictData) A parameter data.
|
1741
|
+
:param result: (Result) A Result instance for return context and status.
|
1742
|
+
:param event: (Event) An Event manager instance that use to cancel this
|
1743
|
+
execution if it forces stopped by parent execution.
|
1610
1744
|
|
1611
1745
|
:rtype: Result
|
1612
1746
|
"""
|
1613
|
-
|
1614
|
-
|
1615
|
-
|
1616
|
-
|
1747
|
+
result: Result = result or Result(
|
1748
|
+
run_id=gen_id(self.name + (self.id or ""), unique=True),
|
1749
|
+
extras=self.extras,
|
1750
|
+
)
|
1617
1751
|
|
1618
1752
|
result.trace.info(f"[STAGE]: Until-Execution: {self.until}")
|
1619
1753
|
item: Union[str, int, bool] = param2template(
|
@@ -1631,12 +1765,12 @@ class UntilStage(BaseStage): # pragma: no cov
|
|
1631
1765
|
context={
|
1632
1766
|
"errors": StageException(
|
1633
1767
|
"Stage was canceled from event that had set "
|
1634
|
-
"before stage
|
1768
|
+
"before stage loop execution."
|
1635
1769
|
).to_dict()
|
1636
1770
|
},
|
1637
1771
|
)
|
1638
1772
|
|
1639
|
-
result, item = self.
|
1773
|
+
result, item = self.execute_loop(
|
1640
1774
|
item=item,
|
1641
1775
|
loop=loop,
|
1642
1776
|
params=params,
|
@@ -1647,10 +1781,10 @@ class UntilStage(BaseStage): # pragma: no cov
|
|
1647
1781
|
loop += 1
|
1648
1782
|
if item is None:
|
1649
1783
|
result.trace.warning(
|
1650
|
-
"...
|
1651
|
-
"default."
|
1784
|
+
f"... Loop-Execute not set item. It use loop: {loop} by "
|
1785
|
+
f"default."
|
1652
1786
|
)
|
1653
|
-
item = loop
|
1787
|
+
item: int = loop
|
1654
1788
|
|
1655
1789
|
next_track: bool = eval(
|
1656
1790
|
param2template(
|
@@ -1663,8 +1797,8 @@ class UntilStage(BaseStage): # pragma: no cov
|
|
1663
1797
|
)
|
1664
1798
|
if not isinstance(next_track, bool):
|
1665
1799
|
raise StageException(
|
1666
|
-
"Return type of until condition
|
1667
|
-
f"
|
1800
|
+
"Return type of until condition not be `boolean`, getting"
|
1801
|
+
f": {next_track!r}"
|
1668
1802
|
)
|
1669
1803
|
track: bool = not next_track
|
1670
1804
|
delay(0.025)
|
@@ -1679,14 +1813,14 @@ class UntilStage(BaseStage): # pragma: no cov
|
|
1679
1813
|
class Match(BaseModel):
|
1680
1814
|
"""Match model for the Case Stage."""
|
1681
1815
|
|
1682
|
-
case:
|
1816
|
+
case: StrOrInt = Field(description="A match case.")
|
1683
1817
|
stages: list[Stage] = Field(
|
1684
1818
|
description="A list of stage to execution for this case."
|
1685
1819
|
)
|
1686
1820
|
|
1687
1821
|
|
1688
1822
|
class CaseStage(BaseStage):
|
1689
|
-
"""Case
|
1823
|
+
"""Case stage executor that execute all stages if the condition was matched.
|
1690
1824
|
|
1691
1825
|
Data Validate:
|
1692
1826
|
>>> stage = {
|
@@ -1742,19 +1876,16 @@ class CaseStage(BaseStage):
|
|
1742
1876
|
|
1743
1877
|
:param case: (str) A case that want to execution.
|
1744
1878
|
:param stages: (list[Stage]) A list of stage.
|
1745
|
-
:param params: (DictData) A parameter
|
1746
|
-
|
1747
|
-
:param
|
1748
|
-
|
1749
|
-
:param event: (Event) An event manager that use to track parent execute
|
1750
|
-
was not force stopped.
|
1879
|
+
:param params: (DictData) A parameter data.
|
1880
|
+
:param result: (Result) A Result instance for return context and status.
|
1881
|
+
:param event: (Event) An Event manager instance that use to cancel this
|
1882
|
+
execution if it forces stopped by parent execution.
|
1751
1883
|
|
1752
1884
|
:rtype: Result
|
1753
1885
|
"""
|
1754
1886
|
context: DictData = copy.deepcopy(params)
|
1755
1887
|
context.update({"case": case})
|
1756
1888
|
output: DictData = {"case": case, "stages": {}}
|
1757
|
-
|
1758
1889
|
for stage in stages:
|
1759
1890
|
|
1760
1891
|
if self.extras:
|
@@ -1830,19 +1961,17 @@ class CaseStage(BaseStage):
|
|
1830
1961
|
) -> Result:
|
1831
1962
|
"""Execute case-match condition that pass to the case field.
|
1832
1963
|
|
1833
|
-
:param params: A parameter
|
1834
|
-
:param result: (Result) A
|
1835
|
-
|
1836
|
-
|
1837
|
-
was not force stopped.
|
1964
|
+
:param params: (DictData) A parameter data.
|
1965
|
+
:param result: (Result) A Result instance for return context and status.
|
1966
|
+
:param event: (Event) An Event manager instance that use to cancel this
|
1967
|
+
execution if it forces stopped by parent execution.
|
1838
1968
|
|
1839
1969
|
:rtype: Result
|
1840
1970
|
"""
|
1841
|
-
|
1842
|
-
|
1843
|
-
|
1844
|
-
|
1845
|
-
)
|
1971
|
+
result: Result = result or Result(
|
1972
|
+
run_id=gen_id(self.name + (self.id or ""), unique=True),
|
1973
|
+
extras=self.extras,
|
1974
|
+
)
|
1846
1975
|
|
1847
1976
|
_case: Optional[str] = param2template(
|
1848
1977
|
self.case, params, extras=self.extras
|
@@ -1898,7 +2027,7 @@ class CaseStage(BaseStage):
|
|
1898
2027
|
|
1899
2028
|
|
1900
2029
|
class RaiseStage(BaseStage): # pragma: no cov
|
1901
|
-
"""Raise error stage
|
2030
|
+
"""Raise error stage executor that raise `StageException` that use a message
|
1902
2031
|
field for making error message before raise.
|
1903
2032
|
|
1904
2033
|
Data Validate:
|
@@ -1911,7 +2040,7 @@ class RaiseStage(BaseStage): # pragma: no cov
|
|
1911
2040
|
|
1912
2041
|
message: str = Field(
|
1913
2042
|
description=(
|
1914
|
-
"An error message that want to raise with StageException class"
|
2043
|
+
"An error message that want to raise with `StageException` class"
|
1915
2044
|
),
|
1916
2045
|
alias="raise",
|
1917
2046
|
)
|
@@ -1925,43 +2054,27 @@ class RaiseStage(BaseStage): # pragma: no cov
|
|
1925
2054
|
) -> Result:
|
1926
2055
|
"""Raise the StageException object with the message field execution.
|
1927
2056
|
|
1928
|
-
:param params: A parameter
|
1929
|
-
:param result: (Result) A
|
1930
|
-
|
1931
|
-
|
1932
|
-
was not force stopped.
|
2057
|
+
:param params: (DictData) A parameter data.
|
2058
|
+
:param result: (Result) A Result instance for return context and status.
|
2059
|
+
:param event: (Event) An Event manager instance that use to cancel this
|
2060
|
+
execution if it forces stopped by parent execution.
|
1933
2061
|
"""
|
1934
|
-
|
1935
|
-
|
1936
|
-
|
1937
|
-
|
1938
|
-
)
|
2062
|
+
result: Result = result or Result(
|
2063
|
+
run_id=gen_id(self.name + (self.id or ""), unique=True),
|
2064
|
+
extras=self.extras,
|
2065
|
+
)
|
1939
2066
|
message: str = param2template(self.message, params, extras=self.extras)
|
1940
2067
|
result.trace.info(f"[STAGE]: Raise-Execute: {message!r}.")
|
1941
2068
|
raise StageException(message)
|
1942
2069
|
|
1943
2070
|
|
1944
|
-
# TODO: Not implement this stages yet
|
1945
|
-
class HookStage(BaseStage): # pragma: no cov
|
1946
|
-
"""Hook stage execution."""
|
1947
|
-
|
1948
|
-
hook: str
|
1949
|
-
args: DictData = Field(default_factory=dict)
|
1950
|
-
callback: str
|
1951
|
-
|
1952
|
-
def execute(
|
1953
|
-
self,
|
1954
|
-
params: DictData,
|
1955
|
-
*,
|
1956
|
-
result: Result | None = None,
|
1957
|
-
event: Event | None = None,
|
1958
|
-
) -> Result:
|
1959
|
-
raise NotImplementedError("Hook Stage does not implement yet.")
|
1960
|
-
|
1961
|
-
|
1962
|
-
# TODO: Not implement this stages yet
|
1963
2071
|
class DockerStage(BaseStage): # pragma: no cov
|
1964
|
-
"""Docker container stage execution
|
2072
|
+
"""Docker container stage execution that will pull the specific Docker image
|
2073
|
+
with custom authentication and run this image by passing environment
|
2074
|
+
variables and mounting local volume to this Docker container.
|
2075
|
+
|
2076
|
+
The volume path that mount to this Docker container will limit. That is
|
2077
|
+
this stage does not allow you to mount any path to this container.
|
1965
2078
|
|
1966
2079
|
Data Validate:
|
1967
2080
|
>>> stage = {
|
@@ -1969,10 +2082,7 @@ class DockerStage(BaseStage): # pragma: no cov
|
|
1969
2082
|
... "image": "image-name.pkg.com",
|
1970
2083
|
... "env": {
|
1971
2084
|
... "ENV": "dev",
|
1972
|
-
... "
|
1973
|
-
... },
|
1974
|
-
... "volume": {
|
1975
|
-
... "secrets": "/secrets",
|
2085
|
+
... "SECRET": "${SPECIFIC_SECRET}",
|
1976
2086
|
... },
|
1977
2087
|
... "auth": {
|
1978
2088
|
... "username": "__json_key",
|
@@ -1985,8 +2095,16 @@ class DockerStage(BaseStage): # pragma: no cov
|
|
1985
2095
|
description="A Docker image url with tag that want to run.",
|
1986
2096
|
)
|
1987
2097
|
tag: str = Field(default="latest", description="An Docker image tag.")
|
1988
|
-
env: DictData = Field(
|
1989
|
-
|
2098
|
+
env: DictData = Field(
|
2099
|
+
default_factory=dict,
|
2100
|
+
description=(
|
2101
|
+
"An environment variable that want pass to Docker container.",
|
2102
|
+
),
|
2103
|
+
)
|
2104
|
+
volume: DictData = Field(
|
2105
|
+
default_factory=dict,
|
2106
|
+
description="A mapping of local and target mounting path.",
|
2107
|
+
)
|
1990
2108
|
auth: DictData = Field(
|
1991
2109
|
default_factory=dict,
|
1992
2110
|
description=(
|
@@ -1998,7 +2116,17 @@ class DockerStage(BaseStage): # pragma: no cov
|
|
1998
2116
|
self,
|
1999
2117
|
params: DictData,
|
2000
2118
|
result: Result,
|
2001
|
-
|
2119
|
+
event: Event | None = None,
|
2120
|
+
) -> Result:
|
2121
|
+
"""Execute Docker container task.
|
2122
|
+
|
2123
|
+
:param params: (DictData) A parameter data.
|
2124
|
+
:param result: (Result) A Result instance for return context and status.
|
2125
|
+
:param event: (Event) An Event manager instance that use to cancel this
|
2126
|
+
execution if it forces stopped by parent execution.
|
2127
|
+
|
2128
|
+
:rtype: Result
|
2129
|
+
"""
|
2002
2130
|
from docker import DockerClient
|
2003
2131
|
from docker.errors import ContainerError
|
2004
2132
|
|
@@ -2016,6 +2144,16 @@ class DockerStage(BaseStage): # pragma: no cov
|
|
2016
2144
|
for line in resp:
|
2017
2145
|
result.trace.info(f"... {line}")
|
2018
2146
|
|
2147
|
+
if event and event.is_set():
|
2148
|
+
error_msg: str = (
|
2149
|
+
"Docker-Stage was canceled from event that had set before "
|
2150
|
+
"run the Docker container."
|
2151
|
+
)
|
2152
|
+
return result.catch(
|
2153
|
+
status=CANCEL,
|
2154
|
+
context={"errors": StageException(error_msg).to_dict()},
|
2155
|
+
)
|
2156
|
+
|
2019
2157
|
unique_image_name: str = f"{self.image}_{datetime.now():%Y%m%d%H%M%S%f}"
|
2020
2158
|
container = client.containers.run(
|
2021
2159
|
image=f"{self.image}:{self.tag}",
|
@@ -2054,6 +2192,13 @@ class DockerStage(BaseStage): # pragma: no cov
|
|
2054
2192
|
f"{self.image}:{self.tag}",
|
2055
2193
|
out,
|
2056
2194
|
)
|
2195
|
+
output_file: Path = Path(f".docker.{result.run_id}.logs/outputs.json")
|
2196
|
+
if not output_file.exists():
|
2197
|
+
return result.catch(status=SUCCESS)
|
2198
|
+
|
2199
|
+
with output_file.open(mode="rt") as f:
|
2200
|
+
data = json.load(f)
|
2201
|
+
return result.catch(status=SUCCESS, context=data)
|
2057
2202
|
|
2058
2203
|
def execute(
|
2059
2204
|
self,
|
@@ -2062,13 +2207,34 @@ class DockerStage(BaseStage): # pragma: no cov
|
|
2062
2207
|
result: Result | None = None,
|
2063
2208
|
event: Event | None = None,
|
2064
2209
|
) -> Result:
|
2210
|
+
"""Execute the Docker image via Python API.
|
2211
|
+
|
2212
|
+
:param params: (DictData) A parameter data.
|
2213
|
+
:param result: (Result) A Result instance for return context and status.
|
2214
|
+
:param event: (Event) An Event manager instance that use to cancel this
|
2215
|
+
execution if it forces stopped by parent execution.
|
2216
|
+
|
2217
|
+
:rtype: Result
|
2218
|
+
"""
|
2219
|
+
result: Result = result or Result(
|
2220
|
+
run_id=gen_id(self.name + (self.id or ""), unique=True),
|
2221
|
+
extras=self.extras,
|
2222
|
+
)
|
2223
|
+
|
2224
|
+
result.trace.info(f"[STAGE]: Docker-Execute: {self.image}:{self.tag}")
|
2225
|
+
|
2065
2226
|
raise NotImplementedError("Docker Stage does not implement yet.")
|
2066
2227
|
|
2067
2228
|
|
2068
|
-
# TODO: Not implement this stages yet
|
2069
2229
|
class VirtualPyStage(PyStage): # pragma: no cov
|
2070
|
-
"""
|
2230
|
+
"""Virtual Python stage executor that run Python statement on the dependent
|
2231
|
+
Python virtual environment via the `uv` package.
|
2232
|
+
"""
|
2071
2233
|
|
2234
|
+
version: str = Field(
|
2235
|
+
default="3.9",
|
2236
|
+
description="A Python version that want to run.",
|
2237
|
+
)
|
2072
2238
|
deps: list[str] = Field(
|
2073
2239
|
description=(
|
2074
2240
|
"list of Python dependency that want to install before execution "
|
@@ -2076,7 +2242,54 @@ class VirtualPyStage(PyStage): # pragma: no cov
|
|
2076
2242
|
),
|
2077
2243
|
)
|
2078
2244
|
|
2079
|
-
|
2245
|
+
@contextlib.contextmanager
|
2246
|
+
def create_py_file(
|
2247
|
+
self,
|
2248
|
+
py: str,
|
2249
|
+
values: DictData,
|
2250
|
+
deps: list[str],
|
2251
|
+
run_id: str | None = None,
|
2252
|
+
) -> Iterator[str]:
|
2253
|
+
"""Create the .py file with an input Python string statement.
|
2254
|
+
|
2255
|
+
:param py: A Python string statement.
|
2256
|
+
:param values: A variable that want to set before running this
|
2257
|
+
:param deps: An additional Python dependencies that want install before
|
2258
|
+
run this python stage.
|
2259
|
+
:param run_id: (str | None) A running ID of this stage execution.
|
2260
|
+
"""
|
2261
|
+
run_id: str = run_id or uuid.uuid4()
|
2262
|
+
f_name: str = f"{run_id}.py"
|
2263
|
+
with open(f"./{f_name}", mode="w", newline="\n") as f:
|
2264
|
+
# NOTE: Create variable mapping that write before running statement.
|
2265
|
+
vars_str: str = "\n ".join(
|
2266
|
+
f"{var} = {value!r}" for var, value in values.items()
|
2267
|
+
)
|
2268
|
+
|
2269
|
+
# NOTE: uv supports PEP 723 — inline TOML metadata.
|
2270
|
+
f.write(
|
2271
|
+
dedent(
|
2272
|
+
f"""
|
2273
|
+
# /// script
|
2274
|
+
# dependencies = [{', '.join(f'"{dep}"' for dep in deps)}]
|
2275
|
+
# ///
|
2276
|
+
{vars_str}
|
2277
|
+
""".strip(
|
2278
|
+
"\n"
|
2279
|
+
)
|
2280
|
+
)
|
2281
|
+
)
|
2282
|
+
|
2283
|
+
# NOTE: make sure that py script file does not have `\r` char.
|
2284
|
+
f.write("\n" + py.replace("\r\n", "\n"))
|
2285
|
+
|
2286
|
+
# NOTE: Make this .py file able to executable.
|
2287
|
+
make_exec(f"./{f_name}")
|
2288
|
+
|
2289
|
+
yield f_name
|
2290
|
+
|
2291
|
+
# Note: Remove .py file that use to run Python.
|
2292
|
+
Path(f"./{f_name}").unlink()
|
2080
2293
|
|
2081
2294
|
def execute(
|
2082
2295
|
self,
|
@@ -2088,31 +2301,61 @@ class VirtualPyStage(PyStage): # pragma: no cov
|
|
2088
2301
|
"""Execute the Python statement via Python virtual environment.
|
2089
2302
|
|
2090
2303
|
Steps:
|
2091
|
-
- Create python file.
|
2092
|
-
-
|
2093
|
-
- Execution python file with uv and specific `.venv`.
|
2304
|
+
- Create python file with the `uv` syntax.
|
2305
|
+
- Execution python file with `uv run` via Python subprocess module.
|
2094
2306
|
|
2095
|
-
:param params: A parameter
|
2096
|
-
:param result: (Result) A
|
2097
|
-
|
2098
|
-
|
2099
|
-
was not force stopped.
|
2307
|
+
:param params: (DictData) A parameter data.
|
2308
|
+
:param result: (Result) A Result instance for return context and status.
|
2309
|
+
:param event: (Event) An Event manager instance that use to cancel this
|
2310
|
+
execution if it forces stopped by parent execution.
|
2100
2311
|
|
2101
2312
|
:rtype: Result
|
2102
2313
|
"""
|
2103
|
-
|
2104
|
-
|
2105
|
-
|
2106
|
-
|
2314
|
+
result: Result = result or Result(
|
2315
|
+
run_id=gen_id(self.name + (self.id or ""), unique=True),
|
2316
|
+
extras=self.extras,
|
2317
|
+
)
|
2107
2318
|
|
2108
2319
|
result.trace.info(f"[STAGE]: Py-Virtual-Execute: {self.name}")
|
2109
|
-
|
2110
|
-
|
2320
|
+
run: str = param2template(dedent(self.run), params, extras=self.extras)
|
2321
|
+
with self.create_py_file(
|
2322
|
+
py=run,
|
2323
|
+
values=param2template(self.vars, params, extras=self.extras),
|
2324
|
+
deps=param2template(self.deps, params, extras=self.extras),
|
2325
|
+
run_id=result.run_id,
|
2326
|
+
) as py:
|
2327
|
+
result.trace.debug(f"... Create `{py}` file.")
|
2328
|
+
rs: CompletedProcess = subprocess.run(
|
2329
|
+
["uv", "run", py, "--no-cache"],
|
2330
|
+
# ["uv", "run", "--python", "3.9", py],
|
2331
|
+
shell=False,
|
2332
|
+
capture_output=True,
|
2333
|
+
text=True,
|
2334
|
+
)
|
2335
|
+
|
2336
|
+
if rs.returncode > 0:
|
2337
|
+
# NOTE: Prepare stderr message that returning from subprocess.
|
2338
|
+
e: str = (
|
2339
|
+
rs.stderr.encode("utf-8").decode("utf-16")
|
2340
|
+
if "\\x00" in rs.stderr
|
2341
|
+
else rs.stderr
|
2342
|
+
).removesuffix("\n")
|
2343
|
+
raise StageException(
|
2344
|
+
f"Subprocess: {e}\nRunning Statement:\n---\n"
|
2345
|
+
f"```python\n{run}\n```"
|
2346
|
+
)
|
2347
|
+
return result.catch(
|
2348
|
+
status=SUCCESS,
|
2349
|
+
context={
|
2350
|
+
"return_code": rs.returncode,
|
2351
|
+
"stdout": None if (out := rs.stdout.strip("\n")) == "" else out,
|
2352
|
+
"stderr": None if (out := rs.stderr.strip("\n")) == "" else out,
|
2353
|
+
},
|
2111
2354
|
)
|
2112
2355
|
|
2113
2356
|
|
2114
2357
|
# NOTE:
|
2115
|
-
# An order of parsing stage model on the Job model with
|
2358
|
+
# An order of parsing stage model on the Job model with `stages` field.
|
2116
2359
|
# From the current build-in stages, they do not have stage that have the same
|
2117
2360
|
# fields that because of parsing on the Job's stages key.
|
2118
2361
|
#
|
@@ -2121,7 +2364,6 @@ Stage = Annotated[
|
|
2121
2364
|
DockerStage,
|
2122
2365
|
BashStage,
|
2123
2366
|
CallStage,
|
2124
|
-
HookStage,
|
2125
2367
|
TriggerStage,
|
2126
2368
|
ForEachStage,
|
2127
2369
|
UntilStage,
|