ddeutil-workflow 0.0.78__py3-none-any.whl → 0.0.80__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ddeutil/workflow/__about__.py +1 -1
- ddeutil/workflow/__init__.py +2 -6
- ddeutil/workflow/api/routes/job.py +2 -2
- ddeutil/workflow/api/routes/logs.py +5 -5
- ddeutil/workflow/api/routes/workflows.py +3 -3
- ddeutil/workflow/audits.py +547 -176
- ddeutil/workflow/cli.py +19 -1
- ddeutil/workflow/conf.py +10 -20
- ddeutil/workflow/event.py +15 -6
- ddeutil/workflow/job.py +147 -74
- ddeutil/workflow/params.py +172 -58
- ddeutil/workflow/plugins/__init__.py +0 -0
- ddeutil/workflow/plugins/providers/__init__.py +0 -0
- ddeutil/workflow/plugins/providers/aws.py +908 -0
- ddeutil/workflow/plugins/providers/az.py +1003 -0
- ddeutil/workflow/plugins/providers/container.py +703 -0
- ddeutil/workflow/plugins/providers/gcs.py +826 -0
- ddeutil/workflow/result.py +6 -4
- ddeutil/workflow/reusables.py +151 -95
- ddeutil/workflow/stages.py +28 -28
- ddeutil/workflow/traces.py +1697 -541
- ddeutil/workflow/utils.py +109 -67
- ddeutil/workflow/workflow.py +42 -30
- {ddeutil_workflow-0.0.78.dist-info → ddeutil_workflow-0.0.80.dist-info}/METADATA +39 -19
- ddeutil_workflow-0.0.80.dist-info/RECORD +36 -0
- ddeutil_workflow-0.0.78.dist-info/RECORD +0 -30
- {ddeutil_workflow-0.0.78.dist-info → ddeutil_workflow-0.0.80.dist-info}/WHEEL +0 -0
- {ddeutil_workflow-0.0.78.dist-info → ddeutil_workflow-0.0.80.dist-info}/entry_points.txt +0 -0
- {ddeutil_workflow-0.0.78.dist-info → ddeutil_workflow-0.0.80.dist-info}/licenses/LICENSE +0 -0
- {ddeutil_workflow-0.0.78.dist-info → ddeutil_workflow-0.0.80.dist-info}/top_level.txt +0 -0
ddeutil/workflow/utils.py
CHANGED
@@ -10,16 +10,24 @@ system for ID generation, datetime handling, string processing, template
|
|
10
10
|
operations, and other common tasks.
|
11
11
|
|
12
12
|
Functions:
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
delay: Create delays in execution
|
18
|
-
to_train: Convert strings to train-case format
|
13
|
+
to_train: Convert camel case strings to train case format
|
14
|
+
prepare_newline: Format messages with multiple newlines
|
15
|
+
replace_sec: Replace seconds and microseconds in datetime objects
|
16
|
+
clear_tz: Clear timezone info from datetime objects
|
19
17
|
get_dt_now: Get current datetime with timezone
|
20
18
|
get_d_now: Get current date
|
19
|
+
get_diff_sec: Calculate time difference in seconds
|
20
|
+
reach_next_minute: Check if datetime reaches next minute
|
21
|
+
wait_until_next_minute: Wait until next minute
|
22
|
+
delay: Add random delay to execution
|
23
|
+
gen_id: Generate unique identifiers for workflow components
|
24
|
+
default_gen_id: Generate default running ID
|
25
|
+
make_exec: Make files executable
|
26
|
+
filter_func: Filter function objects from data structures
|
21
27
|
cross_product: Generate cross product of matrix values
|
22
|
-
|
28
|
+
cut_id: Cut running ID to specified length
|
29
|
+
dump_all: Serialize nested BaseModel objects to dictionaries
|
30
|
+
obj_name: Get object name or class name
|
23
31
|
|
24
32
|
Example:
|
25
33
|
```python
|
@@ -116,10 +124,11 @@ def clear_tz(dt: datetime) -> datetime:
|
|
116
124
|
def get_dt_now(offset: float = 0.0) -> datetime:
|
117
125
|
"""Return the current datetime object.
|
118
126
|
|
119
|
-
:
|
127
|
+
Args:
|
128
|
+
offset: An offset second value to subtract from current time.
|
120
129
|
|
121
|
-
:
|
122
|
-
|
130
|
+
Returns:
|
131
|
+
datetime: The current datetime object with UTC timezone.
|
123
132
|
"""
|
124
133
|
return datetime.now().replace(tzinfo=UTC) - timedelta(seconds=offset)
|
125
134
|
|
@@ -127,10 +136,11 @@ def get_dt_now(offset: float = 0.0) -> datetime:
|
|
127
136
|
def get_d_now(offset: float = 0.0) -> date: # pragma: no cov
|
128
137
|
"""Return the current date object.
|
129
138
|
|
130
|
-
:
|
139
|
+
Args:
|
140
|
+
offset: An offset second value to subtract from current time.
|
131
141
|
|
132
|
-
:
|
133
|
-
|
142
|
+
Returns:
|
143
|
+
date: The current date object.
|
134
144
|
"""
|
135
145
|
return (
|
136
146
|
datetime.now().replace(tzinfo=UTC) - timedelta(seconds=offset)
|
@@ -138,13 +148,14 @@ def get_d_now(offset: float = 0.0) -> date: # pragma: no cov
|
|
138
148
|
|
139
149
|
|
140
150
|
def get_diff_sec(dt: datetime, offset: float = 0.0) -> int:
|
141
|
-
"""Return second value
|
142
|
-
current datetime with specific timezone.
|
151
|
+
"""Return second value from difference between input datetime and current datetime.
|
143
152
|
|
144
|
-
:
|
145
|
-
|
153
|
+
Args:
|
154
|
+
dt: A datetime object to calculate difference from.
|
155
|
+
offset: An offset second value to add to the difference.
|
146
156
|
|
147
|
-
:
|
157
|
+
Returns:
|
158
|
+
int: The difference in seconds between the input datetime and current time.
|
148
159
|
"""
|
149
160
|
return round(
|
150
161
|
(
|
@@ -154,11 +165,17 @@ def get_diff_sec(dt: datetime, offset: float = 0.0) -> int:
|
|
154
165
|
|
155
166
|
|
156
167
|
def reach_next_minute(dt: datetime, offset: float = 0.0) -> bool:
|
157
|
-
"""Check
|
158
|
-
datetime.
|
168
|
+
"""Check if datetime object reaches the next minute level.
|
159
169
|
|
160
|
-
:
|
161
|
-
|
170
|
+
Args:
|
171
|
+
dt: A datetime object to check.
|
172
|
+
offset: An offset second value.
|
173
|
+
|
174
|
+
Returns:
|
175
|
+
bool: True if datetime reaches next minute, False otherwise.
|
176
|
+
|
177
|
+
Raises:
|
178
|
+
ValueError: If the input datetime is less than current date.
|
162
179
|
"""
|
163
180
|
diff: float = (
|
164
181
|
replace_sec(clear_tz(dt)) - replace_sec(get_dt_now(offset=offset))
|
@@ -176,16 +193,21 @@ def reach_next_minute(dt: datetime, offset: float = 0.0) -> bool:
|
|
176
193
|
def wait_until_next_minute(
|
177
194
|
dt: datetime, second: float = 0
|
178
195
|
) -> None: # pragma: no cov
|
179
|
-
"""Wait with sleep to the next minute with an offset second value.
|
196
|
+
"""Wait with sleep to the next minute with an offset second value.
|
197
|
+
|
198
|
+
Args:
|
199
|
+
dt: The datetime to wait until next minute from.
|
200
|
+
second: Additional seconds to wait after reaching next minute.
|
201
|
+
"""
|
180
202
|
future: datetime = replace_sec(dt) + timedelta(minutes=1)
|
181
203
|
time.sleep((future - dt).total_seconds() + second)
|
182
204
|
|
183
205
|
|
184
206
|
def delay(second: float = 0) -> None: # pragma: no cov
|
185
|
-
"""Delay
|
186
|
-
0.00 - 0.99 seconds.
|
207
|
+
"""Delay execution with time.sleep and random second value between 0.00-0.99 seconds.
|
187
208
|
|
188
|
-
:
|
209
|
+
Args:
|
210
|
+
second: Additional seconds to add to the random delay.
|
189
211
|
"""
|
190
212
|
global _DELAY_INDEX
|
191
213
|
cached_random = _CACHED_DELAYS[_DELAY_INDEX % len(_CACHED_DELAYS)]
|
@@ -201,26 +223,24 @@ def gen_id(
|
|
201
223
|
simple_mode: Optional[bool] = None,
|
202
224
|
extras: DictData | None = None,
|
203
225
|
) -> str:
|
204
|
-
"""Generate running ID for
|
205
|
-
|
206
|
-
|
207
|
-
value
|
208
|
-
|
209
|
-
Simple Mode:
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
220
|
-
:
|
221
|
-
|
222
|
-
|
223
|
-
:rtype: str
|
226
|
+
"""Generate running ID for tracking purposes.
|
227
|
+
|
228
|
+
This function uses MD5 algorithm if simple mode is disabled, or cuts the
|
229
|
+
hashing value length to 10 if simple mode is enabled.
|
230
|
+
|
231
|
+
Simple Mode Format:
|
232
|
+
YYYYMMDDHHMMSSffffffTxxxxxxxxxx
|
233
|
+
year month day hour minute second microsecond sep simple-id
|
234
|
+
|
235
|
+
Args:
|
236
|
+
value: A value to add as prefix before hashing with MD5.
|
237
|
+
sensitive: Flag to convert value to lowercase before hashing.
|
238
|
+
unique: Flag to add timestamp at microsecond level before hashing.
|
239
|
+
simple_mode: Flag to generate ID using simple mode.
|
240
|
+
extras: Extra parameters to override config values.
|
241
|
+
|
242
|
+
Returns:
|
243
|
+
str: Generated unique identifier.
|
224
244
|
"""
|
225
245
|
from .conf import dynamic
|
226
246
|
|
@@ -242,31 +262,37 @@ def gen_id(
|
|
242
262
|
|
243
263
|
|
244
264
|
def default_gen_id() -> str:
|
245
|
-
"""Return running ID
|
246
|
-
|
265
|
+
"""Return running ID for making default ID for the Result model.
|
266
|
+
|
267
|
+
This function is used when a run_id field is initialized for the first time.
|
247
268
|
|
248
|
-
:
|
269
|
+
Returns:
|
270
|
+
str: Generated default running ID.
|
249
271
|
"""
|
250
272
|
return gen_id("MOCK", unique=True)
|
251
273
|
|
252
274
|
|
253
275
|
def make_exec(path: Union[Path, str]) -> None:
|
254
|
-
"""Change mode
|
276
|
+
"""Change file mode to be executable.
|
255
277
|
|
256
|
-
:
|
257
|
-
|
278
|
+
Args:
|
279
|
+
path: A file path to make executable.
|
258
280
|
"""
|
259
281
|
f: Path = Path(path) if isinstance(path, str) else path
|
260
282
|
f.chmod(f.stat().st_mode | stat.S_IEXEC)
|
261
283
|
|
262
284
|
|
263
285
|
def filter_func(value: T) -> T:
|
264
|
-
"""Filter out
|
265
|
-
|
266
|
-
|
286
|
+
"""Filter out custom functions from mapping context by replacing with function names.
|
287
|
+
|
288
|
+
This function replaces custom functions with their function names in data
|
289
|
+
structures. Built-in functions remain unchanged.
|
267
290
|
|
268
|
-
:
|
269
|
-
|
291
|
+
Args:
|
292
|
+
value: A value or data structure to filter function values from.
|
293
|
+
|
294
|
+
Returns:
|
295
|
+
T: The filtered value with functions replaced by their names.
|
270
296
|
"""
|
271
297
|
if isinstance(value, dict):
|
272
298
|
return {k: filter_func(value[k]) for k in value}
|
@@ -287,11 +313,13 @@ def filter_func(value: T) -> T:
|
|
287
313
|
|
288
314
|
|
289
315
|
def cross_product(matrix: Matrix) -> Iterator[DictData]:
|
290
|
-
"""
|
316
|
+
"""Generate iterator of product values from matrix.
|
291
317
|
|
292
|
-
:
|
318
|
+
Args:
|
319
|
+
matrix: A matrix to generate cross products from.
|
293
320
|
|
294
|
-
:
|
321
|
+
Returns:
|
322
|
+
Iterator[DictData]: Iterator of dictionary combinations.
|
295
323
|
"""
|
296
324
|
yield from (
|
297
325
|
{_k: _v for e in mapped for _k, _v in e.items()}
|
@@ -302,16 +330,18 @@ def cross_product(matrix: Matrix) -> Iterator[DictData]:
|
|
302
330
|
|
303
331
|
|
304
332
|
def cut_id(run_id: str, *, num: int = 6) -> str:
|
305
|
-
"""
|
333
|
+
"""Cut running ID to specified length.
|
306
334
|
|
307
335
|
Example:
|
308
336
|
>>> cut_id(run_id='20240101081330000000T1354680202')
|
309
337
|
'202401010813680202'
|
310
338
|
|
311
|
-
:
|
312
|
-
|
339
|
+
Args:
|
340
|
+
run_id: A running ID to cut.
|
341
|
+
num: Number of characters to keep from the end.
|
313
342
|
|
314
|
-
:
|
343
|
+
Returns:
|
344
|
+
str: The cut running ID.
|
315
345
|
"""
|
316
346
|
if "T" in run_id:
|
317
347
|
dt, simple = run_id.split("T", maxsplit=1)
|
@@ -333,10 +363,14 @@ def dump_all(
|
|
333
363
|
value: Union[T, BaseModel],
|
334
364
|
by_alias: bool = False,
|
335
365
|
) -> Union[T, DictData]:
|
336
|
-
"""Dump all nested BaseModel
|
366
|
+
"""Dump all nested BaseModel objects to dictionary objects.
|
337
367
|
|
338
|
-
:
|
339
|
-
|
368
|
+
Args:
|
369
|
+
value: A value that may contain BaseModel objects.
|
370
|
+
by_alias: Whether to use field aliases when dumping.
|
371
|
+
|
372
|
+
Returns:
|
373
|
+
Union[T, DictData]: The value with BaseModel objects converted to dictionaries.
|
340
374
|
"""
|
341
375
|
if isinstance(value, dict):
|
342
376
|
return {k: dump_all(value[k], by_alias=by_alias) for k in value}
|
@@ -351,6 +385,14 @@ def dump_all(
|
|
351
385
|
|
352
386
|
|
353
387
|
def obj_name(obj: Optional[Union[str, object]] = None) -> Optional[str]:
|
388
|
+
"""Get object name or class name.
|
389
|
+
|
390
|
+
Args:
|
391
|
+
obj: An object or string to get the name from.
|
392
|
+
|
393
|
+
Returns:
|
394
|
+
Optional[str]: The object name, class name, or None if obj is None.
|
395
|
+
"""
|
354
396
|
if not obj:
|
355
397
|
return None
|
356
398
|
elif isinstance(obj, str):
|
ddeutil/workflow/workflow.py
CHANGED
@@ -42,7 +42,7 @@ from pydantic.functional_validators import field_validator, model_validator
|
|
42
42
|
from typing_extensions import Self
|
43
43
|
|
44
44
|
from .__types import DictData
|
45
|
-
from .audits import Audit,
|
45
|
+
from .audits import Audit, get_audit
|
46
46
|
from .conf import YamlParser, dynamic
|
47
47
|
from .errors import WorkflowCancelError, WorkflowError, WorkflowTimeoutError
|
48
48
|
from .event import Event
|
@@ -61,7 +61,7 @@ from .result import (
|
|
61
61
|
validate_statuses,
|
62
62
|
)
|
63
63
|
from .reusables import has_template, param2template
|
64
|
-
from .traces import
|
64
|
+
from .traces import TraceManager, get_trace
|
65
65
|
from .utils import (
|
66
66
|
UTC,
|
67
67
|
gen_id,
|
@@ -407,7 +407,7 @@ class Workflow(BaseModel):
|
|
407
407
|
with the set `on` field.
|
408
408
|
|
409
409
|
Args:
|
410
|
-
dt: A datetime object that want to validate.
|
410
|
+
dt (datetime): A datetime object that want to validate.
|
411
411
|
|
412
412
|
Returns:
|
413
413
|
datetime: The validated release datetime.
|
@@ -416,6 +416,8 @@ class Workflow(BaseModel):
|
|
416
416
|
dt = dt.replace(tzinfo=UTC)
|
417
417
|
|
418
418
|
release: datetime = replace_sec(dt.astimezone(UTC))
|
419
|
+
|
420
|
+
# NOTE: Return itself if schedule event does not set.
|
419
421
|
if not self.on.schedule:
|
420
422
|
return release
|
421
423
|
|
@@ -454,7 +456,7 @@ class Workflow(BaseModel):
|
|
454
456
|
- Writing result audit
|
455
457
|
|
456
458
|
Args:
|
457
|
-
release
|
459
|
+
release (datetime): A release datetime.
|
458
460
|
params: A workflow parameter that pass to execute method.
|
459
461
|
release_type:
|
460
462
|
run_id: (str) A workflow running ID.
|
@@ -481,7 +483,7 @@ class Workflow(BaseModel):
|
|
481
483
|
parent_run_id: str = run_id
|
482
484
|
|
483
485
|
context: DictData = {"status": WAIT}
|
484
|
-
trace:
|
486
|
+
trace: TraceManager = get_trace(
|
485
487
|
run_id, parent_run_id=parent_run_id, extras=self.extras
|
486
488
|
)
|
487
489
|
release: datetime = self.validate_release(dt=release)
|
@@ -507,24 +509,27 @@ class Workflow(BaseModel):
|
|
507
509
|
trace.info(f"[RELEASE]: End {name!r} : {release:%Y-%m-%d %H:%M:%S}")
|
508
510
|
trace.debug(f"[RELEASE]: Writing audit: {name!r}.")
|
509
511
|
(
|
510
|
-
(audit or
|
511
|
-
|
512
|
-
|
513
|
-
|
514
|
-
|
515
|
-
|
516
|
-
|
517
|
-
|
518
|
-
|
519
|
-
|
520
|
-
|
521
|
-
|
522
|
-
|
523
|
-
|
524
|
-
|
525
|
-
|
526
|
-
|
527
|
-
|
512
|
+
(audit or get_audit(extras=self.extras)).save(
|
513
|
+
data={
|
514
|
+
"name": name,
|
515
|
+
"release": release,
|
516
|
+
"type": release_type,
|
517
|
+
"context": context,
|
518
|
+
"parent_run_id": parent_run_id,
|
519
|
+
"run_id": run_id,
|
520
|
+
"extras": self.extras,
|
521
|
+
"runs_metadata": (
|
522
|
+
(runs_metadata or {})
|
523
|
+
| rs.info
|
524
|
+
| {
|
525
|
+
"timeout": timeout,
|
526
|
+
"original_name": self.name,
|
527
|
+
"audit_excluded": audit_excluded,
|
528
|
+
}
|
529
|
+
),
|
530
|
+
},
|
531
|
+
excluded=audit_excluded,
|
532
|
+
)
|
528
533
|
)
|
529
534
|
return Result(
|
530
535
|
run_id=run_id,
|
@@ -576,7 +581,7 @@ class Workflow(BaseModel):
|
|
576
581
|
Returns:
|
577
582
|
tuple[Status, DictData]: The pair of status and result context data.
|
578
583
|
"""
|
579
|
-
trace:
|
584
|
+
trace: TraceManager = get_trace(
|
580
585
|
run_id, parent_run_id=parent_run_id, extras=self.extras
|
581
586
|
)
|
582
587
|
if event and event.is_set():
|
@@ -693,7 +698,7 @@ class Workflow(BaseModel):
|
|
693
698
|
ts: float = time.monotonic()
|
694
699
|
parent_run_id: Optional[str] = run_id
|
695
700
|
run_id: str = gen_id(self.name, extras=self.extras)
|
696
|
-
trace:
|
701
|
+
trace: TraceManager = get_trace(
|
697
702
|
run_id, parent_run_id=parent_run_id, extras=self.extras
|
698
703
|
)
|
699
704
|
context: DictData = self.parameterize(params)
|
@@ -750,8 +755,12 @@ class Workflow(BaseModel):
|
|
750
755
|
|
751
756
|
with ThreadPoolExecutor(max_job_parallel, "wf") as executor:
|
752
757
|
futures: list[Future] = []
|
753
|
-
|
754
|
-
|
758
|
+
|
759
|
+
# NOTE: Start with smaller sleep time
|
760
|
+
backoff_sleep: float = 0.01
|
761
|
+
|
762
|
+
# NOTE: Track consecutive wait states
|
763
|
+
consecutive_waits: int = 0
|
755
764
|
|
756
765
|
while not job_queue.empty() and (
|
757
766
|
not_timeout_flag := ((time.monotonic() - ts) < timeout)
|
@@ -799,6 +808,7 @@ class Workflow(BaseModel):
|
|
799
808
|
skip_count += 1
|
800
809
|
continue
|
801
810
|
|
811
|
+
# IMPORTANT: Start execution with parallel mode.
|
802
812
|
if max_job_parallel > 1:
|
803
813
|
futures.append(
|
804
814
|
executor.submit(
|
@@ -831,7 +841,9 @@ class Workflow(BaseModel):
|
|
831
841
|
st, _ = future.result()
|
832
842
|
sequence_statuses.append(st)
|
833
843
|
job_queue.put(job_id)
|
834
|
-
|
844
|
+
# NOTE: The release future can not track a cancelled status
|
845
|
+
# because it only has one future.
|
846
|
+
elif future.cancelled(): # pragma: no cov
|
835
847
|
sequence_statuses.append(CANCEL)
|
836
848
|
job_queue.put(job_id)
|
837
849
|
elif future.running() or "state=pending" in str(future):
|
@@ -852,7 +864,7 @@ class Workflow(BaseModel):
|
|
852
864
|
for total, future in enumerate(as_completed(futures), start=0):
|
853
865
|
try:
|
854
866
|
statuses[total], _ = future.result()
|
855
|
-
except WorkflowError as e:
|
867
|
+
except (WorkflowError, Exception) as e:
|
856
868
|
statuses[total] = get_status_from_error(e)
|
857
869
|
|
858
870
|
# NOTE: Update skipped status from the job trigger.
|
@@ -931,7 +943,7 @@ class Workflow(BaseModel):
|
|
931
943
|
ts: float = time.monotonic()
|
932
944
|
parent_run_id: str = run_id
|
933
945
|
run_id: str = gen_id(self.name, extras=self.extras)
|
934
|
-
trace:
|
946
|
+
trace: TraceManager = get_trace(
|
935
947
|
run_id, parent_run_id=parent_run_id, extras=self.extras
|
936
948
|
)
|
937
949
|
if context["status"] == SUCCESS:
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: ddeutil-workflow
|
3
|
-
Version: 0.0.
|
3
|
+
Version: 0.0.80
|
4
4
|
Summary: Lightweight workflow orchestration with YAML template
|
5
5
|
Author-email: ddeutils <korawich.anu@gmail.com>
|
6
6
|
License: MIT
|
@@ -24,7 +24,7 @@ Description-Content-Type: text/markdown
|
|
24
24
|
License-File: LICENSE
|
25
25
|
Requires-Dist: ddeutil[checksum]>=0.4.8
|
26
26
|
Requires-Dist: ddeutil-io[toml,yaml]>=0.2.14
|
27
|
-
Requires-Dist: pydantic<3.0.0,==2.11.
|
27
|
+
Requires-Dist: pydantic<3.0.0,==2.11.7
|
28
28
|
Requires-Dist: pydantic-extra-types<3.0.0,>=2.10.4
|
29
29
|
Requires-Dist: python-dotenv>=1.1.0
|
30
30
|
Requires-Dist: typer>=0.16.0
|
@@ -35,9 +35,17 @@ Requires-Dist: httpx; extra == "all"
|
|
35
35
|
Requires-Dist: ujson; extra == "all"
|
36
36
|
Requires-Dist: aiofiles; extra == "all"
|
37
37
|
Requires-Dist: aiohttp; extra == "all"
|
38
|
-
Requires-Dist: requests==2.32.
|
38
|
+
Requires-Dist: requests==2.32.4; extra == "all"
|
39
39
|
Provides-Extra: docker
|
40
40
|
Requires-Dist: docker==7.1.0; extra == "docker"
|
41
|
+
Provides-Extra: azure
|
42
|
+
Requires-Dist: azure-batch>=13.0.0; extra == "azure"
|
43
|
+
Requires-Dist: azure-storage-blob>=12.0.0; extra == "azure"
|
44
|
+
Provides-Extra: aws
|
45
|
+
Requires-Dist: boto3>=1.39.0; extra == "aws"
|
46
|
+
Provides-Extra: gcp
|
47
|
+
Requires-Dist: google-cloud-batch>=0.17.0; extra == "gcp"
|
48
|
+
Requires-Dist: google-cloud-storage>=2.10.0; extra == "gcp"
|
41
49
|
Dynamic: license-file
|
42
50
|
|
43
51
|
# Workflow Orchestration
|
@@ -140,6 +148,19 @@ If you want to install this package with application add-ons, you should add
|
|
140
148
|
| Python | `ddeutil-workflow` | ✅ |
|
141
149
|
| FastAPI Server | `ddeutil-workflow[all]` | ✅ |
|
142
150
|
|
151
|
+
Check the version of the current workflow package:
|
152
|
+
|
153
|
+
```shell
|
154
|
+
$ pip install ddeutil-workflow
|
155
|
+
$ workflow-cli version
|
156
|
+
```
|
157
|
+
|
158
|
+
Initial workflow project:
|
159
|
+
|
160
|
+
```shell
|
161
|
+
$ workflow-cli init
|
162
|
+
```
|
163
|
+
|
143
164
|
## 📖 Documentation
|
144
165
|
|
145
166
|
For comprehensive API documentation, examples, and best practices:
|
@@ -282,22 +303,21 @@ it will use default value and do not raise any error to you.
|
|
282
303
|
> The config value that you will set on the environment should combine with
|
283
304
|
> prefix, component, and name which is `WORKFLOW_{component}_{name}` (Upper case).
|
284
305
|
|
285
|
-
| Name | Component | Default | Description
|
286
|
-
|
287
|
-
| **REGISTRY_CALLER** | CORE | `.` | List of importable string for the call stage.
|
288
|
-
| **REGISTRY_FILTER** | CORE | `ddeutil.workflow.templates` | List of importable string for the filter template.
|
289
|
-
| **CONF_PATH** | CORE | `./conf` | The config path that keep all template `.yaml` files.
|
290
|
-
| **STAGE_DEFAULT_ID** | CORE | `false` | A flag that enable default stage ID that use for catch an execution output.
|
291
|
-
| **GENERATE_ID_SIMPLE_MODE** | CORE | `true` | A flog that enable generating ID with `md5` algorithm.
|
292
|
-
| **DEBUG_MODE** | LOG | `true` | A flag that enable logging with debug level mode.
|
293
|
-
| **TIMEZONE** | LOG | `Asia/Bangkok` | A Timezone string value that will pass to `ZoneInfo` object.
|
294
|
-
| **FORMAT** | LOG | `%(asctime)s.%(msecs)03d (%(name)-10s, %(process)-5d,%(thread)-5d) [%(levelname)-7s] %(message)-120s (%(filename)s:%(lineno)s)` | A trace message console format.
|
295
|
-
| **FORMAT_FILE** | LOG | `{datetime} ({process:5d}, {thread:5d}) {message:120s} ({filename}:{lineno})` | A trace message format that use to write to target pointer.
|
296
|
-
| **DATETIME_FORMAT** | LOG | `%Y-%m-%d %H:%M:%S` | A datetime format of the trace log.
|
297
|
-
| **
|
298
|
-
| **
|
299
|
-
| **
|
300
|
-
| **AUDIT_ENABLE_WRITE** | LOG | `true` | A flag that enable writing audit log after end execution in the workflow release step. |
|
306
|
+
| Name | Component | Default | Description |
|
307
|
+
|:----------------------------|:---------:|:--------------------------------------------------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------|
|
308
|
+
| **REGISTRY_CALLER** | CORE | `.` | List of importable string for the call stage. |
|
309
|
+
| **REGISTRY_FILTER** | CORE | `ddeutil.workflow.templates` | List of importable string for the filter template. |
|
310
|
+
| **CONF_PATH** | CORE | `./conf` | The config path that keep all template `.yaml` files. |
|
311
|
+
| **STAGE_DEFAULT_ID** | CORE | `false` | A flag that enable default stage ID that use for catch an execution output. |
|
312
|
+
| **GENERATE_ID_SIMPLE_MODE** | CORE | `true` | A flog that enable generating ID with `md5` algorithm. |
|
313
|
+
| **DEBUG_MODE** | LOG | `true` | A flag that enable logging with debug level mode. |
|
314
|
+
| **TIMEZONE** | LOG | `Asia/Bangkok` | A Timezone string value that will pass to `ZoneInfo` object. |
|
315
|
+
| **FORMAT** | LOG | `%(asctime)s.%(msecs)03d (%(name)-10s, %(process)-5d,%(thread)-5d) [%(levelname)-7s] %(message)-120s (%(filename)s:%(lineno)s)` | A trace message console format. |
|
316
|
+
| **FORMAT_FILE** | LOG | `{datetime} ({process:5d}, {thread:5d}) {message:120s} ({filename}:{lineno})` | A trace message format that use to write to target pointer. |
|
317
|
+
| **DATETIME_FORMAT** | LOG | `%Y-%m-%d %H:%M:%S` | A datetime format of the trace log. |
|
318
|
+
| **TRACE_HANDLERS** | LOG | `[{"type": "console"}]` | A pointer URL of trace log that use to emit log message. Now uses optimized handler by default. |
|
319
|
+
| **AUDIT_CONF** | LOG | `{"type": "file", "path": "./audits"}` | A pointer URL of audit log that use to write audit metrix. |
|
320
|
+
| **AUDIT_ENABLE_WRITE** | LOG | `true` | A flag that enable writing audit log after end execution in the workflow release step. |
|
301
321
|
|
302
322
|
## :rocket: Deployment
|
303
323
|
|
@@ -0,0 +1,36 @@
|
|
1
|
+
ddeutil/workflow/__about__.py,sha256=0fYC3KyobGtx1NtfXlGCbR9mrpL_yrH0UY_kp0NWaN8,28
|
2
|
+
ddeutil/workflow/__cron.py,sha256=avOagaHl9xXOmizeRWm13cOrty9Tw0vRjFq-xoEgpAY,29167
|
3
|
+
ddeutil/workflow/__init__.py,sha256=pRnUNCrwnKGrEQNSmdU9Ybf1tEQKg4LfsCpyj1Y3mhg,3241
|
4
|
+
ddeutil/workflow/__main__.py,sha256=Qd-f8z2Q2vpiEP2x6PBFsJrpACWDVxFKQk820MhFmHo,59
|
5
|
+
ddeutil/workflow/__types.py,sha256=tA2vsr6mzTSzbWB1sb62c5GgxODlfVRz6FvgLNJtQao,4788
|
6
|
+
ddeutil/workflow/audits.py,sha256=h7WVHe3z_FqtmLRvAxhnSqae8fFtZPN0tICrVr39wP4,25456
|
7
|
+
ddeutil/workflow/cli.py,sha256=aNFOZ3Re_QJBBP6vkT9Lsjrg8wLxrw_LKrl-1SIvSOg,8152
|
8
|
+
ddeutil/workflow/conf.py,sha256=4bFl1ufX7-p6ely7tJnanpTZ0wJoik81yTcmBrbcyxY,16661
|
9
|
+
ddeutil/workflow/errors.py,sha256=UpUIqoyqkvzqjuxtUQ9535l1HeAsyh-plEG0PgDVR2w,5541
|
10
|
+
ddeutil/workflow/event.py,sha256=qm7QHw-Pozm6oIUzAIxpDkPzzVZVtHgJIUlIle0vEfQ,13943
|
11
|
+
ddeutil/workflow/job.py,sha256=lSmOgh4l3_gBJXrTEWULhSSol648h6zPe6zKzz8jDHQ,46597
|
12
|
+
ddeutil/workflow/params.py,sha256=y9f6DEIyae1j4awbj3Kbeq75-U2UPFlKv9K57Hdo_Go,17188
|
13
|
+
ddeutil/workflow/result.py,sha256=BOk3DZMtmdE7xzQYeEYTGFlIkzJQ4Ed3fYzf0zF8Jo8,8963
|
14
|
+
ddeutil/workflow/reusables.py,sha256=g_Cac3yHy0H5ffl4Bb8_eGl284ELxOuX4LI8GYPMZgw,24983
|
15
|
+
ddeutil/workflow/stages.py,sha256=QufIa2b7A_ngOndVoGzyxKm_o5ZrauNeqxAC4vBkKFM,122678
|
16
|
+
ddeutil/workflow/traces.py,sha256=e12_rDnwVo-XR6Ca1KLwCjo3tAwbJP7yXc1YU62YOt8,73415
|
17
|
+
ddeutil/workflow/utils.py,sha256=-E-Z5hN_UTFuWDk-NpfKhNj0QtLfJSvZNDI5NzJsd5E,12122
|
18
|
+
ddeutil/workflow/workflow.py,sha256=Uojf7k7l91sqOlsPMeSPwQmbrB5pgbWEmx9QgKYngmI,42924
|
19
|
+
ddeutil/workflow/api/__init__.py,sha256=5DzYL3ngceoRshh5HYCSVWChqNJSiP01E1bEd8XxPi0,4799
|
20
|
+
ddeutil/workflow/api/log_conf.py,sha256=WfS3udDLSyrP-C80lWOvxxmhd_XWKvQPkwDqKblcH3E,1834
|
21
|
+
ddeutil/workflow/api/routes/__init__.py,sha256=JRaJZB0D6mgR17MbZo8yLtdYDtD62AA8MdKlFqhG84M,420
|
22
|
+
ddeutil/workflow/api/routes/job.py,sha256=8eu2OAOS3fENQ54OO723lFpzgHMyz1D-b_xZj6OnmcA,2550
|
23
|
+
ddeutil/workflow/api/routes/logs.py,sha256=O0I9L059SvtVHZ-TXCShChxbrHKUoT7MYRK0xZWwIMc,5282
|
24
|
+
ddeutil/workflow/api/routes/workflows.py,sha256=0pEZEsIrscRFBXG9gf6nttKw0aNbcdw7NsAZKLoKWtk,4392
|
25
|
+
ddeutil/workflow/plugins/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
26
|
+
ddeutil/workflow/plugins/providers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
27
|
+
ddeutil/workflow/plugins/providers/aws.py,sha256=61uIFBEWt-_D5Sui24qUPier1Hiqlw_RP_eY-rXBCKc,31551
|
28
|
+
ddeutil/workflow/plugins/providers/az.py,sha256=o3dh011lEtmr7-d7FPZJPgXdT0ytFzKfc5xnVxSyXGU,34867
|
29
|
+
ddeutil/workflow/plugins/providers/container.py,sha256=DSN0RWxMjTJN5ANheeMauDaPa3X6Z2E1eGUcctYkENw,22134
|
30
|
+
ddeutil/workflow/plugins/providers/gcs.py,sha256=KgAOdMBvdbMLTH_z_FwVriBFtZfKEYx8_34jzUOVjTY,27460
|
31
|
+
ddeutil_workflow-0.0.80.dist-info/licenses/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
|
32
|
+
ddeutil_workflow-0.0.80.dist-info/METADATA,sha256=g43RJsjquJoYyQUl_d5Wfuxdoh00tukjjjgW-lL7Hdw,16087
|
33
|
+
ddeutil_workflow-0.0.80.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
34
|
+
ddeutil_workflow-0.0.80.dist-info/entry_points.txt,sha256=qDTpPSauL0ciO6T4iSVt8bJeYrVEkkoEEw_RlGx6Kgk,63
|
35
|
+
ddeutil_workflow-0.0.80.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
|
36
|
+
ddeutil_workflow-0.0.80.dist-info/RECORD,,
|
@@ -1,30 +0,0 @@
|
|
1
|
-
ddeutil/workflow/__about__.py,sha256=9g_DJubkaHxt4W_0-r1PmUT5x3bCo_72c-f60EDtlq8,28
|
2
|
-
ddeutil/workflow/__cron.py,sha256=avOagaHl9xXOmizeRWm13cOrty9Tw0vRjFq-xoEgpAY,29167
|
3
|
-
ddeutil/workflow/__init__.py,sha256=_8sP-CTPOfwsFFhmdwQ2Gp7yY7qJemP7TYsIWgd5jc0,3300
|
4
|
-
ddeutil/workflow/__main__.py,sha256=Qd-f8z2Q2vpiEP2x6PBFsJrpACWDVxFKQk820MhFmHo,59
|
5
|
-
ddeutil/workflow/__types.py,sha256=tA2vsr6mzTSzbWB1sb62c5GgxODlfVRz6FvgLNJtQao,4788
|
6
|
-
ddeutil/workflow/audits.py,sha256=wANG0jEQ7slUSgVZG4JbjlR5PtmF8mHpM9RH-zpYM_g,12679
|
7
|
-
ddeutil/workflow/cli.py,sha256=WnUkxqs2hCc5JVuTvWuEGKp8_EcS_wmhVvSXDhj0eEM,7544
|
8
|
-
ddeutil/workflow/conf.py,sha256=7yMSBVh2W9KcDOZxQjeabJZizB_3ydCLICo4JI0syWU,16892
|
9
|
-
ddeutil/workflow/errors.py,sha256=UpUIqoyqkvzqjuxtUQ9535l1HeAsyh-plEG0PgDVR2w,5541
|
10
|
-
ddeutil/workflow/event.py,sha256=6d5_UvnPI8xRLcX_5wptmvWoXUIGs_JZbjq7khz5oYE,13355
|
11
|
-
ddeutil/workflow/job.py,sha256=4_IxtoIwl1qVuSaNngelYNkqQl9ZOa-PHdKCnXvXu0M,43943
|
12
|
-
ddeutil/workflow/params.py,sha256=Cyz142OcvENIZrM7Efc2xuGPmmFBhROifP5ojoaCezg,13658
|
13
|
-
ddeutil/workflow/result.py,sha256=Xi07E3WuMHS1jLcJg7p4DPuaMFGp0yEDaWCJRotOH6g,8921
|
14
|
-
ddeutil/workflow/reusables.py,sha256=pbCHsEl2V3jGWDRcGyxDvGN5rP5kaRxZNgv9x6-pINQ,23338
|
15
|
-
ddeutil/workflow/stages.py,sha256=ocsk64duS4BHEOLT3Agkx-fbY6iYmygxCaQhwp6YyyM,122482
|
16
|
-
ddeutil/workflow/traces.py,sha256=h7oDlb4Q8LJUp0pste2dWJYOqEaN64KsLTNMfeRUqx8,28475
|
17
|
-
ddeutil/workflow/utils.py,sha256=vSGdpaFgQ5vUPxWvVfbNC2__tu5q16B9mhx1BRbEuJo,10968
|
18
|
-
ddeutil/workflow/workflow.py,sha256=0KaRCTAipPvvmi9SOC7T0V0CAusgebCM6qju0ethGF0,42409
|
19
|
-
ddeutil/workflow/api/__init__.py,sha256=5DzYL3ngceoRshh5HYCSVWChqNJSiP01E1bEd8XxPi0,4799
|
20
|
-
ddeutil/workflow/api/log_conf.py,sha256=WfS3udDLSyrP-C80lWOvxxmhd_XWKvQPkwDqKblcH3E,1834
|
21
|
-
ddeutil/workflow/api/routes/__init__.py,sha256=JRaJZB0D6mgR17MbZo8yLtdYDtD62AA8MdKlFqhG84M,420
|
22
|
-
ddeutil/workflow/api/routes/job.py,sha256=-lbZ_hS9pEdSy6zeke5qrXEgdNxtQ2w9in7cHuM2Jzs,2536
|
23
|
-
ddeutil/workflow/api/routes/logs.py,sha256=RiZ62eQVMWArPHE3lpan955U4DdLLkethlvSMlwF7Mg,5312
|
24
|
-
ddeutil/workflow/api/routes/workflows.py,sha256=1Mqx4Hft4uJglgJI-Wcw-JzkhomFYZrtP0DnQDBkAFQ,4410
|
25
|
-
ddeutil_workflow-0.0.78.dist-info/licenses/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
|
26
|
-
ddeutil_workflow-0.0.78.dist-info/METADATA,sha256=2phJ3JRH9o0Sbr22iOo1YeQBr8nyYx6areM2rwNp_Eg,15681
|
27
|
-
ddeutil_workflow-0.0.78.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
28
|
-
ddeutil_workflow-0.0.78.dist-info/entry_points.txt,sha256=qDTpPSauL0ciO6T4iSVt8bJeYrVEkkoEEw_RlGx6Kgk,63
|
29
|
-
ddeutil_workflow-0.0.78.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
|
30
|
-
ddeutil_workflow-0.0.78.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|