ddeutil-workflow 0.0.78__py3-none-any.whl → 0.0.79__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ddeutil/workflow/__about__.py +1 -1
- ddeutil/workflow/__init__.py +1 -5
- ddeutil/workflow/api/routes/job.py +2 -2
- ddeutil/workflow/audits.py +554 -112
- ddeutil/workflow/cli.py +19 -1
- ddeutil/workflow/conf.py +9 -21
- ddeutil/workflow/event.py +15 -6
- ddeutil/workflow/job.py +147 -73
- ddeutil/workflow/params.py +172 -58
- ddeutil/workflow/plugins/__init__.py +0 -0
- ddeutil/workflow/plugins/providers/__init__.py +0 -0
- ddeutil/workflow/plugins/providers/aws.py +908 -0
- ddeutil/workflow/plugins/providers/az.py +1003 -0
- ddeutil/workflow/plugins/providers/container.py +703 -0
- ddeutil/workflow/plugins/providers/gcs.py +826 -0
- ddeutil/workflow/result.py +6 -4
- ddeutil/workflow/reusables.py +151 -95
- ddeutil/workflow/stages.py +28 -28
- ddeutil/workflow/traces.py +1678 -540
- ddeutil/workflow/utils.py +109 -67
- ddeutil/workflow/workflow.py +20 -11
- {ddeutil_workflow-0.0.78.dist-info → ddeutil_workflow-0.0.79.dist-info}/METADATA +52 -19
- ddeutil_workflow-0.0.79.dist-info/RECORD +36 -0
- ddeutil_workflow-0.0.78.dist-info/RECORD +0 -30
- {ddeutil_workflow-0.0.78.dist-info → ddeutil_workflow-0.0.79.dist-info}/WHEEL +0 -0
- {ddeutil_workflow-0.0.78.dist-info → ddeutil_workflow-0.0.79.dist-info}/entry_points.txt +0 -0
- {ddeutil_workflow-0.0.78.dist-info → ddeutil_workflow-0.0.79.dist-info}/licenses/LICENSE +0 -0
- {ddeutil_workflow-0.0.78.dist-info → ddeutil_workflow-0.0.79.dist-info}/top_level.txt +0 -0
ddeutil/workflow/utils.py
CHANGED
@@ -10,16 +10,24 @@ system for ID generation, datetime handling, string processing, template
|
|
10
10
|
operations, and other common tasks.
|
11
11
|
|
12
12
|
Functions:
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
delay: Create delays in execution
|
18
|
-
to_train: Convert strings to train-case format
|
13
|
+
to_train: Convert camel case strings to train case format
|
14
|
+
prepare_newline: Format messages with multiple newlines
|
15
|
+
replace_sec: Replace seconds and microseconds in datetime objects
|
16
|
+
clear_tz: Clear timezone info from datetime objects
|
19
17
|
get_dt_now: Get current datetime with timezone
|
20
18
|
get_d_now: Get current date
|
19
|
+
get_diff_sec: Calculate time difference in seconds
|
20
|
+
reach_next_minute: Check if datetime reaches next minute
|
21
|
+
wait_until_next_minute: Wait until next minute
|
22
|
+
delay: Add random delay to execution
|
23
|
+
gen_id: Generate unique identifiers for workflow components
|
24
|
+
default_gen_id: Generate default running ID
|
25
|
+
make_exec: Make files executable
|
26
|
+
filter_func: Filter function objects from data structures
|
21
27
|
cross_product: Generate cross product of matrix values
|
22
|
-
|
28
|
+
cut_id: Cut running ID to specified length
|
29
|
+
dump_all: Serialize nested BaseModel objects to dictionaries
|
30
|
+
obj_name: Get object name or class name
|
23
31
|
|
24
32
|
Example:
|
25
33
|
```python
|
@@ -116,10 +124,11 @@ def clear_tz(dt: datetime) -> datetime:
|
|
116
124
|
def get_dt_now(offset: float = 0.0) -> datetime:
|
117
125
|
"""Return the current datetime object.
|
118
126
|
|
119
|
-
:
|
127
|
+
Args:
|
128
|
+
offset: An offset second value to subtract from current time.
|
120
129
|
|
121
|
-
:
|
122
|
-
|
130
|
+
Returns:
|
131
|
+
datetime: The current datetime object with UTC timezone.
|
123
132
|
"""
|
124
133
|
return datetime.now().replace(tzinfo=UTC) - timedelta(seconds=offset)
|
125
134
|
|
@@ -127,10 +136,11 @@ def get_dt_now(offset: float = 0.0) -> datetime:
|
|
127
136
|
def get_d_now(offset: float = 0.0) -> date: # pragma: no cov
|
128
137
|
"""Return the current date object.
|
129
138
|
|
130
|
-
:
|
139
|
+
Args:
|
140
|
+
offset: An offset second value to subtract from current time.
|
131
141
|
|
132
|
-
:
|
133
|
-
|
142
|
+
Returns:
|
143
|
+
date: The current date object.
|
134
144
|
"""
|
135
145
|
return (
|
136
146
|
datetime.now().replace(tzinfo=UTC) - timedelta(seconds=offset)
|
@@ -138,13 +148,14 @@ def get_d_now(offset: float = 0.0) -> date: # pragma: no cov
|
|
138
148
|
|
139
149
|
|
140
150
|
def get_diff_sec(dt: datetime, offset: float = 0.0) -> int:
|
141
|
-
"""Return second value
|
142
|
-
current datetime with specific timezone.
|
151
|
+
"""Return second value from difference between input datetime and current datetime.
|
143
152
|
|
144
|
-
:
|
145
|
-
|
153
|
+
Args:
|
154
|
+
dt: A datetime object to calculate difference from.
|
155
|
+
offset: An offset second value to add to the difference.
|
146
156
|
|
147
|
-
:
|
157
|
+
Returns:
|
158
|
+
int: The difference in seconds between the input datetime and current time.
|
148
159
|
"""
|
149
160
|
return round(
|
150
161
|
(
|
@@ -154,11 +165,17 @@ def get_diff_sec(dt: datetime, offset: float = 0.0) -> int:
|
|
154
165
|
|
155
166
|
|
156
167
|
def reach_next_minute(dt: datetime, offset: float = 0.0) -> bool:
|
157
|
-
"""Check
|
158
|
-
datetime.
|
168
|
+
"""Check if datetime object reaches the next minute level.
|
159
169
|
|
160
|
-
:
|
161
|
-
|
170
|
+
Args:
|
171
|
+
dt: A datetime object to check.
|
172
|
+
offset: An offset second value.
|
173
|
+
|
174
|
+
Returns:
|
175
|
+
bool: True if datetime reaches next minute, False otherwise.
|
176
|
+
|
177
|
+
Raises:
|
178
|
+
ValueError: If the input datetime is less than current date.
|
162
179
|
"""
|
163
180
|
diff: float = (
|
164
181
|
replace_sec(clear_tz(dt)) - replace_sec(get_dt_now(offset=offset))
|
@@ -176,16 +193,21 @@ def reach_next_minute(dt: datetime, offset: float = 0.0) -> bool:
|
|
176
193
|
def wait_until_next_minute(
|
177
194
|
dt: datetime, second: float = 0
|
178
195
|
) -> None: # pragma: no cov
|
179
|
-
"""Wait with sleep to the next minute with an offset second value.
|
196
|
+
"""Wait with sleep to the next minute with an offset second value.
|
197
|
+
|
198
|
+
Args:
|
199
|
+
dt: The datetime to wait until next minute from.
|
200
|
+
second: Additional seconds to wait after reaching next minute.
|
201
|
+
"""
|
180
202
|
future: datetime = replace_sec(dt) + timedelta(minutes=1)
|
181
203
|
time.sleep((future - dt).total_seconds() + second)
|
182
204
|
|
183
205
|
|
184
206
|
def delay(second: float = 0) -> None: # pragma: no cov
|
185
|
-
"""Delay
|
186
|
-
0.00 - 0.99 seconds.
|
207
|
+
"""Delay execution with time.sleep and random second value between 0.00-0.99 seconds.
|
187
208
|
|
188
|
-
:
|
209
|
+
Args:
|
210
|
+
second: Additional seconds to add to the random delay.
|
189
211
|
"""
|
190
212
|
global _DELAY_INDEX
|
191
213
|
cached_random = _CACHED_DELAYS[_DELAY_INDEX % len(_CACHED_DELAYS)]
|
@@ -201,26 +223,24 @@ def gen_id(
|
|
201
223
|
simple_mode: Optional[bool] = None,
|
202
224
|
extras: DictData | None = None,
|
203
225
|
) -> str:
|
204
|
-
"""Generate running ID for
|
205
|
-
|
206
|
-
|
207
|
-
value
|
208
|
-
|
209
|
-
Simple Mode:
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
220
|
-
:
|
221
|
-
|
222
|
-
|
223
|
-
:rtype: str
|
226
|
+
"""Generate running ID for tracking purposes.
|
227
|
+
|
228
|
+
This function uses MD5 algorithm if simple mode is disabled, or cuts the
|
229
|
+
hashing value length to 10 if simple mode is enabled.
|
230
|
+
|
231
|
+
Simple Mode Format:
|
232
|
+
YYYYMMDDHHMMSSffffffTxxxxxxxxxx
|
233
|
+
year month day hour minute second microsecond sep simple-id
|
234
|
+
|
235
|
+
Args:
|
236
|
+
value: A value to add as prefix before hashing with MD5.
|
237
|
+
sensitive: Flag to convert value to lowercase before hashing.
|
238
|
+
unique: Flag to add timestamp at microsecond level before hashing.
|
239
|
+
simple_mode: Flag to generate ID using simple mode.
|
240
|
+
extras: Extra parameters to override config values.
|
241
|
+
|
242
|
+
Returns:
|
243
|
+
str: Generated unique identifier.
|
224
244
|
"""
|
225
245
|
from .conf import dynamic
|
226
246
|
|
@@ -242,31 +262,37 @@ def gen_id(
|
|
242
262
|
|
243
263
|
|
244
264
|
def default_gen_id() -> str:
|
245
|
-
"""Return running ID
|
246
|
-
|
265
|
+
"""Return running ID for making default ID for the Result model.
|
266
|
+
|
267
|
+
This function is used when a run_id field is initialized for the first time.
|
247
268
|
|
248
|
-
:
|
269
|
+
Returns:
|
270
|
+
str: Generated default running ID.
|
249
271
|
"""
|
250
272
|
return gen_id("MOCK", unique=True)
|
251
273
|
|
252
274
|
|
253
275
|
def make_exec(path: Union[Path, str]) -> None:
|
254
|
-
"""Change mode
|
276
|
+
"""Change file mode to be executable.
|
255
277
|
|
256
|
-
:
|
257
|
-
|
278
|
+
Args:
|
279
|
+
path: A file path to make executable.
|
258
280
|
"""
|
259
281
|
f: Path = Path(path) if isinstance(path, str) else path
|
260
282
|
f.chmod(f.stat().st_mode | stat.S_IEXEC)
|
261
283
|
|
262
284
|
|
263
285
|
def filter_func(value: T) -> T:
|
264
|
-
"""Filter out
|
265
|
-
|
266
|
-
|
286
|
+
"""Filter out custom functions from mapping context by replacing with function names.
|
287
|
+
|
288
|
+
This function replaces custom functions with their function names in data
|
289
|
+
structures. Built-in functions remain unchanged.
|
267
290
|
|
268
|
-
:
|
269
|
-
|
291
|
+
Args:
|
292
|
+
value: A value or data structure to filter function values from.
|
293
|
+
|
294
|
+
Returns:
|
295
|
+
T: The filtered value with functions replaced by their names.
|
270
296
|
"""
|
271
297
|
if isinstance(value, dict):
|
272
298
|
return {k: filter_func(value[k]) for k in value}
|
@@ -287,11 +313,13 @@ def filter_func(value: T) -> T:
|
|
287
313
|
|
288
314
|
|
289
315
|
def cross_product(matrix: Matrix) -> Iterator[DictData]:
|
290
|
-
"""
|
316
|
+
"""Generate iterator of product values from matrix.
|
291
317
|
|
292
|
-
:
|
318
|
+
Args:
|
319
|
+
matrix: A matrix to generate cross products from.
|
293
320
|
|
294
|
-
:
|
321
|
+
Returns:
|
322
|
+
Iterator[DictData]: Iterator of dictionary combinations.
|
295
323
|
"""
|
296
324
|
yield from (
|
297
325
|
{_k: _v for e in mapped for _k, _v in e.items()}
|
@@ -302,16 +330,18 @@ def cross_product(matrix: Matrix) -> Iterator[DictData]:
|
|
302
330
|
|
303
331
|
|
304
332
|
def cut_id(run_id: str, *, num: int = 6) -> str:
|
305
|
-
"""
|
333
|
+
"""Cut running ID to specified length.
|
306
334
|
|
307
335
|
Example:
|
308
336
|
>>> cut_id(run_id='20240101081330000000T1354680202')
|
309
337
|
'202401010813680202'
|
310
338
|
|
311
|
-
:
|
312
|
-
|
339
|
+
Args:
|
340
|
+
run_id: A running ID to cut.
|
341
|
+
num: Number of characters to keep from the end.
|
313
342
|
|
314
|
-
:
|
343
|
+
Returns:
|
344
|
+
str: The cut running ID.
|
315
345
|
"""
|
316
346
|
if "T" in run_id:
|
317
347
|
dt, simple = run_id.split("T", maxsplit=1)
|
@@ -333,10 +363,14 @@ def dump_all(
|
|
333
363
|
value: Union[T, BaseModel],
|
334
364
|
by_alias: bool = False,
|
335
365
|
) -> Union[T, DictData]:
|
336
|
-
"""Dump all nested BaseModel
|
366
|
+
"""Dump all nested BaseModel objects to dictionary objects.
|
337
367
|
|
338
|
-
:
|
339
|
-
|
368
|
+
Args:
|
369
|
+
value: A value that may contain BaseModel objects.
|
370
|
+
by_alias: Whether to use field aliases when dumping.
|
371
|
+
|
372
|
+
Returns:
|
373
|
+
Union[T, DictData]: The value with BaseModel objects converted to dictionaries.
|
340
374
|
"""
|
341
375
|
if isinstance(value, dict):
|
342
376
|
return {k: dump_all(value[k], by_alias=by_alias) for k in value}
|
@@ -351,6 +385,14 @@ def dump_all(
|
|
351
385
|
|
352
386
|
|
353
387
|
def obj_name(obj: Optional[Union[str, object]] = None) -> Optional[str]:
|
388
|
+
"""Get object name or class name.
|
389
|
+
|
390
|
+
Args:
|
391
|
+
obj: An object or string to get the name from.
|
392
|
+
|
393
|
+
Returns:
|
394
|
+
Optional[str]: The object name, class name, or None if obj is None.
|
395
|
+
"""
|
354
396
|
if not obj:
|
355
397
|
return None
|
356
398
|
elif isinstance(obj, str):
|
ddeutil/workflow/workflow.py
CHANGED
@@ -61,7 +61,7 @@ from .result import (
|
|
61
61
|
validate_statuses,
|
62
62
|
)
|
63
63
|
from .reusables import has_template, param2template
|
64
|
-
from .traces import
|
64
|
+
from .traces import TraceManager, get_trace
|
65
65
|
from .utils import (
|
66
66
|
UTC,
|
67
67
|
gen_id,
|
@@ -407,7 +407,7 @@ class Workflow(BaseModel):
|
|
407
407
|
with the set `on` field.
|
408
408
|
|
409
409
|
Args:
|
410
|
-
dt: A datetime object that want to validate.
|
410
|
+
dt (datetime): A datetime object that want to validate.
|
411
411
|
|
412
412
|
Returns:
|
413
413
|
datetime: The validated release datetime.
|
@@ -416,6 +416,8 @@ class Workflow(BaseModel):
|
|
416
416
|
dt = dt.replace(tzinfo=UTC)
|
417
417
|
|
418
418
|
release: datetime = replace_sec(dt.astimezone(UTC))
|
419
|
+
|
420
|
+
# NOTE: Return itself if schedule event does not set.
|
419
421
|
if not self.on.schedule:
|
420
422
|
return release
|
421
423
|
|
@@ -454,7 +456,7 @@ class Workflow(BaseModel):
|
|
454
456
|
- Writing result audit
|
455
457
|
|
456
458
|
Args:
|
457
|
-
release
|
459
|
+
release (datetime): A release datetime.
|
458
460
|
params: A workflow parameter that pass to execute method.
|
459
461
|
release_type:
|
460
462
|
run_id: (str) A workflow running ID.
|
@@ -481,7 +483,7 @@ class Workflow(BaseModel):
|
|
481
483
|
parent_run_id: str = run_id
|
482
484
|
|
483
485
|
context: DictData = {"status": WAIT}
|
484
|
-
trace:
|
486
|
+
trace: TraceManager = get_trace(
|
485
487
|
run_id, parent_run_id=parent_run_id, extras=self.extras
|
486
488
|
)
|
487
489
|
release: datetime = self.validate_release(dt=release)
|
@@ -576,7 +578,7 @@ class Workflow(BaseModel):
|
|
576
578
|
Returns:
|
577
579
|
tuple[Status, DictData]: The pair of status and result context data.
|
578
580
|
"""
|
579
|
-
trace:
|
581
|
+
trace: TraceManager = get_trace(
|
580
582
|
run_id, parent_run_id=parent_run_id, extras=self.extras
|
581
583
|
)
|
582
584
|
if event and event.is_set():
|
@@ -693,7 +695,7 @@ class Workflow(BaseModel):
|
|
693
695
|
ts: float = time.monotonic()
|
694
696
|
parent_run_id: Optional[str] = run_id
|
695
697
|
run_id: str = gen_id(self.name, extras=self.extras)
|
696
|
-
trace:
|
698
|
+
trace: TraceManager = get_trace(
|
697
699
|
run_id, parent_run_id=parent_run_id, extras=self.extras
|
698
700
|
)
|
699
701
|
context: DictData = self.parameterize(params)
|
@@ -750,8 +752,12 @@ class Workflow(BaseModel):
|
|
750
752
|
|
751
753
|
with ThreadPoolExecutor(max_job_parallel, "wf") as executor:
|
752
754
|
futures: list[Future] = []
|
753
|
-
|
754
|
-
|
755
|
+
|
756
|
+
# NOTE: Start with smaller sleep time
|
757
|
+
backoff_sleep: float = 0.01
|
758
|
+
|
759
|
+
# NOTE: Track consecutive wait states
|
760
|
+
consecutive_waits: int = 0
|
755
761
|
|
756
762
|
while not job_queue.empty() and (
|
757
763
|
not_timeout_flag := ((time.monotonic() - ts) < timeout)
|
@@ -799,6 +805,7 @@ class Workflow(BaseModel):
|
|
799
805
|
skip_count += 1
|
800
806
|
continue
|
801
807
|
|
808
|
+
# IMPORTANT: Start execution with parallel mode.
|
802
809
|
if max_job_parallel > 1:
|
803
810
|
futures.append(
|
804
811
|
executor.submit(
|
@@ -831,7 +838,9 @@ class Workflow(BaseModel):
|
|
831
838
|
st, _ = future.result()
|
832
839
|
sequence_statuses.append(st)
|
833
840
|
job_queue.put(job_id)
|
834
|
-
|
841
|
+
# NOTE: The release future can not track a cancelled status
|
842
|
+
# because it only has one future.
|
843
|
+
elif future.cancelled(): # pragma: no cov
|
835
844
|
sequence_statuses.append(CANCEL)
|
836
845
|
job_queue.put(job_id)
|
837
846
|
elif future.running() or "state=pending" in str(future):
|
@@ -852,7 +861,7 @@ class Workflow(BaseModel):
|
|
852
861
|
for total, future in enumerate(as_completed(futures), start=0):
|
853
862
|
try:
|
854
863
|
statuses[total], _ = future.result()
|
855
|
-
except WorkflowError as e:
|
864
|
+
except (WorkflowError, Exception) as e:
|
856
865
|
statuses[total] = get_status_from_error(e)
|
857
866
|
|
858
867
|
# NOTE: Update skipped status from the job trigger.
|
@@ -931,7 +940,7 @@ class Workflow(BaseModel):
|
|
931
940
|
ts: float = time.monotonic()
|
932
941
|
parent_run_id: str = run_id
|
933
942
|
run_id: str = gen_id(self.name, extras=self.extras)
|
934
|
-
trace:
|
943
|
+
trace: TraceManager = get_trace(
|
935
944
|
run_id, parent_run_id=parent_run_id, extras=self.extras
|
936
945
|
)
|
937
946
|
if context["status"] == SUCCESS:
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: ddeutil-workflow
|
3
|
-
Version: 0.0.
|
3
|
+
Version: 0.0.79
|
4
4
|
Summary: Lightweight workflow orchestration with YAML template
|
5
5
|
Author-email: ddeutils <korawich.anu@gmail.com>
|
6
6
|
License: MIT
|
@@ -24,7 +24,7 @@ Description-Content-Type: text/markdown
|
|
24
24
|
License-File: LICENSE
|
25
25
|
Requires-Dist: ddeutil[checksum]>=0.4.8
|
26
26
|
Requires-Dist: ddeutil-io[toml,yaml]>=0.2.14
|
27
|
-
Requires-Dist: pydantic<3.0.0,==2.11.
|
27
|
+
Requires-Dist: pydantic<3.0.0,==2.11.7
|
28
28
|
Requires-Dist: pydantic-extra-types<3.0.0,>=2.10.4
|
29
29
|
Requires-Dist: python-dotenv>=1.1.0
|
30
30
|
Requires-Dist: typer>=0.16.0
|
@@ -35,9 +35,17 @@ Requires-Dist: httpx; extra == "all"
|
|
35
35
|
Requires-Dist: ujson; extra == "all"
|
36
36
|
Requires-Dist: aiofiles; extra == "all"
|
37
37
|
Requires-Dist: aiohttp; extra == "all"
|
38
|
-
Requires-Dist: requests==2.32.
|
38
|
+
Requires-Dist: requests==2.32.4; extra == "all"
|
39
39
|
Provides-Extra: docker
|
40
40
|
Requires-Dist: docker==7.1.0; extra == "docker"
|
41
|
+
Provides-Extra: azure
|
42
|
+
Requires-Dist: azure-batch>=13.0.0; extra == "azure"
|
43
|
+
Requires-Dist: azure-storage-blob>=12.0.0; extra == "azure"
|
44
|
+
Provides-Extra: aws
|
45
|
+
Requires-Dist: boto3>=1.39.0; extra == "aws"
|
46
|
+
Provides-Extra: gcp
|
47
|
+
Requires-Dist: google-cloud-batch>=0.17.0; extra == "gcp"
|
48
|
+
Requires-Dist: google-cloud-storage>=2.10.0; extra == "gcp"
|
41
49
|
Dynamic: license-file
|
42
50
|
|
43
51
|
# Workflow Orchestration
|
@@ -140,6 +148,19 @@ If you want to install this package with application add-ons, you should add
|
|
140
148
|
| Python | `ddeutil-workflow` | ✅ |
|
141
149
|
| FastAPI Server | `ddeutil-workflow[all]` | ✅ |
|
142
150
|
|
151
|
+
Check the version of the current workflow package:
|
152
|
+
|
153
|
+
```shell
|
154
|
+
$ pip install ddeutil-workflow
|
155
|
+
$ workflow-cli version
|
156
|
+
```
|
157
|
+
|
158
|
+
Initial workflow project:
|
159
|
+
|
160
|
+
```shell
|
161
|
+
$ workflow-cli init
|
162
|
+
```
|
163
|
+
|
143
164
|
## 📖 Documentation
|
144
165
|
|
145
166
|
For comprehensive API documentation, examples, and best practices:
|
@@ -147,6 +168,19 @@ For comprehensive API documentation, examples, and best practices:
|
|
147
168
|
- **[Full Documentation](https://ddeutils.github.io/ddeutil-workflow/)** - Complete user guide and API reference
|
148
169
|
- **[Getting Started](https://ddeutils.github.io/ddeutil-workflow/getting-started/)** - Quick start guide
|
149
170
|
- **[API Reference](https://ddeutils.github.io/ddeutil-workflow/api/workflow/)** - Detailed API documentation
|
171
|
+
- **[Optimized Tracing](docs/optimized-tracing.md)** - High-performance logging system (2-5x faster)
|
172
|
+
|
173
|
+
## ⚡ Performance Improvements
|
174
|
+
|
175
|
+
The workflow system now includes an optimized tracing system that provides significant performance improvements:
|
176
|
+
|
177
|
+
- **🚀 2-5x faster logging** with buffered I/O operations
|
178
|
+
- **💾 60-80% reduction** in disk I/O operations
|
179
|
+
- **🛡️ Built-in thread safety** with minimal overhead
|
180
|
+
- **🔄 Backward compatible** - existing code automatically benefits
|
181
|
+
- **📊 Lower memory footprint** for high-volume logging
|
182
|
+
|
183
|
+
See [Optimized Tracing Documentation](docs/optimized-tracing.md) for details and performance benchmarks.
|
150
184
|
|
151
185
|
## 🎯 Usage
|
152
186
|
|
@@ -282,22 +316,21 @@ it will use default value and do not raise any error to you.
|
|
282
316
|
> The config value that you will set on the environment should combine with
|
283
317
|
> prefix, component, and name which is `WORKFLOW_{component}_{name}` (Upper case).
|
284
318
|
|
285
|
-
| Name
|
286
|
-
|
287
|
-
| **REGISTRY_CALLER**
|
288
|
-
| **REGISTRY_FILTER**
|
289
|
-
| **CONF_PATH**
|
290
|
-
| **STAGE_DEFAULT_ID**
|
291
|
-
| **GENERATE_ID_SIMPLE_MODE**
|
292
|
-
| **DEBUG_MODE**
|
293
|
-
| **TIMEZONE**
|
294
|
-
| **FORMAT**
|
295
|
-
| **FORMAT_FILE**
|
296
|
-
| **DATETIME_FORMAT**
|
297
|
-
| **
|
298
|
-
| **
|
299
|
-
| **
|
300
|
-
| **AUDIT_ENABLE_WRITE** | LOG | `true` | A flag that enable writing audit log after end execution in the workflow release step. |
|
319
|
+
| Name | Component | Default | Description |
|
320
|
+
|:-----------------------------|:---------:|:--------------------------------------------------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------|
|
321
|
+
| **REGISTRY_CALLER** | CORE | `.` | List of importable string for the call stage. |
|
322
|
+
| **REGISTRY_FILTER** | CORE | `ddeutil.workflow.templates` | List of importable string for the filter template. |
|
323
|
+
| **CONF_PATH** | CORE | `./conf` | The config path that keep all template `.yaml` files. |
|
324
|
+
| **STAGE_DEFAULT_ID** | CORE | `false` | A flag that enable default stage ID that use for catch an execution output. |
|
325
|
+
| **GENERATE_ID_SIMPLE_MODE** | CORE | `true` | A flog that enable generating ID with `md5` algorithm. |
|
326
|
+
| **DEBUG_MODE** | LOG | `true` | A flag that enable logging with debug level mode. |
|
327
|
+
| **TIMEZONE** | LOG | `Asia/Bangkok` | A Timezone string value that will pass to `ZoneInfo` object. |
|
328
|
+
| **FORMAT** | LOG | `%(asctime)s.%(msecs)03d (%(name)-10s, %(process)-5d,%(thread)-5d) [%(levelname)-7s] %(message)-120s (%(filename)s:%(lineno)s)` | A trace message console format. |
|
329
|
+
| **FORMAT_FILE** | LOG | `{datetime} ({process:5d}, {thread:5d}) {message:120s} ({filename}:{lineno})` | A trace message format that use to write to target pointer. |
|
330
|
+
| **DATETIME_FORMAT** | LOG | `%Y-%m-%d %H:%M:%S` | A datetime format of the trace log. |
|
331
|
+
| **TRACE_HANDLERS** | LOG | `[{"type": "console"}]` | A pointer URL of trace log that use to emit log message. Now uses optimized handler by default. |
|
332
|
+
| **AUDIT_URL** | LOG | `file:./audits` | A pointer URL of audit log that use to write audit metrix. |
|
333
|
+
| **AUDIT_ENABLE_WRITE** | LOG | `true` | A flag that enable writing audit log after end execution in the workflow release step. |
|
301
334
|
|
302
335
|
## :rocket: Deployment
|
303
336
|
|
@@ -0,0 +1,36 @@
|
|
1
|
+
ddeutil/workflow/__about__.py,sha256=yvfj2RwP4ItaAPfKbYC9qJNcrKY9lxnnSRd6wZ4-CgQ,28
|
2
|
+
ddeutil/workflow/__cron.py,sha256=avOagaHl9xXOmizeRWm13cOrty9Tw0vRjFq-xoEgpAY,29167
|
3
|
+
ddeutil/workflow/__init__.py,sha256=1xH6m7jXxFly_5FbWCoe8rqhdeSdnnrBMPzoiVo_Exo,3247
|
4
|
+
ddeutil/workflow/__main__.py,sha256=Qd-f8z2Q2vpiEP2x6PBFsJrpACWDVxFKQk820MhFmHo,59
|
5
|
+
ddeutil/workflow/__types.py,sha256=tA2vsr6mzTSzbWB1sb62c5GgxODlfVRz6FvgLNJtQao,4788
|
6
|
+
ddeutil/workflow/audits.py,sha256=7wfRXJmrG3T4YUSArABPfiWq6BzSBf8qoo24JAt433A,28280
|
7
|
+
ddeutil/workflow/cli.py,sha256=aNFOZ3Re_QJBBP6vkT9Lsjrg8wLxrw_LKrl-1SIvSOg,8152
|
8
|
+
ddeutil/workflow/conf.py,sha256=vdTvR1OVk2TZBK5ZwwUpfxgg8GU4ldV8ukLzG1-tGDQ,16603
|
9
|
+
ddeutil/workflow/errors.py,sha256=UpUIqoyqkvzqjuxtUQ9535l1HeAsyh-plEG0PgDVR2w,5541
|
10
|
+
ddeutil/workflow/event.py,sha256=qm7QHw-Pozm6oIUzAIxpDkPzzVZVtHgJIUlIle0vEfQ,13943
|
11
|
+
ddeutil/workflow/job.py,sha256=UvzU66CebZkwIBg1KZch_aA3bZL0jpVrhRUQ2JIyiN4,46615
|
12
|
+
ddeutil/workflow/params.py,sha256=y9f6DEIyae1j4awbj3Kbeq75-U2UPFlKv9K57Hdo_Go,17188
|
13
|
+
ddeutil/workflow/result.py,sha256=BOk3DZMtmdE7xzQYeEYTGFlIkzJQ4Ed3fYzf0zF8Jo8,8963
|
14
|
+
ddeutil/workflow/reusables.py,sha256=g_Cac3yHy0H5ffl4Bb8_eGl284ELxOuX4LI8GYPMZgw,24983
|
15
|
+
ddeutil/workflow/stages.py,sha256=QufIa2b7A_ngOndVoGzyxKm_o5ZrauNeqxAC4vBkKFM,122678
|
16
|
+
ddeutil/workflow/traces.py,sha256=0crly_08a7dfi8-w8QPmYizxh6T7VR8yDnEoxEEiwM0,72838
|
17
|
+
ddeutil/workflow/utils.py,sha256=-E-Z5hN_UTFuWDk-NpfKhNj0QtLfJSvZNDI5NzJsd5E,12122
|
18
|
+
ddeutil/workflow/workflow.py,sha256=WTQAoSUNOmGpvZYgl28ziTY3kxtqQQw4jbTXPJOIBY4,42790
|
19
|
+
ddeutil/workflow/api/__init__.py,sha256=5DzYL3ngceoRshh5HYCSVWChqNJSiP01E1bEd8XxPi0,4799
|
20
|
+
ddeutil/workflow/api/log_conf.py,sha256=WfS3udDLSyrP-C80lWOvxxmhd_XWKvQPkwDqKblcH3E,1834
|
21
|
+
ddeutil/workflow/api/routes/__init__.py,sha256=JRaJZB0D6mgR17MbZo8yLtdYDtD62AA8MdKlFqhG84M,420
|
22
|
+
ddeutil/workflow/api/routes/job.py,sha256=8eu2OAOS3fENQ54OO723lFpzgHMyz1D-b_xZj6OnmcA,2550
|
23
|
+
ddeutil/workflow/api/routes/logs.py,sha256=RiZ62eQVMWArPHE3lpan955U4DdLLkethlvSMlwF7Mg,5312
|
24
|
+
ddeutil/workflow/api/routes/workflows.py,sha256=1Mqx4Hft4uJglgJI-Wcw-JzkhomFYZrtP0DnQDBkAFQ,4410
|
25
|
+
ddeutil/workflow/plugins/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
26
|
+
ddeutil/workflow/plugins/providers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
27
|
+
ddeutil/workflow/plugins/providers/aws.py,sha256=61uIFBEWt-_D5Sui24qUPier1Hiqlw_RP_eY-rXBCKc,31551
|
28
|
+
ddeutil/workflow/plugins/providers/az.py,sha256=o3dh011lEtmr7-d7FPZJPgXdT0ytFzKfc5xnVxSyXGU,34867
|
29
|
+
ddeutil/workflow/plugins/providers/container.py,sha256=DSN0RWxMjTJN5ANheeMauDaPa3X6Z2E1eGUcctYkENw,22134
|
30
|
+
ddeutil/workflow/plugins/providers/gcs.py,sha256=KgAOdMBvdbMLTH_z_FwVriBFtZfKEYx8_34jzUOVjTY,27460
|
31
|
+
ddeutil_workflow-0.0.79.dist-info/licenses/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
|
32
|
+
ddeutil_workflow-0.0.79.dist-info/METADATA,sha256=V6nV4VeqzwryOpJVDmd4sK5U9rqgn8H4qs8WcTf8ugw,16755
|
33
|
+
ddeutil_workflow-0.0.79.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
34
|
+
ddeutil_workflow-0.0.79.dist-info/entry_points.txt,sha256=qDTpPSauL0ciO6T4iSVt8bJeYrVEkkoEEw_RlGx6Kgk,63
|
35
|
+
ddeutil_workflow-0.0.79.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
|
36
|
+
ddeutil_workflow-0.0.79.dist-info/RECORD,,
|
@@ -1,30 +0,0 @@
|
|
1
|
-
ddeutil/workflow/__about__.py,sha256=9g_DJubkaHxt4W_0-r1PmUT5x3bCo_72c-f60EDtlq8,28
|
2
|
-
ddeutil/workflow/__cron.py,sha256=avOagaHl9xXOmizeRWm13cOrty9Tw0vRjFq-xoEgpAY,29167
|
3
|
-
ddeutil/workflow/__init__.py,sha256=_8sP-CTPOfwsFFhmdwQ2Gp7yY7qJemP7TYsIWgd5jc0,3300
|
4
|
-
ddeutil/workflow/__main__.py,sha256=Qd-f8z2Q2vpiEP2x6PBFsJrpACWDVxFKQk820MhFmHo,59
|
5
|
-
ddeutil/workflow/__types.py,sha256=tA2vsr6mzTSzbWB1sb62c5GgxODlfVRz6FvgLNJtQao,4788
|
6
|
-
ddeutil/workflow/audits.py,sha256=wANG0jEQ7slUSgVZG4JbjlR5PtmF8mHpM9RH-zpYM_g,12679
|
7
|
-
ddeutil/workflow/cli.py,sha256=WnUkxqs2hCc5JVuTvWuEGKp8_EcS_wmhVvSXDhj0eEM,7544
|
8
|
-
ddeutil/workflow/conf.py,sha256=7yMSBVh2W9KcDOZxQjeabJZizB_3ydCLICo4JI0syWU,16892
|
9
|
-
ddeutil/workflow/errors.py,sha256=UpUIqoyqkvzqjuxtUQ9535l1HeAsyh-plEG0PgDVR2w,5541
|
10
|
-
ddeutil/workflow/event.py,sha256=6d5_UvnPI8xRLcX_5wptmvWoXUIGs_JZbjq7khz5oYE,13355
|
11
|
-
ddeutil/workflow/job.py,sha256=4_IxtoIwl1qVuSaNngelYNkqQl9ZOa-PHdKCnXvXu0M,43943
|
12
|
-
ddeutil/workflow/params.py,sha256=Cyz142OcvENIZrM7Efc2xuGPmmFBhROifP5ojoaCezg,13658
|
13
|
-
ddeutil/workflow/result.py,sha256=Xi07E3WuMHS1jLcJg7p4DPuaMFGp0yEDaWCJRotOH6g,8921
|
14
|
-
ddeutil/workflow/reusables.py,sha256=pbCHsEl2V3jGWDRcGyxDvGN5rP5kaRxZNgv9x6-pINQ,23338
|
15
|
-
ddeutil/workflow/stages.py,sha256=ocsk64duS4BHEOLT3Agkx-fbY6iYmygxCaQhwp6YyyM,122482
|
16
|
-
ddeutil/workflow/traces.py,sha256=h7oDlb4Q8LJUp0pste2dWJYOqEaN64KsLTNMfeRUqx8,28475
|
17
|
-
ddeutil/workflow/utils.py,sha256=vSGdpaFgQ5vUPxWvVfbNC2__tu5q16B9mhx1BRbEuJo,10968
|
18
|
-
ddeutil/workflow/workflow.py,sha256=0KaRCTAipPvvmi9SOC7T0V0CAusgebCM6qju0ethGF0,42409
|
19
|
-
ddeutil/workflow/api/__init__.py,sha256=5DzYL3ngceoRshh5HYCSVWChqNJSiP01E1bEd8XxPi0,4799
|
20
|
-
ddeutil/workflow/api/log_conf.py,sha256=WfS3udDLSyrP-C80lWOvxxmhd_XWKvQPkwDqKblcH3E,1834
|
21
|
-
ddeutil/workflow/api/routes/__init__.py,sha256=JRaJZB0D6mgR17MbZo8yLtdYDtD62AA8MdKlFqhG84M,420
|
22
|
-
ddeutil/workflow/api/routes/job.py,sha256=-lbZ_hS9pEdSy6zeke5qrXEgdNxtQ2w9in7cHuM2Jzs,2536
|
23
|
-
ddeutil/workflow/api/routes/logs.py,sha256=RiZ62eQVMWArPHE3lpan955U4DdLLkethlvSMlwF7Mg,5312
|
24
|
-
ddeutil/workflow/api/routes/workflows.py,sha256=1Mqx4Hft4uJglgJI-Wcw-JzkhomFYZrtP0DnQDBkAFQ,4410
|
25
|
-
ddeutil_workflow-0.0.78.dist-info/licenses/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
|
26
|
-
ddeutil_workflow-0.0.78.dist-info/METADATA,sha256=2phJ3JRH9o0Sbr22iOo1YeQBr8nyYx6areM2rwNp_Eg,15681
|
27
|
-
ddeutil_workflow-0.0.78.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
28
|
-
ddeutil_workflow-0.0.78.dist-info/entry_points.txt,sha256=qDTpPSauL0ciO6T4iSVt8bJeYrVEkkoEEw_RlGx6Kgk,63
|
29
|
-
ddeutil_workflow-0.0.78.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
|
30
|
-
ddeutil_workflow-0.0.78.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|