ddeutil-workflow 0.0.41__py3-none-any.whl → 0.0.42__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ddeutil/workflow/__about__.py +1 -1
- ddeutil/workflow/api/api.py +7 -7
- ddeutil/workflow/api/routes/schedules.py +5 -5
- ddeutil/workflow/api/routes/workflows.py +2 -2
- ddeutil/workflow/conf.py +39 -28
- ddeutil/workflow/cron.py +12 -13
- ddeutil/workflow/job.py +18 -10
- ddeutil/workflow/logs.py +33 -6
- ddeutil/workflow/reusables.py +16 -13
- ddeutil/workflow/scheduler.py +26 -28
- ddeutil/workflow/stages.py +257 -62
- ddeutil/workflow/utils.py +0 -1
- ddeutil/workflow/workflow.py +111 -74
- {ddeutil_workflow-0.0.41.dist-info → ddeutil_workflow-0.0.42.dist-info}/METADATA +6 -9
- ddeutil_workflow-0.0.42.dist-info/RECORD +30 -0
- ddeutil/workflow/context.py +0 -61
- ddeutil_workflow-0.0.41.dist-info/RECORD +0 -31
- {ddeutil_workflow-0.0.41.dist-info → ddeutil_workflow-0.0.42.dist-info}/WHEEL +0 -0
- {ddeutil_workflow-0.0.41.dist-info → ddeutil_workflow-0.0.42.dist-info}/licenses/LICENSE +0 -0
- {ddeutil_workflow-0.0.41.dist-info → ddeutil_workflow-0.0.42.dist-info}/top_level.txt +0 -0
ddeutil/workflow/workflow.py
CHANGED
@@ -3,6 +3,7 @@
|
|
3
3
|
# Licensed under the MIT License. See LICENSE in the project root for
|
4
4
|
# license information.
|
5
5
|
# ------------------------------------------------------------------------------
|
6
|
+
# [x] Use dynamic config`
|
6
7
|
"""A Workflow module that is the core model of this package."""
|
7
8
|
from __future__ import annotations
|
8
9
|
|
@@ -24,14 +25,14 @@ from textwrap import dedent
|
|
24
25
|
from threading import Event
|
25
26
|
from typing import Optional
|
26
27
|
|
27
|
-
from pydantic import BaseModel, ConfigDict, Field
|
28
|
+
from pydantic import BaseModel, ConfigDict, Field, ValidationInfo
|
28
29
|
from pydantic.dataclasses import dataclass
|
29
30
|
from pydantic.functional_validators import field_validator, model_validator
|
30
31
|
from typing_extensions import Self
|
31
32
|
|
32
33
|
from .__cron import CronJob, CronRunner
|
33
34
|
from .__types import DictData, TupleStr
|
34
|
-
from .conf import Loader, SimLoad,
|
35
|
+
from .conf import Loader, SimLoad, dynamic, get_logger
|
35
36
|
from .cron import On
|
36
37
|
from .exceptions import JobException, WorkflowException
|
37
38
|
from .job import Job, TriggerState
|
@@ -92,11 +93,15 @@ class Release:
|
|
92
93
|
return f"{self.date:%Y-%m-%d %H:%M:%S}"
|
93
94
|
|
94
95
|
@classmethod
|
95
|
-
def from_dt(
|
96
|
+
def from_dt(
|
97
|
+
cls, dt: datetime | str, *, externals: Optional[DictData] = None
|
98
|
+
) -> Self:
|
96
99
|
"""Construct Release via datetime object only.
|
97
100
|
|
98
101
|
:param dt: (datetime | str) A datetime object or string that want to
|
99
102
|
construct to the Release object.
|
103
|
+
:param externals: An external parameters that want to pass to override
|
104
|
+
config.
|
100
105
|
|
101
106
|
:raise TypeError: If the type of the dt argument does not valid with
|
102
107
|
datetime or str object.
|
@@ -115,7 +120,9 @@ class Release:
|
|
115
120
|
date=dt,
|
116
121
|
offset=0,
|
117
122
|
end_date=dt + timedelta(days=1),
|
118
|
-
runner=CronJob("* * * * *").schedule(
|
123
|
+
runner=CronJob("* * * * *").schedule(
|
124
|
+
dt.replace(tzinfo=dynamic("tz", extras=externals))
|
125
|
+
),
|
119
126
|
)
|
120
127
|
|
121
128
|
def __eq__(self, other: Release | datetime) -> bool:
|
@@ -150,6 +157,10 @@ class ReleaseQueue:
|
|
150
157
|
queue: list[Release] = field(default_factory=list)
|
151
158
|
running: list[Release] = field(default_factory=list)
|
152
159
|
complete: list[Release] = field(default_factory=list)
|
160
|
+
extras: DictData = Field(
|
161
|
+
default_factory=dict,
|
162
|
+
description="An extra override config values.",
|
163
|
+
)
|
153
164
|
|
154
165
|
@classmethod
|
155
166
|
def from_list(
|
@@ -232,8 +243,8 @@ class ReleaseQueue:
|
|
232
243
|
|
233
244
|
# NOTE: Remove complete queue on workflow that keep more than the
|
234
245
|
# maximum config.
|
235
|
-
num_complete_delete: int = (
|
236
|
-
|
246
|
+
num_complete_delete: int = len(self.complete) - dynamic(
|
247
|
+
"max_queue_complete_hist", extras=self.extras
|
237
248
|
)
|
238
249
|
|
239
250
|
if num_complete_delete > 0:
|
@@ -252,6 +263,11 @@ class Workflow(BaseModel):
|
|
252
263
|
execute method on it.
|
253
264
|
"""
|
254
265
|
|
266
|
+
extras: DictData = Field(
|
267
|
+
default_factory=dict,
|
268
|
+
description="An extra override config values.",
|
269
|
+
)
|
270
|
+
|
255
271
|
name: str = Field(description="A workflow name.")
|
256
272
|
desc: Optional[str] = Field(
|
257
273
|
default=None,
|
@@ -271,30 +287,26 @@ class Workflow(BaseModel):
|
|
271
287
|
default_factory=dict,
|
272
288
|
description="A mapping of job ID and job model that already loaded.",
|
273
289
|
)
|
274
|
-
extras: DictData = Field(
|
275
|
-
default_factory=dict,
|
276
|
-
description="An extra override values.",
|
277
|
-
)
|
278
290
|
|
279
291
|
@classmethod
|
280
|
-
def
|
292
|
+
def from_conf(
|
281
293
|
cls,
|
282
294
|
name: str,
|
283
|
-
|
295
|
+
extras: DictData | None = None,
|
284
296
|
) -> Self:
|
285
297
|
"""Create Workflow instance from the Loader object that only receive
|
286
298
|
an input workflow name. The loader object will use this workflow name to
|
287
299
|
searching configuration data of this workflow model in conf path.
|
288
300
|
|
289
301
|
:param name: A workflow name that want to pass to Loader object.
|
290
|
-
:param
|
302
|
+
:param extras: An extra parameters that want to pass to Loader
|
291
303
|
object.
|
292
304
|
|
293
305
|
:raise ValueError: If the type does not match with current object.
|
294
306
|
|
295
307
|
:rtype: Self
|
296
308
|
"""
|
297
|
-
loader: Loader = Loader(name, externals=(
|
309
|
+
loader: Loader = Loader(name, externals=(extras or {}))
|
298
310
|
|
299
311
|
# NOTE: Validate the config type match with current connection model
|
300
312
|
if loader.type != cls.__name__:
|
@@ -302,12 +314,10 @@ class Workflow(BaseModel):
|
|
302
314
|
|
303
315
|
loader_data: DictData = copy.deepcopy(loader.data)
|
304
316
|
loader_data["name"] = name.replace(" ", "_")
|
305
|
-
if
|
306
|
-
loader_data["extras"] =
|
317
|
+
if extras: # pragma: no cov
|
318
|
+
loader_data["extras"] = extras
|
307
319
|
|
308
|
-
cls.__bypass_on__(
|
309
|
-
loader_data, path=loader.conf_path, externals=externals
|
310
|
-
)
|
320
|
+
cls.__bypass_on__(loader_data, path=loader.conf_path, extras=extras)
|
311
321
|
return cls.model_validate(obj=loader_data)
|
312
322
|
|
313
323
|
@classmethod
|
@@ -315,7 +325,7 @@ class Workflow(BaseModel):
|
|
315
325
|
cls,
|
316
326
|
name: str,
|
317
327
|
path: Path,
|
318
|
-
|
328
|
+
extras: DictData | None = None,
|
319
329
|
) -> Self:
|
320
330
|
"""Create Workflow instance from the specific path. The loader object
|
321
331
|
will use this workflow name and path to searching configuration data of
|
@@ -323,7 +333,7 @@ class Workflow(BaseModel):
|
|
323
333
|
|
324
334
|
:param name: (str) A workflow name that want to pass to Loader object.
|
325
335
|
:param path: (Path) A config path that want to search.
|
326
|
-
:param
|
336
|
+
:param extras: An extra parameters that want to pass to Loader
|
327
337
|
object.
|
328
338
|
|
329
339
|
:raise ValueError: If the type does not match with current object.
|
@@ -331,7 +341,7 @@ class Workflow(BaseModel):
|
|
331
341
|
:rtype: Self
|
332
342
|
"""
|
333
343
|
loader: SimLoad = SimLoad(
|
334
|
-
name, conf_path=path, externals=(
|
344
|
+
name, conf_path=path, externals=(extras or {})
|
335
345
|
)
|
336
346
|
# NOTE: Validate the config type match with current connection model
|
337
347
|
if loader.type != cls.__name__:
|
@@ -339,10 +349,10 @@ class Workflow(BaseModel):
|
|
339
349
|
|
340
350
|
loader_data: DictData = copy.deepcopy(loader.data)
|
341
351
|
loader_data["name"] = name.replace(" ", "_")
|
342
|
-
if
|
343
|
-
loader_data["extras"] =
|
352
|
+
if extras: # pragma: no cov
|
353
|
+
loader_data["extras"] = extras
|
344
354
|
|
345
|
-
cls.__bypass_on__(loader_data, path=path,
|
355
|
+
cls.__bypass_on__(loader_data, path=path, extras=extras)
|
346
356
|
return cls.model_validate(obj=loader_data)
|
347
357
|
|
348
358
|
@classmethod
|
@@ -350,14 +360,15 @@ class Workflow(BaseModel):
|
|
350
360
|
cls,
|
351
361
|
data: DictData,
|
352
362
|
path: Path,
|
353
|
-
|
363
|
+
extras: DictData | None = None,
|
354
364
|
) -> DictData:
|
355
365
|
"""Bypass the on data to loaded config data.
|
356
366
|
|
357
367
|
:param data: A data to construct to this Workflow model.
|
358
368
|
:param path: A config path.
|
359
|
-
:param
|
369
|
+
:param extras: An extra parameters that want to pass to SimLoad
|
360
370
|
object.
|
371
|
+
|
361
372
|
:rtype: DictData
|
362
373
|
"""
|
363
374
|
if on := data.pop("on", []):
|
@@ -370,7 +381,7 @@ class Workflow(BaseModel):
|
|
370
381
|
# field.
|
371
382
|
data["on"] = [
|
372
383
|
(
|
373
|
-
SimLoad(n, conf_path=path, externals=(
|
384
|
+
SimLoad(n, conf_path=path, externals=(extras or {})).data
|
374
385
|
if isinstance(n, str)
|
375
386
|
else n
|
376
387
|
)
|
@@ -403,7 +414,11 @@ class Workflow(BaseModel):
|
|
403
414
|
return dedent(value)
|
404
415
|
|
405
416
|
@field_validator("on", mode="after")
|
406
|
-
def __on_no_dup_and_reach_limit__(
|
417
|
+
def __on_no_dup_and_reach_limit__(
|
418
|
+
cls,
|
419
|
+
value: list[On],
|
420
|
+
info: ValidationInfo,
|
421
|
+
) -> list[On]:
|
407
422
|
"""Validate the on fields should not contain duplicate values and if it
|
408
423
|
contains the every minute value more than one value, it will remove to
|
409
424
|
only one value.
|
@@ -427,10 +442,12 @@ class Workflow(BaseModel):
|
|
427
442
|
# "only one value in the on field."
|
428
443
|
# )
|
429
444
|
|
430
|
-
|
445
|
+
extras: DictData = info.data.get("extras", {})
|
446
|
+
if len(set_ons) > (
|
447
|
+
conf := dynamic("max_on_per_workflow", extras=extras)
|
448
|
+
):
|
431
449
|
raise ValueError(
|
432
|
-
f"The number of the on should not more than "
|
433
|
-
f"{config.max_on_per_workflow} crontab."
|
450
|
+
f"The number of the on should not more than {conf} crontabs."
|
434
451
|
)
|
435
452
|
return value
|
436
453
|
|
@@ -604,9 +621,11 @@ class Workflow(BaseModel):
|
|
604
621
|
release_params: DictData = {
|
605
622
|
"release": {
|
606
623
|
"logical_date": release.date,
|
607
|
-
"execute_date": datetime.now(
|
624
|
+
"execute_date": datetime.now(
|
625
|
+
tz=dynamic("tz", extras=self.extras)
|
626
|
+
),
|
608
627
|
"run_id": result.run_id,
|
609
|
-
"timezone":
|
628
|
+
"timezone": dynamic("tz", extras=self.extras),
|
610
629
|
}
|
611
630
|
}
|
612
631
|
|
@@ -616,7 +635,7 @@ class Workflow(BaseModel):
|
|
616
635
|
# ... {"params": ..., "jobs": ...}
|
617
636
|
#
|
618
637
|
self.execute(
|
619
|
-
params=param2template(params, release_params),
|
638
|
+
params=param2template(params, release_params, extras=self.extras),
|
620
639
|
result=result,
|
621
640
|
parent_run_id=result.parent_run_id,
|
622
641
|
)
|
@@ -656,7 +675,6 @@ class Workflow(BaseModel):
|
|
656
675
|
return result.catch(
|
657
676
|
status=Status.SUCCESS,
|
658
677
|
context={
|
659
|
-
# NOTE: Update the real params that pass in this method.
|
660
678
|
"params": params,
|
661
679
|
"release": {
|
662
680
|
"type": release.type,
|
@@ -700,10 +718,11 @@ class Workflow(BaseModel):
|
|
700
718
|
for on in self.on:
|
701
719
|
|
702
720
|
runner: CronRunner = on.next(
|
703
|
-
get_dt_now(
|
721
|
+
get_dt_now(
|
722
|
+
tz=dynamic("tz", extras=self.extras), offset=offset
|
723
|
+
).replace(microsecond=0)
|
704
724
|
)
|
705
725
|
|
706
|
-
# NOTE: Skip this runner date if it more than the end date.
|
707
726
|
if runner.date > end_date:
|
708
727
|
continue
|
709
728
|
|
@@ -730,7 +749,6 @@ class Workflow(BaseModel):
|
|
730
749
|
if runner.date > end_date:
|
731
750
|
continue
|
732
751
|
|
733
|
-
# NOTE: Push the Release object to queue.
|
734
752
|
heappush(queue.queue, workflow_release)
|
735
753
|
|
736
754
|
return queue
|
@@ -745,6 +763,7 @@ class Workflow(BaseModel):
|
|
745
763
|
audit: Audit | None = None,
|
746
764
|
force_run: bool = False,
|
747
765
|
timeout: int = 1800,
|
766
|
+
max_poking_pool_worker: int = 4,
|
748
767
|
) -> Result:
|
749
768
|
"""Poke function with a start datetime value that will pass to its
|
750
769
|
`on` field on the threading executor pool for execute the `release`
|
@@ -765,6 +784,7 @@ class Workflow(BaseModel):
|
|
765
784
|
that release was pointed.
|
766
785
|
:param timeout: A second value for timeout while waiting all futures
|
767
786
|
run completely.
|
787
|
+
:param max_poking_pool_worker: The maximum poking pool worker.
|
768
788
|
|
769
789
|
:rtype: Result
|
770
790
|
:return: A list of all results that return from `self.release` method.
|
@@ -780,8 +800,6 @@ class Workflow(BaseModel):
|
|
780
800
|
"The period of poking should be int and grater or equal than 1."
|
781
801
|
)
|
782
802
|
|
783
|
-
# NOTE: If this workflow does not set the on schedule, it will return
|
784
|
-
# empty result.
|
785
803
|
if len(self.on) == 0:
|
786
804
|
result.trace.info(
|
787
805
|
f"[POKING]: {self.name!r} does not have any schedule to run."
|
@@ -789,15 +807,15 @@ class Workflow(BaseModel):
|
|
789
807
|
return result.catch(status=Status.SUCCESS, context={"outputs": []})
|
790
808
|
|
791
809
|
# NOTE: Create the current date that change microsecond to 0
|
792
|
-
current_date: datetime = datetime.now(
|
793
|
-
|
794
|
-
)
|
810
|
+
current_date: datetime = datetime.now(
|
811
|
+
tz=dynamic("tz", extras=self.extras)
|
812
|
+
).replace(microsecond=0)
|
795
813
|
|
796
814
|
# NOTE: Create start_date and offset variables.
|
797
815
|
if start_date and start_date <= current_date:
|
798
|
-
start_date = start_date.replace(
|
799
|
-
|
800
|
-
)
|
816
|
+
start_date = start_date.replace(
|
817
|
+
tzinfo=dynamic("tz", extras=self.extras)
|
818
|
+
).replace(microsecond=0)
|
801
819
|
offset: float = (current_date - start_date).total_seconds()
|
802
820
|
else:
|
803
821
|
# NOTE: Force change start date if it gathers than the current date,
|
@@ -837,7 +855,11 @@ class Workflow(BaseModel):
|
|
837
855
|
# NOTE: Start create the thread pool executor for running this poke
|
838
856
|
# process.
|
839
857
|
with ThreadPoolExecutor(
|
840
|
-
max_workers=
|
858
|
+
max_workers=dynamic(
|
859
|
+
"max_poking_pool_worker",
|
860
|
+
f=max_poking_pool_worker,
|
861
|
+
extras=self.extras,
|
862
|
+
),
|
841
863
|
thread_name_prefix="wf_poking_",
|
842
864
|
) as executor:
|
843
865
|
|
@@ -848,14 +870,22 @@ class Workflow(BaseModel):
|
|
848
870
|
# NOTE: Pop the latest Release object from the release queue.
|
849
871
|
release: Release = heappop(q.queue)
|
850
872
|
|
851
|
-
if reach_next_minute(
|
873
|
+
if reach_next_minute(
|
874
|
+
release.date,
|
875
|
+
tz=dynamic("tz", extras=self.extras),
|
876
|
+
offset=offset,
|
877
|
+
):
|
852
878
|
result.trace.debug(
|
853
879
|
f"[POKING]: The latest release, "
|
854
880
|
f"{release.date:%Y-%m-%d %H:%M:%S}, is not able to run "
|
855
881
|
f"on this minute"
|
856
882
|
)
|
857
883
|
heappush(q.queue, release)
|
858
|
-
wait_to_next_minute(
|
884
|
+
wait_to_next_minute(
|
885
|
+
get_dt_now(
|
886
|
+
tz=dynamic("tz", extras=self.extras), offset=offset
|
887
|
+
)
|
888
|
+
)
|
859
889
|
|
860
890
|
# WARNING: I already call queue poking again because issue
|
861
891
|
# about the every minute crontab.
|
@@ -935,12 +965,6 @@ class Workflow(BaseModel):
|
|
935
965
|
"job execution."
|
936
966
|
)
|
937
967
|
|
938
|
-
# IMPORTANT:
|
939
|
-
# This execution change all job running IDs to the current workflow
|
940
|
-
# running ID, but it still trac log to the same parent running ID
|
941
|
-
# (with passing `run_id` and `parent_run_id` to the job execution
|
942
|
-
# arguments).
|
943
|
-
#
|
944
968
|
try:
|
945
969
|
job: Job = self.jobs[job_id]
|
946
970
|
if job.is_skipped(params=params):
|
@@ -975,8 +999,10 @@ class Workflow(BaseModel):
|
|
975
999
|
*,
|
976
1000
|
run_id: str | None = None,
|
977
1001
|
parent_run_id: str | None = None,
|
978
|
-
timeout: int =
|
1002
|
+
timeout: int = 600,
|
979
1003
|
result: Result | None = None,
|
1004
|
+
max_job_parallel: int = 2,
|
1005
|
+
event: Event | None = None,
|
980
1006
|
) -> Result:
|
981
1007
|
"""Execute workflow with passing a dynamic parameters to all jobs that
|
982
1008
|
included in this workflow model with ``jobs`` field.
|
@@ -1001,6 +1027,9 @@ class Workflow(BaseModel):
|
|
1001
1027
|
limit time. (default: 0)
|
1002
1028
|
:param result: (Result) A result object for keeping context and status
|
1003
1029
|
data.
|
1030
|
+
:param max_job_parallel: (int) The maximum threads of job execution.
|
1031
|
+
:param event: (Event) An event manager that pass to the
|
1032
|
+
PoolThreadExecutor.
|
1004
1033
|
|
1005
1034
|
:rtype: Result
|
1006
1035
|
"""
|
@@ -1038,13 +1067,19 @@ class Workflow(BaseModel):
|
|
1038
1067
|
context: DictData = self.parameterize(params)
|
1039
1068
|
status: Status = Status.SUCCESS
|
1040
1069
|
try:
|
1041
|
-
if
|
1070
|
+
if (
|
1071
|
+
dynamic(
|
1072
|
+
"max_job_parallel", f=max_job_parallel, extras=self.extras
|
1073
|
+
)
|
1074
|
+
== 1
|
1075
|
+
):
|
1042
1076
|
self.__exec_non_threading(
|
1043
1077
|
result=result,
|
1044
1078
|
context=context,
|
1045
1079
|
ts=ts,
|
1046
1080
|
job_queue=jq,
|
1047
1081
|
timeout=timeout,
|
1082
|
+
event=event,
|
1048
1083
|
)
|
1049
1084
|
else:
|
1050
1085
|
self.__exec_threading(
|
@@ -1053,6 +1088,7 @@ class Workflow(BaseModel):
|
|
1053
1088
|
ts=ts,
|
1054
1089
|
job_queue=jq,
|
1055
1090
|
timeout=timeout,
|
1091
|
+
event=event,
|
1056
1092
|
)
|
1057
1093
|
except WorkflowException as err:
|
1058
1094
|
status = Status.FAILED
|
@@ -1067,8 +1103,9 @@ class Workflow(BaseModel):
|
|
1067
1103
|
ts: float,
|
1068
1104
|
job_queue: Queue,
|
1069
1105
|
*,
|
1070
|
-
timeout: int =
|
1106
|
+
timeout: int = 600,
|
1071
1107
|
thread_timeout: int = 1800,
|
1108
|
+
event: Event | None = None,
|
1072
1109
|
) -> DictData:
|
1073
1110
|
"""Workflow execution by threading strategy that use multithreading.
|
1074
1111
|
|
@@ -1082,18 +1119,19 @@ class Workflow(BaseModel):
|
|
1082
1119
|
:param job_queue: (Queue) A job queue object.
|
1083
1120
|
:param timeout: (int) A second value unit that bounding running time.
|
1084
1121
|
:param thread_timeout: A timeout to waiting all futures complete.
|
1122
|
+
:param event: (Event) An event manager that pass to the
|
1123
|
+
PoolThreadExecutor.
|
1085
1124
|
|
1086
1125
|
:rtype: DictData
|
1087
1126
|
"""
|
1088
1127
|
not_timeout_flag: bool = True
|
1089
|
-
timeout: int =
|
1090
|
-
|
1128
|
+
timeout: int = dynamic(
|
1129
|
+
"max_job_exec_timeout", f=timeout, extras=self.extras
|
1130
|
+
)
|
1131
|
+
event: Event = event or Event()
|
1091
1132
|
result.trace.debug(f"[WORKFLOW]: Run {self.name!r} with threading.")
|
1092
|
-
|
1093
|
-
# IMPORTANT: The job execution can run parallel and waiting by
|
1094
|
-
# needed.
|
1095
1133
|
with ThreadPoolExecutor(
|
1096
|
-
max_workers=
|
1134
|
+
max_workers=dynamic("max_job_parallel", extras=self.extras),
|
1097
1135
|
thread_name_prefix="wf_exec_threading_",
|
1098
1136
|
) as executor:
|
1099
1137
|
futures: list[Future] = []
|
@@ -1153,7 +1191,6 @@ class Workflow(BaseModel):
|
|
1153
1191
|
result.trace.error(f"[WORKFLOW]: {err}")
|
1154
1192
|
raise WorkflowException(str(err))
|
1155
1193
|
|
1156
|
-
# NOTE: This getting result does not do anything.
|
1157
1194
|
future.result()
|
1158
1195
|
|
1159
1196
|
return context
|
@@ -1174,7 +1211,8 @@ class Workflow(BaseModel):
|
|
1174
1211
|
ts: float,
|
1175
1212
|
job_queue: Queue,
|
1176
1213
|
*,
|
1177
|
-
timeout: int =
|
1214
|
+
timeout: int = 600,
|
1215
|
+
event: Event | None = None,
|
1178
1216
|
) -> DictData:
|
1179
1217
|
"""Workflow execution with non-threading strategy that use sequential
|
1180
1218
|
job running and waiting previous job was run successful.
|
@@ -1187,12 +1225,16 @@ class Workflow(BaseModel):
|
|
1187
1225
|
:param ts: (float) A start timestamp that use for checking execute time
|
1188
1226
|
should time out.
|
1189
1227
|
:param timeout: (int) A second value unit that bounding running time.
|
1228
|
+
:param event: (Event) An event manager that pass to the
|
1229
|
+
PoolThreadExecutor.
|
1190
1230
|
|
1191
1231
|
:rtype: DictData
|
1192
1232
|
"""
|
1193
1233
|
not_timeout_flag: bool = True
|
1194
|
-
timeout: int =
|
1195
|
-
|
1234
|
+
timeout: int = dynamic(
|
1235
|
+
"max_job_exec_timeout", f=timeout, extras=self.extras
|
1236
|
+
)
|
1237
|
+
event: Event = event or Event()
|
1196
1238
|
future: Future | None = None
|
1197
1239
|
result.trace.debug(f"[WORKFLOW]: Run {self.name!r} with non-threading.")
|
1198
1240
|
|
@@ -1207,7 +1249,6 @@ class Workflow(BaseModel):
|
|
1207
1249
|
job_id: str = job_queue.get()
|
1208
1250
|
job: Job = self.jobs[job_id]
|
1209
1251
|
|
1210
|
-
# NOTE: Waiting dependency job run successful before release.
|
1211
1252
|
if (check := job.check_needs(context["jobs"])).is_waiting():
|
1212
1253
|
job_queue.task_done()
|
1213
1254
|
job_queue.put(job_id)
|
@@ -1256,13 +1297,9 @@ class Workflow(BaseModel):
|
|
1256
1297
|
f"that not running."
|
1257
1298
|
)
|
1258
1299
|
|
1259
|
-
# NOTE: Mark this job queue done.
|
1260
1300
|
job_queue.task_done()
|
1261
1301
|
|
1262
1302
|
if not_timeout_flag:
|
1263
|
-
|
1264
|
-
# NOTE: Wait for all items to finish processing by `task_done()`
|
1265
|
-
# method.
|
1266
1303
|
job_queue.join()
|
1267
1304
|
executor.shutdown()
|
1268
1305
|
return context
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: ddeutil-workflow
|
3
|
-
Version: 0.0.
|
3
|
+
Version: 0.0.42
|
4
4
|
Summary: Lightweight workflow orchestration
|
5
5
|
Author-email: ddeutils <korawich.anu@gmail.com>
|
6
6
|
License: MIT
|
@@ -264,7 +264,7 @@ it will use default value and do not raise any error to you.
|
|
264
264
|
| Name | Component | Default | Description |
|
265
265
|
|:-----------------------------|:---------:|:--------------------------------------------------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------|
|
266
266
|
| **ROOT_PATH** | Core | `.` | The root path of the workflow application. |
|
267
|
-
| **
|
267
|
+
| **REGISTRY_CALLER** | Core | `.` | List of importable string for the call stage. |
|
268
268
|
| **REGISTRY_FILTER** | Core | `ddeutil.workflow.templates` | List of importable string for the filter template. |
|
269
269
|
| **CONF_PATH** | Core | `conf` | The config path that keep all template `.yaml` files. |
|
270
270
|
| **TIMEZONE** | Core | `Asia/Bangkok` | A Timezone string value that will pass to `ZoneInfo` object. |
|
@@ -272,20 +272,17 @@ it will use default value and do not raise any error to you.
|
|
272
272
|
| **STAGE_RAISE_ERROR** | Core | `false` | A flag that all stage raise StageException from stage execution. |
|
273
273
|
| **JOB_DEFAULT_ID** | Core | `false` | A flag that enable default job ID that use for catch an execution output. The ID that use will be sequence number. |
|
274
274
|
| **JOB_RAISE_ERROR** | Core | `true` | A flag that all job raise JobException from job strategy execution. |
|
275
|
-
| **MAX_NUM_POKING** | Core | `4` | . |
|
276
|
-
| **MAX_JOB_PARALLEL** | Core | `2` | The maximum job number that able to run parallel in workflow executor. |
|
277
|
-
| **MAX_JOB_EXEC_TIMEOUT** | Core | `600` | |
|
278
275
|
| **MAX_CRON_PER_WORKFLOW** | Core | `5` | |
|
279
276
|
| **MAX_QUEUE_COMPLETE_HIST** | Core | `16` | |
|
280
277
|
| **GENERATE_ID_SIMPLE_MODE** | Core | `true` | A flog that enable generating ID with `md5` algorithm. |
|
281
|
-
| **
|
278
|
+
| **TRACE_PATH** | Log | `./logs` | The log path of the workflow saving log. |
|
282
279
|
| **DEBUG_MODE** | Log | `true` | A flag that enable logging with debug level mode. |
|
283
280
|
| **FORMAT** | Log | `%(asctime)s.%(msecs)03d (%(name)-10s, %(process)-5d,%(thread)-5d) [%(levelname)-7s] %(message)-120s (%(filename)s:%(lineno)s)` | |
|
284
281
|
| **FORMAT_FILE** | Log | `{datetime} ({process:5d}, {thread:5d}) {message:120s} ({filename}:{lineno})` | |
|
285
282
|
| **DATETIME_FORMAT** | Log | `%Y-%m-%d %H:%M:%S` | |
|
286
|
-
| **
|
287
|
-
| **
|
288
|
-
| **
|
283
|
+
| **TRACE_ENABLE_WRITE** | Log | `false` | |
|
284
|
+
| **AUDIT_PATH** | Log | `./audits` | |
|
285
|
+
| **AUDIT_ENABLE_WRITE** | Log | `true` | A flag that enable logging object saving log to its destination. |
|
289
286
|
| **MAX_PROCESS** | App | `2` | The maximum process worker number that run in scheduler app module. |
|
290
287
|
| **MAX_SCHEDULE_PER_PROCESS** | App | `100` | A schedule per process that run parallel. |
|
291
288
|
| **STOP_BOUNDARY_DELTA** | App | `'{"minutes": 5, "seconds": 20}'` | A time delta value that use to stop scheduler app in json string format. |
|
@@ -0,0 +1,30 @@
|
|
1
|
+
ddeutil/workflow/__about__.py,sha256=B3KiwXw9ATZmMG1vER6qdPImMLkQPPjYkRvYepuIhF4,28
|
2
|
+
ddeutil/workflow/__cron.py,sha256=h8rLeIUAAEB2SdZ4Jhch7LU1Yl3bbJ-iNNJ3tQ0eYVM,28095
|
3
|
+
ddeutil/workflow/__init__.py,sha256=cYWwG2utpsYvdwqvkFSRWi_Q6gylDgNQBcIWcF5NFs4,1861
|
4
|
+
ddeutil/workflow/__types.py,sha256=8jBdbfb3aZSetjz0mvNrpGHwwxJff7mK8_4v41cLqlc,4316
|
5
|
+
ddeutil/workflow/conf.py,sha256=lDzWiVSNlNAhTzxbNIhIbQAIF1ggbmetAp0yn2fgnsc,12385
|
6
|
+
ddeutil/workflow/cron.py,sha256=80SijzMdDOBxTWRsiF-Fmuz7Ym7leY0XT2lzRAPGdXc,8781
|
7
|
+
ddeutil/workflow/exceptions.py,sha256=fO37f9p7lOjIJgVOpKE_1X44yJTwBepyukZV9a7NNm4,1241
|
8
|
+
ddeutil/workflow/job.py,sha256=vsayKMzwKDpjchgYQnbshZHnp-vuM9CobpFWhUJETRU,30315
|
9
|
+
ddeutil/workflow/logs.py,sha256=RkM5o_JPoWhFY7NrbYAARZQWjLC62YB_FYzTTcyDp8U,19816
|
10
|
+
ddeutil/workflow/params.py,sha256=Mv-D2DY5inm1ug0lsgCPDkO5wT_AUhc5XEF5jxgDx6U,8036
|
11
|
+
ddeutil/workflow/result.py,sha256=ynZB0g_vEEXn24034J-hatjNWDBmRAj38S8SqGRM-8I,4029
|
12
|
+
ddeutil/workflow/reusables.py,sha256=AtZO83HDFu1uK_azUinv5d8jsA36f2i3n_tqMrolbvc,17529
|
13
|
+
ddeutil/workflow/scheduler.py,sha256=wFEgcnxtgF-8y5otv8RqT1MuBttZl7mu-bBu5ffwV_Y,27534
|
14
|
+
ddeutil/workflow/stages.py,sha256=prw1-za1zwYehbrjeAnoJ79GxpfTqdKLsI2PY0OuSlY,48417
|
15
|
+
ddeutil/workflow/utils.py,sha256=sblje9qOtejCHVt8EVrbC0KY98vKqvxccaR5HIkRiTA,7363
|
16
|
+
ddeutil/workflow/workflow.py,sha256=Y1D5arh2KSobkIZGJ1fWSTe15heURi9OhhdfIr0jHyo,50591
|
17
|
+
ddeutil/workflow/api/__init__.py,sha256=F53NMBWtb9IKaDWkPU5KvybGGfKAcbehgn6TLBwHuuM,21
|
18
|
+
ddeutil/workflow/api/api.py,sha256=b-bMg0aRsEqt8Qb2hNUtamEt2Fq2CgNotF2oXSAdDu8,5226
|
19
|
+
ddeutil/workflow/api/log.py,sha256=NMTnOnsBrDB5129329xF2myLdrb-z9k1MQrmrP7qXJw,1818
|
20
|
+
ddeutil/workflow/api/repeat.py,sha256=cycd1-91j-4v6uY1SkrZHd9l95e-YgVC4UCSNNFuGJ8,5277
|
21
|
+
ddeutil/workflow/api/routes/__init__.py,sha256=qoGtOMyVgQ5nTUc8J8wH27A8isaxl3IFCX8qoyibeCY,484
|
22
|
+
ddeutil/workflow/api/routes/job.py,sha256=YVta083i8vU8-o4WdKFwDpfdC9vN1dZ6goZSmNlQXHA,1954
|
23
|
+
ddeutil/workflow/api/routes/logs.py,sha256=TeRDrEelbKS2Hu_EovgLh0bOdmSv9mfnrIZsrE7uPD4,5353
|
24
|
+
ddeutil/workflow/api/routes/schedules.py,sha256=rUWBm5RgLS1PNBHSWwWXJ0l-c5mYWfl9os0BA9_OTEw,4810
|
25
|
+
ddeutil/workflow/api/routes/workflows.py,sha256=ctgQGxXfpIV6bHFDM9IQ1_qaQHT6n5-HjJ1-D4GKWpc,4527
|
26
|
+
ddeutil_workflow-0.0.42.dist-info/licenses/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
|
27
|
+
ddeutil_workflow-0.0.42.dist-info/METADATA,sha256=TJp1M40eLXYOInkTpl_XhFOWGHLd0hIHktXQXiFsmEw,18853
|
28
|
+
ddeutil_workflow-0.0.42.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
|
29
|
+
ddeutil_workflow-0.0.42.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
|
30
|
+
ddeutil_workflow-0.0.42.dist-info/RECORD,,
|
ddeutil/workflow/context.py
DELETED
@@ -1,61 +0,0 @@
|
|
1
|
-
from __future__ import annotations
|
2
|
-
|
3
|
-
from typing import Optional, Union
|
4
|
-
|
5
|
-
from pydantic import BaseModel, ConfigDict, Field
|
6
|
-
|
7
|
-
from .__types import DictData
|
8
|
-
|
9
|
-
|
10
|
-
class ErrorContext(BaseModel): # pragma: no cov
|
11
|
-
model_config = ConfigDict(arbitrary_types_allowed=True)
|
12
|
-
|
13
|
-
obj: Exception = Field(alias="class")
|
14
|
-
name: str = Field(description="A name of exception class.")
|
15
|
-
message: str = Field(description="A exception message.")
|
16
|
-
|
17
|
-
|
18
|
-
class OutputContext(BaseModel): # pragma: no cov
|
19
|
-
outputs: DictData = Field(default_factory=dict)
|
20
|
-
errors: Optional[ErrorContext] = Field(default=None)
|
21
|
-
skipped: bool = Field(default=False)
|
22
|
-
|
23
|
-
def is_exception(self) -> bool:
|
24
|
-
return self.errors is not None
|
25
|
-
|
26
|
-
|
27
|
-
class StageContext(BaseModel): # pragma: no cov
|
28
|
-
stages: dict[str, OutputContext]
|
29
|
-
errors: Optional[ErrorContext] = Field(default=None)
|
30
|
-
|
31
|
-
def is_exception(self) -> bool:
|
32
|
-
return self.errors is not None
|
33
|
-
|
34
|
-
|
35
|
-
class MatrixContext(StageContext): # pragma: no cov
|
36
|
-
matrix: DictData = Field(default_factory=dict)
|
37
|
-
|
38
|
-
|
39
|
-
MatrixStageContext = dict[
|
40
|
-
str, Union[MatrixContext, StageContext]
|
41
|
-
] # pragma: no cov
|
42
|
-
|
43
|
-
|
44
|
-
class StrategyContext(BaseModel): # pragma: no cov
|
45
|
-
strategies: MatrixStageContext
|
46
|
-
errors: Optional[ErrorContext] = Field(default=None)
|
47
|
-
|
48
|
-
def is_exception(self) -> bool:
|
49
|
-
return self.errors is not None
|
50
|
-
|
51
|
-
|
52
|
-
StrategyMatrixContext = Union[
|
53
|
-
StrategyContext, MatrixStageContext
|
54
|
-
] # pragma: no cov
|
55
|
-
|
56
|
-
|
57
|
-
class JobContext(BaseModel): # pragma: no cov
|
58
|
-
params: DictData = Field(description="A parameterize value")
|
59
|
-
jobs: dict[str, StrategyMatrixContext]
|
60
|
-
errors: Optional[ErrorContext] = Field(default=None)
|
61
|
-
skipped: bool = Field(default=False)
|