ddeutil-workflow 0.0.37__py3-none-any.whl → 0.0.39__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ddeutil/workflow/__about__.py +1 -1
- ddeutil/workflow/__init__.py +4 -1
- ddeutil/workflow/__types.py +2 -0
- ddeutil/workflow/api/routes/job.py +3 -1
- ddeutil/workflow/api/routes/logs.py +12 -4
- ddeutil/workflow/audit.py +7 -5
- ddeutil/workflow/caller.py +17 -12
- ddeutil/workflow/context.py +61 -0
- ddeutil/workflow/exceptions.py +14 -1
- ddeutil/workflow/job.py +224 -135
- ddeutil/workflow/logs.py +6 -1
- ddeutil/workflow/result.py +1 -1
- ddeutil/workflow/stages.py +403 -133
- ddeutil/workflow/templates.py +39 -20
- ddeutil/workflow/utils.py +1 -44
- ddeutil/workflow/workflow.py +168 -84
- {ddeutil_workflow-0.0.37.dist-info → ddeutil_workflow-0.0.39.dist-info}/METADATA +9 -3
- ddeutil_workflow-0.0.39.dist-info/RECORD +33 -0
- {ddeutil_workflow-0.0.37.dist-info → ddeutil_workflow-0.0.39.dist-info}/WHEEL +1 -1
- ddeutil_workflow-0.0.37.dist-info/RECORD +0 -32
- {ddeutil_workflow-0.0.37.dist-info → ddeutil_workflow-0.0.39.dist-info/licenses}/LICENSE +0 -0
- {ddeutil_workflow-0.0.37.dist-info → ddeutil_workflow-0.0.39.dist-info}/top_level.txt +0 -0
ddeutil/workflow/workflow.py
CHANGED
@@ -20,6 +20,7 @@ from functools import partial, total_ordering
|
|
20
20
|
from heapq import heappop, heappush
|
21
21
|
from queue import Queue
|
22
22
|
from textwrap import dedent
|
23
|
+
from threading import Event
|
23
24
|
from typing import Optional
|
24
25
|
|
25
26
|
from pydantic import BaseModel, ConfigDict, Field
|
@@ -33,7 +34,7 @@ from .audit import Audit, get_audit
|
|
33
34
|
from .conf import Loader, config, get_logger
|
34
35
|
from .cron import On
|
35
36
|
from .exceptions import JobException, WorkflowException
|
36
|
-
from .job import Job
|
37
|
+
from .job import Job, TriggerState
|
37
38
|
from .params import Param
|
38
39
|
from .result import Result, Status
|
39
40
|
from .templates import has_template, param2template
|
@@ -68,34 +69,45 @@ class ReleaseType(str, Enum):
|
|
68
69
|
config=ConfigDict(arbitrary_types_allowed=True, use_enum_values=True)
|
69
70
|
)
|
70
71
|
class Release:
|
71
|
-
"""Release Pydantic dataclass object that use for represent
|
72
|
-
|
72
|
+
"""Release Pydantic dataclass object that use for represent the release data
|
73
|
+
that use with the `workflow.release` method.
|
74
|
+
"""
|
73
75
|
|
74
|
-
date: datetime
|
75
|
-
offset: float
|
76
|
-
end_date: datetime
|
77
|
-
runner: CronRunner
|
76
|
+
date: datetime = field()
|
77
|
+
offset: float = field()
|
78
|
+
end_date: datetime = field()
|
79
|
+
runner: CronRunner = field()
|
78
80
|
type: ReleaseType = field(default=ReleaseType.DEFAULT)
|
79
81
|
|
80
82
|
def __repr__(self) -> str:
|
83
|
+
"""Represent string"""
|
81
84
|
return repr(f"{self.date:%Y-%m-%d %H:%M:%S}")
|
82
85
|
|
83
86
|
def __str__(self) -> str:
|
87
|
+
"""Override string value of this release object with the date field.
|
88
|
+
|
89
|
+
:rtype: str
|
90
|
+
"""
|
84
91
|
return f"{self.date:%Y-%m-%d %H:%M:%S}"
|
85
92
|
|
86
93
|
@classmethod
|
87
94
|
def from_dt(cls, dt: datetime | str) -> Self:
|
88
95
|
"""Construct Release via datetime object only.
|
89
96
|
|
90
|
-
:param dt: A datetime object
|
97
|
+
:param dt: (datetime | str) A datetime object or string that want to
|
98
|
+
construct to the Release object.
|
91
99
|
|
92
|
-
:
|
100
|
+
:raise TypeError: If the type of the dt argument does not valid with
|
101
|
+
datetime or str object.
|
102
|
+
|
103
|
+
:rtype: Release
|
93
104
|
"""
|
94
105
|
if isinstance(dt, str):
|
95
106
|
dt: datetime = datetime.fromisoformat(dt)
|
96
107
|
elif not isinstance(dt, datetime):
|
97
108
|
raise TypeError(
|
98
|
-
"The `from_dt` need argument type be str or datetime
|
109
|
+
"The `from_dt` need the `dt` argument type be str or datetime "
|
110
|
+
"only."
|
99
111
|
)
|
100
112
|
|
101
113
|
return cls(
|
@@ -108,6 +120,8 @@ class Release:
|
|
108
120
|
def __eq__(self, other: Release | datetime) -> bool:
|
109
121
|
"""Override equal property that will compare only the same type or
|
110
122
|
datetime.
|
123
|
+
|
124
|
+
:rtype: bool
|
111
125
|
"""
|
112
126
|
if isinstance(other, self.__class__):
|
113
127
|
return self.date == other.date
|
@@ -118,6 +132,8 @@ class Release:
|
|
118
132
|
def __lt__(self, other: Release | datetime) -> bool:
|
119
133
|
"""Override equal property that will compare only the same type or
|
120
134
|
datetime.
|
135
|
+
|
136
|
+
:rtype: bool
|
121
137
|
"""
|
122
138
|
if isinstance(other, self.__class__):
|
123
139
|
return self.date < other.date
|
@@ -143,7 +159,7 @@ class ReleaseQueue:
|
|
143
159
|
|
144
160
|
:raise TypeError: If the type of input queue does not valid.
|
145
161
|
|
146
|
-
:rtype:
|
162
|
+
:rtype: ReleaseQueue
|
147
163
|
"""
|
148
164
|
if queue is None:
|
149
165
|
return cls()
|
@@ -174,7 +190,7 @@ class ReleaseQueue:
|
|
174
190
|
"""Check an input Release object is the first value of the
|
175
191
|
waiting queue.
|
176
192
|
|
177
|
-
:rtype:
|
193
|
+
:rtype: Release
|
178
194
|
"""
|
179
195
|
return self.queue[0]
|
180
196
|
|
@@ -265,12 +281,12 @@ class Workflow(BaseModel):
|
|
265
281
|
an input workflow name. The loader object will use this workflow name to
|
266
282
|
searching configuration data of this workflow model in conf path.
|
267
283
|
|
268
|
-
:raise ValueError: If the type does not match with current object.
|
269
|
-
|
270
284
|
:param name: A workflow name that want to pass to Loader object.
|
271
285
|
:param externals: An external parameters that want to pass to Loader
|
272
286
|
object.
|
273
287
|
|
288
|
+
:raise ValueError: If the type does not match with current object.
|
289
|
+
|
274
290
|
:rtype: Self
|
275
291
|
"""
|
276
292
|
loader: Loader = Loader(name, externals=(externals or {}))
|
@@ -285,11 +301,11 @@ class Workflow(BaseModel):
|
|
285
301
|
loader_data["name"] = name.replace(" ", "_")
|
286
302
|
|
287
303
|
# NOTE: Prepare `on` data
|
288
|
-
cls.
|
304
|
+
cls.__bypass_on__(loader_data, externals=externals)
|
289
305
|
return cls.model_validate(obj=loader_data)
|
290
306
|
|
291
307
|
@classmethod
|
292
|
-
def
|
308
|
+
def __bypass_on__(
|
293
309
|
cls,
|
294
310
|
data: DictData,
|
295
311
|
externals: DictData | None = None,
|
@@ -405,10 +421,13 @@ class Workflow(BaseModel):
|
|
405
421
|
return self
|
406
422
|
|
407
423
|
def job(self, name: str) -> Job:
|
408
|
-
"""Return
|
424
|
+
"""Return the workflow's job model that searching with an input job's
|
425
|
+
name or job's ID.
|
409
426
|
|
410
|
-
:param name: A job name that want to get from a mapping of
|
411
|
-
|
427
|
+
:param name: (str) A job name or ID that want to get from a mapping of
|
428
|
+
job models.
|
429
|
+
|
430
|
+
:raise ValueError: If a name or ID does not exist on the jobs field.
|
412
431
|
|
413
432
|
:rtype: Job
|
414
433
|
:return: A job model that exists on this workflow by input name.
|
@@ -505,18 +524,19 @@ class Workflow(BaseModel):
|
|
505
524
|
:param result: (Result) A result object for keeping context and status
|
506
525
|
data.
|
507
526
|
|
527
|
+
:raise TypeError: If a queue parameter does not match with ReleaseQueue
|
528
|
+
type.
|
529
|
+
|
508
530
|
:rtype: Result
|
509
531
|
"""
|
510
532
|
audit: type[Audit] = audit or get_audit()
|
511
533
|
name: str = override_log_name or self.name
|
512
|
-
|
513
|
-
|
514
|
-
|
515
|
-
|
516
|
-
|
517
|
-
|
518
|
-
elif parent_run_id: # pragma: no cov
|
519
|
-
result.set_parent_run_id(parent_run_id)
|
534
|
+
result: Result = Result.construct_with_rs_or_id(
|
535
|
+
result,
|
536
|
+
run_id=run_id,
|
537
|
+
parent_run_id=parent_run_id,
|
538
|
+
id_logic=name,
|
539
|
+
)
|
520
540
|
|
521
541
|
if queue is not None and not isinstance(queue, ReleaseQueue):
|
522
542
|
raise TypeError(
|
@@ -795,9 +815,7 @@ class Workflow(BaseModel):
|
|
795
815
|
partial_queue(q)
|
796
816
|
continue
|
797
817
|
|
798
|
-
# NOTE: Push the latest Release to the running queue.
|
799
818
|
heappush(q.running, release)
|
800
|
-
|
801
819
|
futures.append(
|
802
820
|
executor.submit(
|
803
821
|
self.release,
|
@@ -827,6 +845,7 @@ class Workflow(BaseModel):
|
|
827
845
|
params: DictData,
|
828
846
|
*,
|
829
847
|
result: Result | None = None,
|
848
|
+
event: Event | None = None,
|
830
849
|
raise_error: bool = True,
|
831
850
|
) -> Result:
|
832
851
|
"""Job execution with passing dynamic parameters from the main workflow
|
@@ -842,10 +861,12 @@ class Workflow(BaseModel):
|
|
842
861
|
|
843
862
|
:param job_id: A job ID that want to execute.
|
844
863
|
:param params: A params that was parameterized from workflow execution.
|
845
|
-
:param raise_error: A flag that raise error instead catching to result
|
846
|
-
if it gets exception from job execution.
|
847
864
|
:param result: (Result) A result object for keeping context and status
|
848
865
|
data.
|
866
|
+
:param event: (Event) An event manager that pass to the
|
867
|
+
PoolThreadExecutor.
|
868
|
+
:param raise_error: A flag that raise error instead catching to result
|
869
|
+
if it gets exception from job execution.
|
849
870
|
|
850
871
|
:rtype: Result
|
851
872
|
:return: Return the result object that receive the job execution result
|
@@ -861,7 +882,11 @@ class Workflow(BaseModel):
|
|
861
882
|
f"workflow."
|
862
883
|
)
|
863
884
|
|
864
|
-
|
885
|
+
if event and event.is_set(): # pragma: no cov
|
886
|
+
raise WorkflowException(
|
887
|
+
"Workflow job was canceled from event that had set before "
|
888
|
+
"job execution."
|
889
|
+
)
|
865
890
|
|
866
891
|
# IMPORTANT:
|
867
892
|
# This execution change all job running IDs to the current workflow
|
@@ -871,14 +896,20 @@ class Workflow(BaseModel):
|
|
871
896
|
#
|
872
897
|
try:
|
873
898
|
job: Job = self.jobs[job_id]
|
874
|
-
job.
|
875
|
-
|
876
|
-
|
877
|
-
|
878
|
-
|
879
|
-
|
880
|
-
|
881
|
-
|
899
|
+
if job.is_skipped(params=params):
|
900
|
+
result.trace.info(f"[JOB]: Skip job: {job_id!r}")
|
901
|
+
job.set_outputs(output={"SKIP": {"skipped": True}}, to=params)
|
902
|
+
else:
|
903
|
+
result.trace.info(f"[JOB]: Start execute job: {job_id!r}")
|
904
|
+
job.set_outputs(
|
905
|
+
job.execute(
|
906
|
+
params=params,
|
907
|
+
run_id=result.run_id,
|
908
|
+
parent_run_id=result.parent_run_id,
|
909
|
+
event=event,
|
910
|
+
).context,
|
911
|
+
to=params,
|
912
|
+
)
|
882
913
|
except JobException as err:
|
883
914
|
result.trace.error(f"[WORKFLOW]: {err.__class__.__name__}: {err}")
|
884
915
|
if raise_error:
|
@@ -889,7 +920,7 @@ class Workflow(BaseModel):
|
|
889
920
|
"Handle error from the job execution does not support yet."
|
890
921
|
) from None
|
891
922
|
|
892
|
-
return result.catch(status=
|
923
|
+
return result.catch(status=Status.SUCCESS, context=params)
|
893
924
|
|
894
925
|
def execute(
|
895
926
|
self,
|
@@ -927,22 +958,21 @@ class Workflow(BaseModel):
|
|
927
958
|
:rtype: Result
|
928
959
|
"""
|
929
960
|
ts: float = time.monotonic()
|
930
|
-
|
931
|
-
result
|
932
|
-
|
933
|
-
|
934
|
-
|
935
|
-
|
936
|
-
result.set_parent_run_id(parent_run_id)
|
961
|
+
result: Result = Result.construct_with_rs_or_id(
|
962
|
+
result,
|
963
|
+
run_id=run_id,
|
964
|
+
parent_run_id=parent_run_id,
|
965
|
+
id_logic=self.name,
|
966
|
+
)
|
937
967
|
|
938
968
|
result.trace.info(f"[WORKFLOW]: Start Execute: {self.name!r} ...")
|
939
969
|
|
940
970
|
# NOTE: It should not do anything if it does not have job.
|
941
971
|
if not self.jobs:
|
942
972
|
result.trace.warning(
|
943
|
-
f"[WORKFLOW]:
|
973
|
+
f"[WORKFLOW]: {self.name!r} does not have any jobs"
|
944
974
|
)
|
945
|
-
return result.catch(status=
|
975
|
+
return result.catch(status=Status.SUCCESS, context=params)
|
946
976
|
|
947
977
|
# NOTE: Create a job queue that keep the job that want to run after
|
948
978
|
# its dependency condition.
|
@@ -959,7 +989,7 @@ class Workflow(BaseModel):
|
|
959
989
|
# }
|
960
990
|
#
|
961
991
|
context: DictData = self.parameterize(params)
|
962
|
-
status:
|
992
|
+
status: Status = Status.SUCCESS
|
963
993
|
try:
|
964
994
|
if config.max_job_parallel == 1:
|
965
995
|
self.__exec_non_threading(
|
@@ -978,16 +1008,9 @@ class Workflow(BaseModel):
|
|
978
1008
|
timeout=timeout,
|
979
1009
|
)
|
980
1010
|
except WorkflowException as err:
|
981
|
-
status
|
982
|
-
context.update(
|
983
|
-
|
984
|
-
"errors": {
|
985
|
-
"class": err,
|
986
|
-
"name": err.__class__.__name__,
|
987
|
-
"message": f"{err.__class__.__name__}: {err}",
|
988
|
-
},
|
989
|
-
},
|
990
|
-
)
|
1011
|
+
status = Status.FAILED
|
1012
|
+
context.update({"errors": err.to_dict()})
|
1013
|
+
|
991
1014
|
return result.catch(status=status, context=context)
|
992
1015
|
|
993
1016
|
def __exec_threading(
|
@@ -1005,18 +1028,19 @@ class Workflow(BaseModel):
|
|
1005
1028
|
If a job need dependency, it will check dependency job ID from
|
1006
1029
|
context data before allow it run.
|
1007
1030
|
|
1008
|
-
:param result: A result model.
|
1031
|
+
:param result: (Result) A result model.
|
1009
1032
|
:param context: A context workflow data that want to downstream passing.
|
1010
1033
|
:param ts: A start timestamp that use for checking execute time should
|
1011
1034
|
time out.
|
1012
|
-
:param job_queue: A job queue object.
|
1013
|
-
:param timeout: A second value unit that bounding running time.
|
1035
|
+
:param job_queue: (Queue) A job queue object.
|
1036
|
+
:param timeout: (int) A second value unit that bounding running time.
|
1014
1037
|
:param thread_timeout: A timeout to waiting all futures complete.
|
1015
1038
|
|
1016
1039
|
:rtype: DictData
|
1017
1040
|
"""
|
1018
1041
|
not_timeout_flag: bool = True
|
1019
1042
|
timeout: int = timeout or config.max_job_exec_timeout
|
1043
|
+
event: Event = Event()
|
1020
1044
|
result.trace.debug(f"[WORKFLOW]: Run {self.name!r} with threading.")
|
1021
1045
|
|
1022
1046
|
# IMPORTANT: The job execution can run parallel and waiting by
|
@@ -1033,10 +1057,19 @@ class Workflow(BaseModel):
|
|
1033
1057
|
job_id: str = job_queue.get()
|
1034
1058
|
job: Job = self.jobs[job_id]
|
1035
1059
|
|
1036
|
-
if
|
1060
|
+
if (check := job.check_needs(context["jobs"])).is_waiting():
|
1037
1061
|
job_queue.task_done()
|
1038
1062
|
job_queue.put(job_id)
|
1039
|
-
time.sleep(0.
|
1063
|
+
time.sleep(0.15)
|
1064
|
+
continue
|
1065
|
+
elif check == TriggerState.failed: # pragma: no cov
|
1066
|
+
raise WorkflowException(
|
1067
|
+
"Check job trigger rule was failed."
|
1068
|
+
)
|
1069
|
+
elif check == TriggerState.skipped: # pragma: no cov
|
1070
|
+
result.trace.info(f"[JOB]: Skip job: {job_id!r}")
|
1071
|
+
job.set_outputs({"SKIP": {"skipped": True}}, to=context)
|
1072
|
+
job_queue.task_done()
|
1040
1073
|
continue
|
1041
1074
|
|
1042
1075
|
# NOTE: Start workflow job execution with deep copy context data
|
@@ -1055,6 +1088,7 @@ class Workflow(BaseModel):
|
|
1055
1088
|
job_id=job_id,
|
1056
1089
|
params=context,
|
1057
1090
|
result=result,
|
1091
|
+
event=event,
|
1058
1092
|
),
|
1059
1093
|
)
|
1060
1094
|
|
@@ -1077,11 +1111,13 @@ class Workflow(BaseModel):
|
|
1077
1111
|
|
1078
1112
|
return context
|
1079
1113
|
|
1114
|
+
result.trace.error(
|
1115
|
+
f"[WORKFLOW]: Execution: {self.name!r} was timeout."
|
1116
|
+
)
|
1117
|
+
event.set()
|
1080
1118
|
for future in futures:
|
1081
1119
|
future.cancel()
|
1082
1120
|
|
1083
|
-
# NOTE: Raise timeout error.
|
1084
|
-
result.trace.error(f"[WORKFLOW]: Execution: {self.name!r} was timeout.")
|
1085
1121
|
raise WorkflowException(f"Execution: {self.name!r} was timeout.")
|
1086
1122
|
|
1087
1123
|
def __exec_non_threading(
|
@@ -1099,18 +1135,25 @@ class Workflow(BaseModel):
|
|
1099
1135
|
If a job need dependency, it will check dependency job ID from
|
1100
1136
|
context data before allow it run.
|
1101
1137
|
|
1102
|
-
:param result: A result model.
|
1138
|
+
:param result: (Result) A result model.
|
1103
1139
|
:param context: A context workflow data that want to downstream passing.
|
1104
|
-
:param ts: A start timestamp that use for checking execute time
|
1105
|
-
time out.
|
1106
|
-
:param timeout: A second value unit that bounding running time.
|
1140
|
+
:param ts: (float) A start timestamp that use for checking execute time
|
1141
|
+
should time out.
|
1142
|
+
:param timeout: (int) A second value unit that bounding running time.
|
1107
1143
|
|
1108
1144
|
:rtype: DictData
|
1109
1145
|
"""
|
1110
1146
|
not_timeout_flag: bool = True
|
1111
1147
|
timeout: int = timeout or config.max_job_exec_timeout
|
1148
|
+
event: Event = Event()
|
1149
|
+
future: Future | None = None
|
1112
1150
|
result.trace.debug(f"[WORKFLOW]: Run {self.name!r} with non-threading.")
|
1113
1151
|
|
1152
|
+
executor = ThreadPoolExecutor(
|
1153
|
+
max_workers=1,
|
1154
|
+
thread_name_prefix="wf_exec_non_threading_",
|
1155
|
+
)
|
1156
|
+
|
1114
1157
|
while not job_queue.empty() and (
|
1115
1158
|
not_timeout_flag := ((time.monotonic() - ts) < timeout)
|
1116
1159
|
):
|
@@ -1118,11 +1161,18 @@ class Workflow(BaseModel):
|
|
1118
1161
|
job: Job = self.jobs[job_id]
|
1119
1162
|
|
1120
1163
|
# NOTE: Waiting dependency job run successful before release.
|
1121
|
-
if
|
1164
|
+
if (check := job.check_needs(context["jobs"])).is_waiting():
|
1122
1165
|
job_queue.task_done()
|
1123
1166
|
job_queue.put(job_id)
|
1124
1167
|
time.sleep(0.075)
|
1125
1168
|
continue
|
1169
|
+
elif check == TriggerState.failed: # pragma: no cov
|
1170
|
+
raise WorkflowException("Check job trigger rule was failed.")
|
1171
|
+
elif check == TriggerState.skipped: # pragma: no cov
|
1172
|
+
result.trace.info(f"[JOB]: Skip job: {job_id!r}")
|
1173
|
+
job.set_outputs({"SKIP": {"skipped": True}}, to=context)
|
1174
|
+
job_queue.task_done()
|
1175
|
+
continue
|
1126
1176
|
|
1127
1177
|
# NOTE: Start workflow job execution with deep copy context data
|
1128
1178
|
# before release. This job execution process will run until
|
@@ -1132,7 +1182,32 @@ class Workflow(BaseModel):
|
|
1132
1182
|
# 'params': <input-params>,
|
1133
1183
|
# 'jobs': {},
|
1134
1184
|
# }
|
1135
|
-
|
1185
|
+
if future is None:
|
1186
|
+
future: Future = executor.submit(
|
1187
|
+
self.execute_job,
|
1188
|
+
job_id=job_id,
|
1189
|
+
params=context,
|
1190
|
+
result=result,
|
1191
|
+
event=event,
|
1192
|
+
)
|
1193
|
+
result.trace.debug(f"[WORKFLOW]: Make future: {future}")
|
1194
|
+
time.sleep(0.025)
|
1195
|
+
elif future.done():
|
1196
|
+
if err := future.exception():
|
1197
|
+
result.trace.error(f"[WORKFLOW]: {err}")
|
1198
|
+
raise WorkflowException(str(err))
|
1199
|
+
|
1200
|
+
future = None
|
1201
|
+
job_queue.put(job_id)
|
1202
|
+
elif future.running():
|
1203
|
+
time.sleep(0.075)
|
1204
|
+
job_queue.put(job_id)
|
1205
|
+
else: # pragma: no cov
|
1206
|
+
job_queue.put(job_id)
|
1207
|
+
result.trace.debug(
|
1208
|
+
f"Execution non-threading does not handle case: {future} "
|
1209
|
+
f"that not running."
|
1210
|
+
)
|
1136
1211
|
|
1137
1212
|
# NOTE: Mark this job queue done.
|
1138
1213
|
job_queue.task_done()
|
@@ -1142,11 +1217,12 @@ class Workflow(BaseModel):
|
|
1142
1217
|
# NOTE: Wait for all items to finish processing by `task_done()`
|
1143
1218
|
# method.
|
1144
1219
|
job_queue.join()
|
1145
|
-
|
1220
|
+
executor.shutdown()
|
1146
1221
|
return context
|
1147
1222
|
|
1148
|
-
# NOTE: Raise timeout error.
|
1149
1223
|
result.trace.error(f"[WORKFLOW]: Execution: {self.name!r} was timeout.")
|
1224
|
+
event.set()
|
1225
|
+
executor.shutdown()
|
1150
1226
|
raise WorkflowException(f"Execution: {self.name!r} was timeout.")
|
1151
1227
|
|
1152
1228
|
|
@@ -1162,9 +1238,9 @@ class WorkflowTask:
|
|
1162
1238
|
arguments before passing to the parent release method.
|
1163
1239
|
"""
|
1164
1240
|
|
1165
|
-
alias: str
|
1166
|
-
workflow: Workflow
|
1167
|
-
runner: CronRunner
|
1241
|
+
alias: str = field()
|
1242
|
+
workflow: Workflow = field()
|
1243
|
+
runner: CronRunner = field()
|
1168
1244
|
values: DictData = field(default_factory=dict)
|
1169
1245
|
|
1170
1246
|
def release(
|
@@ -1185,6 +1261,11 @@ class WorkflowTask:
|
|
1185
1261
|
:param audit: An audit class that want to save the execution result.
|
1186
1262
|
:param queue: A ReleaseQueue object that use to mark complete.
|
1187
1263
|
|
1264
|
+
:raise ValueError: If a queue parameter does not pass while release
|
1265
|
+
is None.
|
1266
|
+
:raise TypeError: If a queue parameter does not match with ReleaseQueue
|
1267
|
+
type.
|
1268
|
+
|
1188
1269
|
:rtype: Result
|
1189
1270
|
"""
|
1190
1271
|
audit: type[Audit] = audit or get_audit()
|
@@ -1209,7 +1290,6 @@ class WorkflowTask:
|
|
1209
1290
|
else:
|
1210
1291
|
release = self.runner.date
|
1211
1292
|
|
1212
|
-
# NOTE: Call the workflow release method.
|
1213
1293
|
return self.workflow.release(
|
1214
1294
|
release=release,
|
1215
1295
|
params=self.values,
|
@@ -1264,13 +1344,14 @@ class WorkflowTask:
|
|
1264
1344
|
if self.runner.date > end_date:
|
1265
1345
|
return queue
|
1266
1346
|
|
1267
|
-
# NOTE: Push the Release object to queue.
|
1268
1347
|
heappush(queue.queue, workflow_release)
|
1269
|
-
|
1270
1348
|
return queue
|
1271
1349
|
|
1272
1350
|
def __repr__(self) -> str:
|
1273
|
-
"""Override the `__repr__` method.
|
1351
|
+
"""Override the `__repr__` method.
|
1352
|
+
|
1353
|
+
:rtype: str
|
1354
|
+
"""
|
1274
1355
|
return (
|
1275
1356
|
f"{self.__class__.__name__}(alias={self.alias!r}, "
|
1276
1357
|
f"workflow={self.workflow.name!r}, runner={self.runner!r}, "
|
@@ -1278,7 +1359,10 @@ class WorkflowTask:
|
|
1278
1359
|
)
|
1279
1360
|
|
1280
1361
|
def __eq__(self, other: WorkflowTask) -> bool:
|
1281
|
-
"""Override equal property that will compare only the same type.
|
1362
|
+
"""Override the equal property that will compare only the same type.
|
1363
|
+
|
1364
|
+
:rtype: bool
|
1365
|
+
"""
|
1282
1366
|
if isinstance(other, WorkflowTask):
|
1283
1367
|
return (
|
1284
1368
|
self.workflow.name == other.workflow.name
|
@@ -1,6 +1,6 @@
|
|
1
|
-
Metadata-Version: 2.
|
1
|
+
Metadata-Version: 2.4
|
2
2
|
Name: ddeutil-workflow
|
3
|
-
Version: 0.0.
|
3
|
+
Version: 0.0.39
|
4
4
|
Summary: Lightweight workflow orchestration
|
5
5
|
Author-email: ddeutils <korawich.anu@gmail.com>
|
6
6
|
License: MIT
|
@@ -31,6 +31,10 @@ Provides-Extra: api
|
|
31
31
|
Requires-Dist: fastapi<1.0.0,>=0.115.0; extra == "api"
|
32
32
|
Requires-Dist: httpx; extra == "api"
|
33
33
|
Requires-Dist: ujson; extra == "api"
|
34
|
+
Provides-Extra: async
|
35
|
+
Requires-Dist: aiofiles; extra == "async"
|
36
|
+
Requires-Dist: aiohttp; extra == "async"
|
37
|
+
Dynamic: license-file
|
34
38
|
|
35
39
|
# Workflow Orchestration
|
36
40
|
|
@@ -126,7 +130,7 @@ flowchart LR
|
|
126
130
|
> - [Google **Workflows**](https://cloud.google.com/workflows)
|
127
131
|
> - [AWS **Step Functions**](https://aws.amazon.com/step-functions/)
|
128
132
|
|
129
|
-
##
|
133
|
+
## 📦 Installation
|
130
134
|
|
131
135
|
This project need `ddeutil` and `ddeutil-io` extension namespace packages.
|
132
136
|
If you want to install this package with application add-ons, you should add
|
@@ -165,6 +169,8 @@ run-py-local:
|
|
165
169
|
run-date: datetime
|
166
170
|
jobs:
|
167
171
|
getting-api-data:
|
172
|
+
runs-on:
|
173
|
+
type: local
|
168
174
|
stages:
|
169
175
|
- name: "Retrieve API Data"
|
170
176
|
id: retrieve-api
|
@@ -0,0 +1,33 @@
|
|
1
|
+
ddeutil/workflow/__about__.py,sha256=lWjP4jp8-3HcVlFUQBiNI9hwnZOyEL0S-RmG6USjiag,28
|
2
|
+
ddeutil/workflow/__cron.py,sha256=3i-wmjTlh0ADCzN9pLKaWHzJkXzC72aIBmVEQSbyCCE,26895
|
3
|
+
ddeutil/workflow/__init__.py,sha256=hIM2Ha-7F3YF3aLsu4IY3N-UvD_EE1ai6DO6WL2mILg,1908
|
4
|
+
ddeutil/workflow/__types.py,sha256=8jBdbfb3aZSetjz0mvNrpGHwwxJff7mK8_4v41cLqlc,4316
|
5
|
+
ddeutil/workflow/audit.py,sha256=BpzLI6ZKXi6xQO8C2sAHxSi8RAfF31dMPdjn3DbYlw8,8364
|
6
|
+
ddeutil/workflow/caller.py,sha256=M7_nan1DbRUAHEIQoQc1qIX4AHgI5fKFNx0KDbzhDMk,5736
|
7
|
+
ddeutil/workflow/conf.py,sha256=MHzBeLZukFeIQ-YhxOz5uKCnGYqbhYdpwAEh9A9h_OM,12216
|
8
|
+
ddeutil/workflow/context.py,sha256=vsk4JQL7t3KsnKPfshw3O7YrPFo2h4rnnNd3B-G9Kj4,1700
|
9
|
+
ddeutil/workflow/cron.py,sha256=j8EeoHst70toRfnD_frix41vrI-eLYVJkZ9yeJtpfnI,8871
|
10
|
+
ddeutil/workflow/exceptions.py,sha256=fO37f9p7lOjIJgVOpKE_1X44yJTwBepyukZV9a7NNm4,1241
|
11
|
+
ddeutil/workflow/job.py,sha256=LCQf_3gyTmnj6aAQ0ksz4lJxU10-F5MfVFHczkWt_VE,28669
|
12
|
+
ddeutil/workflow/logs.py,sha256=mAKQjNri-oourd3dBLLG3Fqoo4QG27U9IO2h6IqCOU0,10196
|
13
|
+
ddeutil/workflow/params.py,sha256=qw9XJyjh2ocf9pf6h_XiYHLOvQN4R5TMqPElmItKnRM,8019
|
14
|
+
ddeutil/workflow/result.py,sha256=cDkItrhpzZfMS1Oj8IZX8O-KBD4KZYDi43XJZvvC3Gc,4318
|
15
|
+
ddeutil/workflow/scheduler.py,sha256=YMebYpNjqg6RWaE17sicwM3uthupeBGSGCnDGy4aKd8,26286
|
16
|
+
ddeutil/workflow/stages.py,sha256=ONBR7GjgLEbA21wNioBrhOucwqj6M1v2ht5JhipoWSg,36994
|
17
|
+
ddeutil/workflow/templates.py,sha256=yA2xgrSXcxfBxNT2hc6v06HkVY_0RKsc1UwdJRip9EE,11554
|
18
|
+
ddeutil/workflow/utils.py,sha256=Fz5y-LK_JfikDfvMKcFaxad_VvCnr7UC2C9KFCbzPNA,7105
|
19
|
+
ddeutil/workflow/workflow.py,sha256=dbOnLx54KtPERuv5M_QU2nZiyiAnzv6pApX49rJi2eI,47833
|
20
|
+
ddeutil/workflow/api/__init__.py,sha256=F53NMBWtb9IKaDWkPU5KvybGGfKAcbehgn6TLBwHuuM,21
|
21
|
+
ddeutil/workflow/api/api.py,sha256=gGQtqkzyJNaJIfka_w2M1lrCS3Ep46re2Dznsk9RxYQ,5191
|
22
|
+
ddeutil/workflow/api/log.py,sha256=NMTnOnsBrDB5129329xF2myLdrb-z9k1MQrmrP7qXJw,1818
|
23
|
+
ddeutil/workflow/api/repeat.py,sha256=cycd1-91j-4v6uY1SkrZHd9l95e-YgVC4UCSNNFuGJ8,5277
|
24
|
+
ddeutil/workflow/api/routes/__init__.py,sha256=qoGtOMyVgQ5nTUc8J8wH27A8isaxl3IFCX8qoyibeCY,484
|
25
|
+
ddeutil/workflow/api/routes/job.py,sha256=YVta083i8vU8-o4WdKFwDpfdC9vN1dZ6goZSmNlQXHA,1954
|
26
|
+
ddeutil/workflow/api/routes/logs.py,sha256=uEQ6k5PwRg-k0eSiSFArXfyeDq5xzepxILrRnwgAe1I,5373
|
27
|
+
ddeutil/workflow/api/routes/schedules.py,sha256=uWYDOwlV8w56hKQmfkQFwdZ6t2gZSJeCdBIzMmJenAQ,4824
|
28
|
+
ddeutil/workflow/api/routes/workflows.py,sha256=KVywA7vD9b4QrfmWBdSFF5chj34yJe1zNCzl6iBMeGI,4538
|
29
|
+
ddeutil_workflow-0.0.39.dist-info/licenses/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
|
30
|
+
ddeutil_workflow-0.0.39.dist-info/METADATA,sha256=r6e0p1K2dUtDOl5XJa-FqxtqgcRzJz7gZWcw9HuhWDc,19500
|
31
|
+
ddeutil_workflow-0.0.39.dist-info/WHEEL,sha256=1tXe9gY0PYatrMPMDd6jXqjfpz_B-Wqm32CPfRC58XU,91
|
32
|
+
ddeutil_workflow-0.0.39.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
|
33
|
+
ddeutil_workflow-0.0.39.dist-info/RECORD,,
|
@@ -1,32 +0,0 @@
|
|
1
|
-
ddeutil/workflow/__about__.py,sha256=UJZ9dzvQ9h4mY_1tAZ0imJIZbhNW3TD-wWNEVs22HwA,28
|
2
|
-
ddeutil/workflow/__cron.py,sha256=3i-wmjTlh0ADCzN9pLKaWHzJkXzC72aIBmVEQSbyCCE,26895
|
3
|
-
ddeutil/workflow/__init__.py,sha256=d643WDkk93MjCt9ujD46hX07Mb7yc9eTzKV3QwnEZQg,1845
|
4
|
-
ddeutil/workflow/__types.py,sha256=CK1jfzyHP9P-MB0ElhpJZ59ZFGJC9MkQuAop5739_9k,4304
|
5
|
-
ddeutil/workflow/audit.py,sha256=wx70RKRdHj1d2431ilpt9OPTInMByjqXkYff7l5pvF4,8230
|
6
|
-
ddeutil/workflow/caller.py,sha256=pmZ9a5m1JlBTzR_xePOWZa98zyFE7jgJUlAXCx874Fs,5521
|
7
|
-
ddeutil/workflow/conf.py,sha256=MHzBeLZukFeIQ-YhxOz5uKCnGYqbhYdpwAEh9A9h_OM,12216
|
8
|
-
ddeutil/workflow/cron.py,sha256=j8EeoHst70toRfnD_frix41vrI-eLYVJkZ9yeJtpfnI,8871
|
9
|
-
ddeutil/workflow/exceptions.py,sha256=5ghT443VLq0IeU87loHNEqqrrrctklP7YfxwJ51ImWU,949
|
10
|
-
ddeutil/workflow/job.py,sha256=WnNkk_XhZytmLPzN6Kb41_BdfvBdtYhbn0SnDJ5ZgEw,25417
|
11
|
-
ddeutil/workflow/logs.py,sha256=EJDb9Xt3XWjTGE8CeEvw0eDU8kyaeStALQNAtTl5HQw,10027
|
12
|
-
ddeutil/workflow/params.py,sha256=qw9XJyjh2ocf9pf6h_XiYHLOvQN4R5TMqPElmItKnRM,8019
|
13
|
-
ddeutil/workflow/result.py,sha256=fbM2An3VyweMjAy4Iw7h8H-KkoQsDrZe_KjGztXAFkE,4319
|
14
|
-
ddeutil/workflow/scheduler.py,sha256=YMebYpNjqg6RWaE17sicwM3uthupeBGSGCnDGy4aKd8,26286
|
15
|
-
ddeutil/workflow/stages.py,sha256=SJD7T7BdIhxoPSaDf8s5I8U6sv7wJsG2BynG-_aZ004,28165
|
16
|
-
ddeutil/workflow/templates.py,sha256=A0JgZFGkBv-AX-EskZj656nG5zFd3j1PpLpyXihf6Xg,10967
|
17
|
-
ddeutil/workflow/utils.py,sha256=JppsS2c545hPqog0GWjpQnTVMnzjqnhx4K8GkMV_CP0,8132
|
18
|
-
ddeutil/workflow/workflow.py,sha256=_LYfs15AcXWVpM8CaO4oH6SWoJnqzF5FU08QTDoHT5w,44529
|
19
|
-
ddeutil/workflow/api/__init__.py,sha256=F53NMBWtb9IKaDWkPU5KvybGGfKAcbehgn6TLBwHuuM,21
|
20
|
-
ddeutil/workflow/api/api.py,sha256=gGQtqkzyJNaJIfka_w2M1lrCS3Ep46re2Dznsk9RxYQ,5191
|
21
|
-
ddeutil/workflow/api/log.py,sha256=NMTnOnsBrDB5129329xF2myLdrb-z9k1MQrmrP7qXJw,1818
|
22
|
-
ddeutil/workflow/api/repeat.py,sha256=cycd1-91j-4v6uY1SkrZHd9l95e-YgVC4UCSNNFuGJ8,5277
|
23
|
-
ddeutil/workflow/api/routes/__init__.py,sha256=qoGtOMyVgQ5nTUc8J8wH27A8isaxl3IFCX8qoyibeCY,484
|
24
|
-
ddeutil/workflow/api/routes/job.py,sha256=vCUTtsoCOtubVqjgk6MYUcGYim_l5Vh_NdtnQGx1SYM,1898
|
25
|
-
ddeutil/workflow/api/routes/logs.py,sha256=7xPKu814PGxaMsij8zB3MLeXBfTC7NKT7GlTOJ-PV2U,5173
|
26
|
-
ddeutil/workflow/api/routes/schedules.py,sha256=uWYDOwlV8w56hKQmfkQFwdZ6t2gZSJeCdBIzMmJenAQ,4824
|
27
|
-
ddeutil/workflow/api/routes/workflows.py,sha256=KVywA7vD9b4QrfmWBdSFF5chj34yJe1zNCzl6iBMeGI,4538
|
28
|
-
ddeutil_workflow-0.0.37.dist-info/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
|
29
|
-
ddeutil_workflow-0.0.37.dist-info/METADATA,sha256=lazCqQyB5Cm-UWkYN9bGz3RnJpv_XhT0JOcmMf-7i5Y,19342
|
30
|
-
ddeutil_workflow-0.0.37.dist-info/WHEEL,sha256=beeZ86-EfXScwlR_HKu4SllMC9wUEj_8Z_4FJ3egI2w,91
|
31
|
-
ddeutil_workflow-0.0.37.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
|
32
|
-
ddeutil_workflow-0.0.37.dist-info/RECORD,,
|
File without changes
|
File without changes
|