ddeutil-workflow 0.0.55__py3-none-any.whl → 0.0.57__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ddeutil/workflow/__about__.py +1 -1
- ddeutil/workflow/__cron.py +26 -12
- ddeutil/workflow/__init__.py +4 -2
- ddeutil/workflow/__main__.py +30 -0
- ddeutil/workflow/__types.py +1 -0
- ddeutil/workflow/conf.py +163 -101
- ddeutil/workflow/{cron.py → event.py} +37 -20
- ddeutil/workflow/exceptions.py +44 -14
- ddeutil/workflow/job.py +87 -58
- ddeutil/workflow/logs.py +13 -5
- ddeutil/workflow/result.py +9 -4
- ddeutil/workflow/scheduler.py +38 -73
- ddeutil/workflow/stages.py +370 -147
- ddeutil/workflow/utils.py +37 -6
- ddeutil/workflow/workflow.py +243 -302
- {ddeutil_workflow-0.0.55.dist-info → ddeutil_workflow-0.0.57.dist-info}/METADATA +41 -35
- ddeutil_workflow-0.0.57.dist-info/RECORD +31 -0
- {ddeutil_workflow-0.0.55.dist-info → ddeutil_workflow-0.0.57.dist-info}/WHEEL +1 -1
- ddeutil_workflow-0.0.57.dist-info/entry_points.txt +2 -0
- ddeutil_workflow-0.0.55.dist-info/RECORD +0 -30
- {ddeutil_workflow-0.0.55.dist-info → ddeutil_workflow-0.0.57.dist-info}/licenses/LICENSE +0 -0
- {ddeutil_workflow-0.0.55.dist-info → ddeutil_workflow-0.0.57.dist-info}/top_level.txt +0 -0
ddeutil/workflow/workflow.py
CHANGED
@@ -3,12 +3,14 @@
|
|
3
3
|
# Licensed under the MIT License. See LICENSE in the project root for
|
4
4
|
# license information.
|
5
5
|
# ------------------------------------------------------------------------------
|
6
|
-
|
7
|
-
|
8
|
-
Release, ReleaseQueue, and Workflow Pydantic models.
|
6
|
+
"""Workflow module is the core module of this package. It keeps Release,
|
7
|
+
ReleaseQueue, and Workflow models.
|
9
8
|
|
10
9
|
This package implement timeout strategy on the workflow execution layer only
|
11
10
|
because the main propose of this package is using Workflow to be orchestrator.
|
11
|
+
|
12
|
+
ReleaseQueue is the memory storage of Release for tracking this release
|
13
|
+
already run or pending in the current session.
|
12
14
|
"""
|
13
15
|
from __future__ import annotations
|
14
16
|
|
@@ -27,30 +29,32 @@ from heapq import heappop, heappush
|
|
27
29
|
from pathlib import Path
|
28
30
|
from queue import Queue
|
29
31
|
from textwrap import dedent
|
30
|
-
from threading import Event
|
31
|
-
from typing import Optional
|
32
|
+
from threading import Event, Lock
|
33
|
+
from typing import Any, Optional, Union
|
34
|
+
from zoneinfo import ZoneInfo
|
32
35
|
|
33
36
|
from pydantic import BaseModel, ConfigDict, Field, ValidationInfo
|
34
37
|
from pydantic.dataclasses import dataclass
|
35
38
|
from pydantic.functional_validators import field_validator, model_validator
|
36
39
|
from typing_extensions import Self
|
37
40
|
|
38
|
-
from .__cron import
|
41
|
+
from .__cron import CronRunner
|
39
42
|
from .__types import DictData, TupleStr
|
40
|
-
from .conf import
|
41
|
-
from .
|
42
|
-
from .exceptions import
|
43
|
+
from .conf import FileLoad, Loader, dynamic
|
44
|
+
from .event import On
|
45
|
+
from .exceptions import WorkflowException
|
43
46
|
from .job import Job
|
44
47
|
from .logs import Audit, get_audit
|
45
48
|
from .params import Param
|
46
|
-
from .result import FAILED, SKIP, SUCCESS, WAIT, Result
|
49
|
+
from .result import CANCEL, FAILED, SKIP, SUCCESS, WAIT, Result
|
47
50
|
from .reusables import has_template, param2template
|
48
51
|
from .utils import (
|
49
|
-
|
52
|
+
clear_tz,
|
50
53
|
gen_id,
|
51
54
|
get_dt_now,
|
52
55
|
reach_next_minute,
|
53
|
-
|
56
|
+
replace_sec,
|
57
|
+
wait_until_next_minute,
|
54
58
|
)
|
55
59
|
|
56
60
|
__all__: TupleStr = (
|
@@ -66,46 +70,50 @@ class ReleaseType(str, Enum):
|
|
66
70
|
"""Release Type Enum support the type field on the Release dataclass."""
|
67
71
|
|
68
72
|
DEFAULT: str = "manual"
|
69
|
-
|
70
|
-
|
73
|
+
SCHEDULE: str = "schedule"
|
74
|
+
POKING: str = "poking"
|
75
|
+
FORCE: str = "force"
|
71
76
|
|
72
77
|
|
73
78
|
@total_ordering
|
74
|
-
@dataclass(
|
75
|
-
config=ConfigDict(arbitrary_types_allowed=True, use_enum_values=True)
|
76
|
-
)
|
79
|
+
@dataclass(config=ConfigDict(use_enum_values=True))
|
77
80
|
class Release:
|
78
|
-
"""Release
|
79
|
-
that use with the `workflow.release` method.
|
80
|
-
"""
|
81
|
+
"""Release object that use for represent the release datetime."""
|
81
82
|
|
82
|
-
date: datetime
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
83
|
+
date: datetime = Field(
|
84
|
+
description=(
|
85
|
+
"A release date that should has second and millisecond equal 0."
|
86
|
+
)
|
87
|
+
)
|
88
|
+
type: ReleaseType = Field(
|
89
|
+
default=ReleaseType.DEFAULT,
|
90
|
+
description="A type of release that create before start execution.",
|
91
|
+
)
|
87
92
|
|
88
93
|
def __repr__(self) -> str:
|
89
|
-
"""
|
94
|
+
"""Override __repr__ method for represent value of `date` field.
|
95
|
+
|
96
|
+
:rtype: str
|
97
|
+
"""
|
90
98
|
return repr(f"{self.date:%Y-%m-%d %H:%M:%S}")
|
91
99
|
|
92
100
|
def __str__(self) -> str:
|
93
|
-
"""Override string value of this release object with the date field.
|
101
|
+
"""Override string value of this release object with the `date` field.
|
94
102
|
|
95
103
|
:rtype: str
|
96
104
|
"""
|
97
105
|
return f"{self.date:%Y-%m-%d %H:%M:%S}"
|
98
106
|
|
99
107
|
@classmethod
|
100
|
-
def from_dt(
|
101
|
-
|
102
|
-
|
103
|
-
|
108
|
+
def from_dt(cls, dt: datetime | str) -> Self:
|
109
|
+
"""Construct Release object from `datetime` or `str` objects.
|
110
|
+
|
111
|
+
This method will replace second and millisecond value to 0 and
|
112
|
+
replace timezone to the `tz` config setting or extras overriding before
|
113
|
+
create Release object.
|
104
114
|
|
105
115
|
:param dt: (datetime | str) A datetime object or string that want to
|
106
116
|
construct to the Release object.
|
107
|
-
:param extras: (DictData) An extra parameters that want to pass to
|
108
|
-
override config values.
|
109
117
|
|
110
118
|
:raise TypeError: If the type of the dt argument does not valid with
|
111
119
|
datetime or str object.
|
@@ -116,20 +124,10 @@ class Release:
|
|
116
124
|
dt: datetime = datetime.fromisoformat(dt)
|
117
125
|
elif not isinstance(dt, datetime):
|
118
126
|
raise TypeError(
|
119
|
-
"The `from_dt` need the `dt`
|
120
|
-
"only."
|
127
|
+
f"The `from_dt` need the `dt` parameter type be `str` or "
|
128
|
+
f"`datetime` only, not {type(dt)}."
|
121
129
|
)
|
122
|
-
|
123
|
-
return cls(
|
124
|
-
date=dt,
|
125
|
-
offset=0,
|
126
|
-
end_date=dt + timedelta(days=1),
|
127
|
-
runner=(
|
128
|
-
CronJob("* * * * *").schedule(
|
129
|
-
dt.replace(tzinfo=dynamic("tz", extras=extras))
|
130
|
-
)
|
131
|
-
),
|
132
|
-
)
|
130
|
+
return cls(date=replace_sec(dt.replace(tzinfo=None)))
|
133
131
|
|
134
132
|
def __eq__(self, other: Release | datetime) -> bool:
|
135
133
|
"""Override equal property that will compare only the same type or
|
@@ -144,7 +142,7 @@ class Release:
|
|
144
142
|
return NotImplemented
|
145
143
|
|
146
144
|
def __lt__(self, other: Release | datetime) -> bool:
|
147
|
-
"""Override
|
145
|
+
"""Override less-than property that will compare only the same type or
|
148
146
|
datetime.
|
149
147
|
|
150
148
|
:rtype: bool
|
@@ -156,31 +154,33 @@ class Release:
|
|
156
154
|
return NotImplemented
|
157
155
|
|
158
156
|
|
159
|
-
@dataclass
|
160
157
|
class ReleaseQueue:
|
161
|
-
"""
|
158
|
+
"""ReleaseQueue object that is storage management of Release objects on
|
159
|
+
the memory with list object.
|
160
|
+
"""
|
162
161
|
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
162
|
+
def __init__(
|
163
|
+
self,
|
164
|
+
queue: Optional[list[Release]] = None,
|
165
|
+
running: Optional[list[Release]] = None,
|
166
|
+
complete: Optional[list[Release]] = None,
|
167
|
+
extras: Optional[DictData] = None,
|
168
|
+
):
|
169
|
+
self.queue: list[Release] = queue or []
|
170
|
+
self.running: list[Release] = running or []
|
171
|
+
self.complete: list[Release] = complete or []
|
172
|
+
self.extras: DictData = extras or {}
|
173
|
+
self.lock: Lock = Lock()
|
171
174
|
|
172
175
|
@classmethod
|
173
176
|
def from_list(
|
174
177
|
cls,
|
175
|
-
queue: list[datetime]
|
176
|
-
extras: Optional[DictData] = None,
|
178
|
+
queue: Optional[Union[list[datetime], list[Release]]] = None,
|
177
179
|
) -> Self:
|
178
180
|
"""Construct ReleaseQueue object from an input queue value that passing
|
179
181
|
with list of datetime or list of Release.
|
180
182
|
|
181
|
-
:param queue:
|
182
|
-
:param extras: An extra parameter that want to override core config
|
183
|
-
values.
|
183
|
+
:param queue: A queue object for create ReleaseQueue instance.
|
184
184
|
|
185
185
|
:raise TypeError: If the type of input queue does not valid.
|
186
186
|
|
@@ -190,13 +190,8 @@ class ReleaseQueue:
|
|
190
190
|
return cls()
|
191
191
|
|
192
192
|
if isinstance(queue, list):
|
193
|
-
|
194
193
|
if all(isinstance(q, datetime) for q in queue):
|
195
|
-
return cls(
|
196
|
-
queue=[
|
197
|
-
Release.from_dt(q, extras=(extras or {})) for q in queue
|
198
|
-
]
|
199
|
-
)
|
194
|
+
return cls(queue=[Release.from_dt(q) for q in queue])
|
200
195
|
|
201
196
|
if all(isinstance(q, Release) for q in queue):
|
202
197
|
return cls(queue=queue)
|
@@ -224,30 +219,39 @@ class ReleaseQueue:
|
|
224
219
|
:rtype: bool
|
225
220
|
"""
|
226
221
|
if isinstance(value, datetime):
|
227
|
-
value = Release.from_dt(value
|
222
|
+
value = Release.from_dt(value)
|
228
223
|
|
229
|
-
|
230
|
-
(
|
231
|
-
|
232
|
-
|
233
|
-
|
224
|
+
with self.lock:
|
225
|
+
return (
|
226
|
+
(value in self.queue)
|
227
|
+
or (value in self.running)
|
228
|
+
or (value in self.complete)
|
229
|
+
)
|
234
230
|
|
235
231
|
def mark_complete(self, value: Release) -> Self:
|
236
|
-
"""Push Release to the complete queue.
|
232
|
+
"""Push Release to the complete queue. After push the release, it will
|
233
|
+
delete old release base on the `CORE_MAX_QUEUE_COMPLETE_HIST` value.
|
234
|
+
|
235
|
+
:param value: (Release) A Release value that want to push to the
|
236
|
+
complete field.
|
237
237
|
|
238
238
|
:rtype: Self
|
239
239
|
"""
|
240
|
-
|
240
|
+
with self.lock:
|
241
|
+
if value in self.running:
|
242
|
+
self.running.remove(value)
|
241
243
|
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
|
246
|
-
|
244
|
+
heappush(self.complete, value)
|
245
|
+
|
246
|
+
# NOTE: Remove complete queue on workflow that keep more than the
|
247
|
+
# maximum config value.
|
248
|
+
num_complete_delete: int = len(self.complete) - dynamic(
|
249
|
+
"max_queue_complete_hist", extras=self.extras
|
250
|
+
)
|
247
251
|
|
248
|
-
|
249
|
-
|
250
|
-
|
252
|
+
if num_complete_delete > 0:
|
253
|
+
for _ in range(num_complete_delete):
|
254
|
+
heappop(self.complete)
|
251
255
|
|
252
256
|
return self
|
253
257
|
|
@@ -258,11 +262,10 @@ class ReleaseQueue:
|
|
258
262
|
runner: CronRunner,
|
259
263
|
name: str,
|
260
264
|
*,
|
261
|
-
offset: float = 0,
|
262
265
|
force_run: bool = False,
|
263
266
|
extras: Optional[DictData] = None,
|
264
267
|
) -> Self:
|
265
|
-
"""Generate Release model to queue.
|
268
|
+
"""Generate a Release model to the queue field with an input CronRunner.
|
266
269
|
|
267
270
|
Steps:
|
268
271
|
- Create Release object from the current date that not reach the end
|
@@ -274,50 +277,44 @@ class ReleaseQueue:
|
|
274
277
|
:param end_date: (datetime) An end datetime object.
|
275
278
|
:param audit: (type[Audit]) An audit class that want to make audit
|
276
279
|
instance.
|
277
|
-
:param runner: (CronRunner) A CronRunner object.
|
280
|
+
:param runner: (CronRunner) A `CronRunner` object.
|
278
281
|
:param name: (str) A target name that want to check at pointer of audit.
|
279
|
-
:param
|
280
|
-
|
281
|
-
|
282
|
-
|
282
|
+
:param force_run: (bool) A flag that allow to release workflow if the
|
283
|
+
audit with that release was pointed. (Default is False).
|
284
|
+
:param extras: (DictDatA) An extra parameter that want to override core
|
285
|
+
config values.
|
283
286
|
|
284
287
|
:rtype: ReleaseQueue
|
285
288
|
|
286
289
|
"""
|
287
|
-
if runner.date > end_date:
|
290
|
+
if clear_tz(runner.date) > clear_tz(end_date):
|
288
291
|
return self
|
289
292
|
|
290
|
-
|
291
|
-
date=runner.date,
|
292
|
-
|
293
|
-
end_date=end_date,
|
294
|
-
runner=runner,
|
295
|
-
type=ReleaseType.POKE,
|
293
|
+
release = Release(
|
294
|
+
date=clear_tz(runner.date),
|
295
|
+
type=(ReleaseType.FORCE if force_run else ReleaseType.POKING),
|
296
296
|
)
|
297
297
|
|
298
|
-
while self.check_queue(
|
299
|
-
audit.is_pointed(
|
300
|
-
name=name, release=workflow_release.date, extras=extras
|
301
|
-
)
|
298
|
+
while self.check_queue(release) or (
|
299
|
+
audit.is_pointed(name=name, release=release.date, extras=extras)
|
302
300
|
and not force_run
|
303
301
|
):
|
304
|
-
|
305
|
-
date=runner.next,
|
306
|
-
|
307
|
-
end_date=end_date,
|
308
|
-
runner=runner,
|
309
|
-
type=ReleaseType.POKE,
|
302
|
+
release = Release(
|
303
|
+
date=clear_tz(runner.next),
|
304
|
+
type=(ReleaseType.FORCE if force_run else ReleaseType.POKING),
|
310
305
|
)
|
311
306
|
|
312
|
-
if runner.date > end_date:
|
307
|
+
if clear_tz(runner.date) > clear_tz(end_date):
|
313
308
|
return self
|
314
309
|
|
315
|
-
|
310
|
+
with self.lock:
|
311
|
+
heappush(self.queue, release)
|
312
|
+
|
316
313
|
return self
|
317
314
|
|
318
315
|
|
319
316
|
class Workflow(BaseModel):
|
320
|
-
"""Workflow
|
317
|
+
"""Workflow model that use to keep the `Job` and `On` models.
|
321
318
|
|
322
319
|
This is the main future of this project because it uses to be workflow
|
323
320
|
data for running everywhere that you want or using it to scheduler task in
|
@@ -355,6 +352,7 @@ class Workflow(BaseModel):
|
|
355
352
|
cls,
|
356
353
|
name: str,
|
357
354
|
*,
|
355
|
+
path: Optional[Path] = None,
|
358
356
|
extras: DictData | None = None,
|
359
357
|
loader: type[Loader] = None,
|
360
358
|
) -> Self:
|
@@ -362,45 +360,8 @@ class Workflow(BaseModel):
|
|
362
360
|
an input workflow name. The loader object will use this workflow name to
|
363
361
|
searching configuration data of this workflow model in conf path.
|
364
362
|
|
365
|
-
:param name: A workflow name that want to pass to Loader object.
|
366
|
-
:param extras: An extra parameters that want to pass to Loader
|
367
|
-
object.
|
368
|
-
:param loader: A loader class for override default loader object.
|
369
|
-
|
370
|
-
:raise ValueError: If the type does not match with current object.
|
371
|
-
|
372
|
-
:rtype: Self
|
373
|
-
"""
|
374
|
-
loader: Loader = (loader or Loader)(name, externals=(extras or {}))
|
375
|
-
|
376
|
-
# NOTE: Validate the config type match with current connection model
|
377
|
-
if loader.type != cls.__name__:
|
378
|
-
raise ValueError(f"Type {loader.type} does not match with {cls}")
|
379
|
-
|
380
|
-
loader_data: DictData = copy.deepcopy(loader.data)
|
381
|
-
loader_data["name"] = name.replace(" ", "_")
|
382
|
-
|
383
|
-
if extras:
|
384
|
-
loader_data["extras"] = extras
|
385
|
-
|
386
|
-
cls.__bypass_on__(loader_data, path=loader.conf_path, extras=extras)
|
387
|
-
return cls.model_validate(obj=loader_data)
|
388
|
-
|
389
|
-
@classmethod
|
390
|
-
def from_path(
|
391
|
-
cls,
|
392
|
-
name: str,
|
393
|
-
path: Path,
|
394
|
-
*,
|
395
|
-
extras: DictData | None = None,
|
396
|
-
loader: type[Loader] = None,
|
397
|
-
) -> Self:
|
398
|
-
"""Create Workflow instance from the specific path. The loader object
|
399
|
-
will use this workflow name and path to searching configuration data of
|
400
|
-
this workflow model.
|
401
|
-
|
402
363
|
:param name: (str) A workflow name that want to pass to Loader object.
|
403
|
-
:param path: (Path)
|
364
|
+
:param path: (Path) An override config path.
|
404
365
|
:param extras: (DictData) An extra parameters that want to override core
|
405
366
|
config values.
|
406
367
|
:param loader: A loader class for override default loader object.
|
@@ -409,21 +370,21 @@ class Workflow(BaseModel):
|
|
409
370
|
|
410
371
|
:rtype: Self
|
411
372
|
"""
|
412
|
-
loader:
|
413
|
-
|
414
|
-
|
373
|
+
loader: type[Loader] = loader or FileLoad
|
374
|
+
load: Loader = loader(name, path=path, extras=extras)
|
375
|
+
|
415
376
|
# NOTE: Validate the config type match with current connection model
|
416
|
-
if
|
417
|
-
raise ValueError(f"Type {
|
377
|
+
if load.type != cls.__name__:
|
378
|
+
raise ValueError(f"Type {load.type} does not match with {cls}")
|
418
379
|
|
419
|
-
|
420
|
-
|
380
|
+
data: DictData = copy.deepcopy(load.data)
|
381
|
+
data["name"] = name
|
421
382
|
|
422
383
|
if extras:
|
423
|
-
|
384
|
+
data["extras"] = extras
|
424
385
|
|
425
|
-
cls.__bypass_on__(
|
426
|
-
return cls.model_validate(obj=
|
386
|
+
cls.__bypass_on__(data, path=load.path, extras=extras, loader=loader)
|
387
|
+
return cls.model_validate(obj=data)
|
427
388
|
|
428
389
|
@classmethod
|
429
390
|
def __bypass_on__(
|
@@ -431,6 +392,7 @@ class Workflow(BaseModel):
|
|
431
392
|
data: DictData,
|
432
393
|
path: Path,
|
433
394
|
extras: DictData | None = None,
|
395
|
+
loader: type[Loader] = None,
|
434
396
|
) -> DictData:
|
435
397
|
"""Bypass the on data to loaded config data.
|
436
398
|
|
@@ -451,7 +413,7 @@ class Workflow(BaseModel):
|
|
451
413
|
# field.
|
452
414
|
data["on"] = [
|
453
415
|
(
|
454
|
-
|
416
|
+
(loader or FileLoad)(n, path=path, extras=extras).data
|
455
417
|
if isinstance(n, str)
|
456
418
|
else n
|
457
419
|
)
|
@@ -460,11 +422,10 @@ class Workflow(BaseModel):
|
|
460
422
|
return data
|
461
423
|
|
462
424
|
@model_validator(mode="before")
|
463
|
-
def __prepare_model_before__(cls,
|
425
|
+
def __prepare_model_before__(cls, data: Any) -> Any:
|
464
426
|
"""Prepare the params key in the data model before validating."""
|
465
|
-
|
466
|
-
|
467
|
-
values["params"] = {
|
427
|
+
if isinstance(data, dict) and (params := data.pop("params", {})):
|
428
|
+
data["params"] = {
|
468
429
|
p: (
|
469
430
|
{"type": params[p]}
|
470
431
|
if isinstance(params[p], str)
|
@@ -472,7 +433,7 @@ class Workflow(BaseModel):
|
|
472
433
|
)
|
473
434
|
for p in params
|
474
435
|
}
|
475
|
-
return
|
436
|
+
return data
|
476
437
|
|
477
438
|
@field_validator("desc", mode="after")
|
478
439
|
def __dedent_desc__(cls, value: str) -> str:
|
@@ -481,7 +442,7 @@ class Workflow(BaseModel):
|
|
481
442
|
:param value: A description string value that want to dedent.
|
482
443
|
:rtype: str
|
483
444
|
"""
|
484
|
-
return dedent(value)
|
445
|
+
return dedent(value.lstrip("\n"))
|
485
446
|
|
486
447
|
@field_validator("on", mode="after")
|
487
448
|
def __on_no_dup_and_reach_limit__(
|
@@ -511,6 +472,12 @@ class Workflow(BaseModel):
|
|
511
472
|
# "If it has every minute cronjob on value, it should have "
|
512
473
|
# "only one value in the on field."
|
513
474
|
# )
|
475
|
+
set_tz: set[str] = {on.tz for on in value}
|
476
|
+
if len(set_tz) > 1:
|
477
|
+
raise ValueError(
|
478
|
+
f"The on fields should not contain multiple timezone, "
|
479
|
+
f"{list[set_tz]}."
|
480
|
+
)
|
514
481
|
|
515
482
|
extras: Optional[DictData] = info.data.get("extras")
|
516
483
|
if len(set_ons) > (
|
@@ -632,9 +599,10 @@ class Workflow(BaseModel):
|
|
632
599
|
result: Optional[Result] = None,
|
633
600
|
timeout: int = 600,
|
634
601
|
) -> Result:
|
635
|
-
"""Release the workflow
|
636
|
-
|
637
|
-
|
602
|
+
"""Release the workflow which is executes workflow with writing audit
|
603
|
+
log tracking. The method is overriding parameter with the release
|
604
|
+
templating that include logical date (release date), execution date,
|
605
|
+
or running id to the params.
|
638
606
|
|
639
607
|
This method allow workflow use audit object to save the execution
|
640
608
|
result to audit destination like file audit to the local `./logs` path.
|
@@ -675,33 +643,32 @@ class Workflow(BaseModel):
|
|
675
643
|
)
|
676
644
|
|
677
645
|
# VALIDATE: check type of queue that valid with ReleaseQueue.
|
678
|
-
if queue and not isinstance(queue, ReleaseQueue):
|
646
|
+
if queue is not None and not isinstance(queue, ReleaseQueue):
|
679
647
|
raise TypeError(
|
680
648
|
"The queue argument should be ReleaseQueue object only."
|
681
649
|
)
|
682
650
|
|
683
651
|
# VALIDATE: Change release value to Release object.
|
684
652
|
if isinstance(release, datetime):
|
685
|
-
release: Release = Release.from_dt(release
|
653
|
+
release: Release = Release.from_dt(release)
|
686
654
|
|
687
655
|
result.trace.info(
|
688
656
|
f"[RELEASE]: Start {name!r} : {release.date:%Y-%m-%d %H:%M:%S}"
|
689
657
|
)
|
658
|
+
tz: ZoneInfo = dynamic("tz", extras=self.extras)
|
659
|
+
values: DictData = param2template(
|
660
|
+
params,
|
661
|
+
params={
|
662
|
+
"release": {
|
663
|
+
"logical_date": release.date,
|
664
|
+
"execute_date": datetime.now(tz=tz),
|
665
|
+
"run_id": result.run_id,
|
666
|
+
}
|
667
|
+
},
|
668
|
+
extras=self.extras,
|
669
|
+
)
|
690
670
|
rs: Result = self.execute(
|
691
|
-
params=
|
692
|
-
params,
|
693
|
-
params={
|
694
|
-
"release": {
|
695
|
-
"logical_date": release.date,
|
696
|
-
"execute_date": datetime.now(
|
697
|
-
tz=dynamic("tz", extras=self.extras)
|
698
|
-
),
|
699
|
-
"run_id": result.run_id,
|
700
|
-
"timezone": dynamic("tz", extras=self.extras),
|
701
|
-
}
|
702
|
-
},
|
703
|
-
extras=self.extras,
|
704
|
-
),
|
671
|
+
params=values,
|
705
672
|
result=result,
|
706
673
|
parent_run_id=result.parent_run_id,
|
707
674
|
timeout=timeout,
|
@@ -709,10 +676,7 @@ class Workflow(BaseModel):
|
|
709
676
|
result.trace.info(
|
710
677
|
f"[RELEASE]: End {name!r} : {release.date:%Y-%m-%d %H:%M:%S}"
|
711
678
|
)
|
712
|
-
|
713
|
-
# NOTE: Saving execution result to destination of the input audit
|
714
|
-
# object.
|
715
|
-
result.trace.debug(f"[LOG]: Writing audit: {name!r}.")
|
679
|
+
result.trace.debug(f"[RELEASE]: Writing audit: {name!r}.")
|
716
680
|
(
|
717
681
|
audit(
|
718
682
|
name=name,
|
@@ -727,8 +691,6 @@ class Workflow(BaseModel):
|
|
727
691
|
)
|
728
692
|
|
729
693
|
if queue:
|
730
|
-
if release in queue.running:
|
731
|
-
queue.running.remove(release)
|
732
694
|
queue.mark_complete(release)
|
733
695
|
|
734
696
|
return result.catch(
|
@@ -738,9 +700,13 @@ class Workflow(BaseModel):
|
|
738
700
|
"release": {
|
739
701
|
"type": release.type,
|
740
702
|
"logical_date": release.date,
|
741
|
-
"release": release,
|
742
703
|
},
|
743
|
-
|
704
|
+
**{"jobs": result.context.pop("jobs", {})},
|
705
|
+
**(
|
706
|
+
result.context["errors"]
|
707
|
+
if "errors" in result.context
|
708
|
+
else {}
|
709
|
+
),
|
744
710
|
},
|
745
711
|
)
|
746
712
|
|
@@ -770,13 +736,8 @@ class Workflow(BaseModel):
|
|
770
736
|
queue.gen(
|
771
737
|
end_date,
|
772
738
|
audit,
|
773
|
-
on.next(
|
774
|
-
get_dt_now(
|
775
|
-
tz=dynamic("tz", extras=self.extras), offset=offset
|
776
|
-
).replace(microsecond=0)
|
777
|
-
),
|
739
|
+
on.next(get_dt_now(offset=offset).replace(microsecond=0)),
|
778
740
|
self.name,
|
779
|
-
offset=offset,
|
780
741
|
force_run=force_run,
|
781
742
|
)
|
782
743
|
|
@@ -784,36 +745,41 @@ class Workflow(BaseModel):
|
|
784
745
|
|
785
746
|
def poke(
|
786
747
|
self,
|
787
|
-
start_date: datetime | None = None,
|
788
748
|
params: DictData | None = None,
|
749
|
+
start_date: datetime | None = None,
|
789
750
|
*,
|
790
751
|
run_id: str | None = None,
|
791
752
|
periods: int = 1,
|
792
753
|
audit: Audit | None = None,
|
793
754
|
force_run: bool = False,
|
794
755
|
timeout: int = 1800,
|
795
|
-
max_poking_pool_worker: int =
|
756
|
+
max_poking_pool_worker: int = 2,
|
796
757
|
) -> Result:
|
797
|
-
"""Poke
|
758
|
+
"""Poke workflow with a start datetime value that will pass to its
|
798
759
|
`on` field on the threading executor pool for execute the `release`
|
799
760
|
method (It run all schedules that was set on the `on` values).
|
800
761
|
|
801
|
-
This method will observe its
|
762
|
+
This method will observe its `on` field that nearing to run with the
|
802
763
|
`self.release()` method.
|
803
764
|
|
804
|
-
The limitation of this method is not allow run a date that
|
765
|
+
The limitation of this method is not allow run a date that gather
|
805
766
|
than the current date.
|
806
767
|
|
807
|
-
:param
|
808
|
-
:param
|
809
|
-
:param run_id: A workflow running ID for this poke.
|
810
|
-
:param periods: A periods in minutes value that use to run this
|
811
|
-
|
812
|
-
:param
|
813
|
-
|
814
|
-
:param
|
815
|
-
|
816
|
-
:param
|
768
|
+
:param params: (DictData) A parameter data.
|
769
|
+
:param start_date: (datetime) A start datetime object.
|
770
|
+
:param run_id: (str) A workflow running ID for this poke.
|
771
|
+
:param periods: (int) A periods in minutes value that use to run this
|
772
|
+
poking. (Default is 1)
|
773
|
+
:param audit: (Audit) An audit object that want to use on this poking
|
774
|
+
process.
|
775
|
+
:param force_run: (bool) A flag that allow to release workflow if the
|
776
|
+
audit with that release was pointed. (Default is False)
|
777
|
+
:param timeout: (int) A second value for timeout while waiting all
|
778
|
+
futures run completely.
|
779
|
+
:param max_poking_pool_worker: (int) The maximum poking pool worker.
|
780
|
+
(Default is 2 workers)
|
781
|
+
|
782
|
+
:raise WorkflowException: If the periods parameter less or equal than 0.
|
817
783
|
|
818
784
|
:rtype: Result
|
819
785
|
:return: A list of all results that return from `self.release` method.
|
@@ -826,45 +792,49 @@ class Workflow(BaseModel):
|
|
826
792
|
# VALIDATE: Check the periods value should gather than 0.
|
827
793
|
if periods <= 0:
|
828
794
|
raise WorkflowException(
|
829
|
-
"The period of poking should be int and grater or equal
|
795
|
+
"The period of poking should be `int` and grater or equal "
|
796
|
+
"than 1."
|
830
797
|
)
|
831
798
|
|
832
799
|
if len(self.on) == 0:
|
833
|
-
result.trace.
|
834
|
-
f"[POKING]: {self.name!r}
|
800
|
+
result.trace.warning(
|
801
|
+
f"[POKING]: {self.name!r} not have any schedule!!!"
|
835
802
|
)
|
836
803
|
return result.catch(status=SUCCESS, context={"outputs": []})
|
837
804
|
|
838
805
|
# NOTE: Create the current date that change microsecond to 0
|
839
|
-
current_date: datetime = datetime.now(
|
840
|
-
|
841
|
-
|
842
|
-
|
843
|
-
# NOTE: Create start_date and offset variables.
|
844
|
-
if start_date and start_date <= current_date:
|
845
|
-
start_date = start_date.replace(
|
846
|
-
tzinfo=dynamic("tz", extras=self.extras)
|
847
|
-
).replace(microsecond=0)
|
848
|
-
offset: float = (current_date - start_date).total_seconds()
|
849
|
-
else:
|
806
|
+
current_date: datetime = datetime.now().replace(microsecond=0)
|
807
|
+
|
808
|
+
if start_date is None:
|
850
809
|
# NOTE: Force change start date if it gathers than the current date,
|
851
810
|
# or it does not pass to this method.
|
852
811
|
start_date: datetime = current_date
|
853
812
|
offset: float = 0
|
813
|
+
elif start_date <= current_date:
|
814
|
+
start_date = start_date.replace(microsecond=0)
|
815
|
+
offset: float = (current_date - start_date).total_seconds()
|
816
|
+
else:
|
817
|
+
raise WorkflowException(
|
818
|
+
f"The start datetime should less than or equal the current "
|
819
|
+
f"datetime, {current_date:%Y-%m-%d %H:%M:%S}."
|
820
|
+
)
|
854
821
|
|
855
822
|
# NOTE: The end date is using to stop generate queue with an input
|
856
|
-
# periods value.
|
857
|
-
|
823
|
+
# periods value. It will change to MM:59.
|
824
|
+
# For example:
|
825
|
+
# (input) start_date = 12:04:12, offset = 2
|
826
|
+
# (output) end_date = 12:06:59
|
827
|
+
end_date: datetime = start_date.replace(second=0) + timedelta(
|
828
|
+
minutes=periods + 1, seconds=-1
|
829
|
+
)
|
858
830
|
|
859
831
|
result.trace.info(
|
860
|
-
f"[POKING]:
|
861
|
-
f"{start_date:%Y-%m-%d %H:%M:%S}
|
832
|
+
f"[POKING]: Execute Poking: {self.name!r} ("
|
833
|
+
f"{start_date:%Y-%m-%d %H:%M:%S} ==> {end_date:%Y-%m-%d %H:%M:%S})"
|
862
834
|
)
|
863
835
|
|
864
836
|
params: DictData = {} if params is None else params
|
865
837
|
context: list[Result] = []
|
866
|
-
|
867
|
-
# NOTE: Create empty ReleaseQueue object.
|
868
838
|
q: ReleaseQueue = ReleaseQueue()
|
869
839
|
|
870
840
|
# NOTE: Create reusable partial function and add Release to the release
|
@@ -873,16 +843,12 @@ class Workflow(BaseModel):
|
|
873
843
|
self.queue, offset, end_date, audit=audit, force_run=force_run
|
874
844
|
)
|
875
845
|
partial_queue(q)
|
876
|
-
|
877
|
-
# NOTE: Return the empty result if it does not have any Release.
|
878
846
|
if not q.is_queued:
|
879
|
-
result.trace.
|
880
|
-
f"[POKING]: {self.name!r}
|
847
|
+
result.trace.warning(
|
848
|
+
f"[POKING]: Skip {self.name!r}, not have any queue!!!"
|
881
849
|
)
|
882
850
|
return result.catch(status=SUCCESS, context={"outputs": []})
|
883
851
|
|
884
|
-
# NOTE: Start create the thread pool executor for running this poke
|
885
|
-
# process.
|
886
852
|
with ThreadPoolExecutor(
|
887
853
|
max_workers=dynamic(
|
888
854
|
"max_poking_pool_worker",
|
@@ -901,16 +867,11 @@ class Workflow(BaseModel):
|
|
901
867
|
|
902
868
|
if reach_next_minute(release.date, offset=offset):
|
903
869
|
result.trace.debug(
|
904
|
-
f"[POKING]:
|
905
|
-
f"{release.date:%Y-%m-%d %H:%M:%S}
|
906
|
-
f"this time"
|
870
|
+
f"[POKING]: Skip Release: "
|
871
|
+
f"{release.date:%Y-%m-%d %H:%M:%S}"
|
907
872
|
)
|
908
873
|
heappush(q.queue, release)
|
909
|
-
|
910
|
-
get_dt_now(
|
911
|
-
tz=dynamic("tz", extras=self.extras), offset=offset
|
912
|
-
)
|
913
|
-
)
|
874
|
+
wait_until_next_minute(get_dt_now(offset=offset))
|
914
875
|
|
915
876
|
# WARNING: I already call queue poking again because issue
|
916
877
|
# about the every minute crontab.
|
@@ -943,7 +904,7 @@ class Workflow(BaseModel):
|
|
943
904
|
|
944
905
|
def execute_job(
|
945
906
|
self,
|
946
|
-
|
907
|
+
job: Job,
|
947
908
|
params: DictData,
|
948
909
|
*,
|
949
910
|
result: Result | None = None,
|
@@ -956,10 +917,9 @@ class Workflow(BaseModel):
|
|
956
917
|
model. It different with `self.execute` because this method run only
|
957
918
|
one job and return with context of this job data.
|
958
919
|
|
959
|
-
:raise WorkflowException: If execute with not exist job's ID.
|
960
920
|
:raise WorkflowException: If the job execution raise JobException.
|
961
921
|
|
962
|
-
:param
|
922
|
+
:param job: (Job) A job model that want to execute.
|
963
923
|
:param params: (DictData) A parameter data.
|
964
924
|
:param result: (Result) A Result instance for return context and status.
|
965
925
|
:param event: (Event) An Event manager instance that use to cancel this
|
@@ -967,48 +927,37 @@ class Workflow(BaseModel):
|
|
967
927
|
|
968
928
|
:rtype: Result
|
969
929
|
"""
|
970
|
-
|
971
|
-
result: Result = Result(run_id=gen_id(self.name, unique=True))
|
930
|
+
result: Result = result or Result(run_id=gen_id(self.name, unique=True))
|
972
931
|
|
973
|
-
# VALIDATE: check a job ID that exists in this workflow or not.
|
974
|
-
if job_id not in self.jobs:
|
975
|
-
raise WorkflowException(
|
976
|
-
f"The job: {job_id!r} does not exists in {self.name!r} "
|
977
|
-
f"workflow."
|
978
|
-
)
|
979
|
-
|
980
|
-
job: Job = self.job(name=job_id)
|
981
932
|
if job.is_skipped(params=params):
|
982
|
-
result.trace.info(f"[WORKFLOW]: Skip
|
933
|
+
result.trace.info(f"[WORKFLOW]: Skip Job: {job.id!r}")
|
983
934
|
job.set_outputs(output={"skipped": True}, to=params)
|
984
935
|
return result.catch(status=SKIP, context=params)
|
985
936
|
|
986
|
-
if event and event.is_set():
|
987
|
-
|
988
|
-
|
989
|
-
|
937
|
+
if event and event.is_set():
|
938
|
+
return result.catch(
|
939
|
+
status=CANCEL,
|
940
|
+
context={
|
941
|
+
"errors": WorkflowException(
|
942
|
+
"Workflow job was canceled because event was set."
|
943
|
+
).to_dict(),
|
944
|
+
},
|
990
945
|
)
|
991
946
|
|
992
|
-
|
993
|
-
|
994
|
-
|
995
|
-
|
996
|
-
|
997
|
-
|
998
|
-
|
947
|
+
result.trace.info(f"[WORKFLOW]: Execute Job: {job.id!r}")
|
948
|
+
rs: Result = job.execute(
|
949
|
+
params=params,
|
950
|
+
run_id=result.run_id,
|
951
|
+
parent_run_id=result.parent_run_id,
|
952
|
+
event=event,
|
953
|
+
)
|
954
|
+
job.set_outputs(rs.context, to=params)
|
955
|
+
if rs.status in (FAILED, CANCEL):
|
956
|
+
error_msg: str = (
|
957
|
+
f"Job, {job.id!r}, return `{rs.status.name}` status."
|
999
958
|
)
|
1000
|
-
job.set_outputs(rs.context, to=params)
|
1001
|
-
except (JobException, UtilException) as e:
|
1002
|
-
result.trace.error(f"[WORKFLOW]: {e.__class__.__name__}: {e}")
|
1003
|
-
raise WorkflowException(
|
1004
|
-
f"Job {job_id!r} raise {e.__class__.__name__}: {e}"
|
1005
|
-
) from None
|
1006
|
-
|
1007
|
-
if rs.status == FAILED:
|
1008
|
-
error_msg: str = f"Workflow job, {job.id!r}, return FAILED status."
|
1009
|
-
result.trace.warning(f"[WORKFLOW]: {error_msg}")
|
1010
959
|
return result.catch(
|
1011
|
-
status=
|
960
|
+
status=rs.status,
|
1012
961
|
context={
|
1013
962
|
"errors": WorkflowException(error_msg).to_dict(),
|
1014
963
|
**params,
|
@@ -1073,7 +1022,7 @@ class Workflow(BaseModel):
|
|
1073
1022
|
extras=self.extras,
|
1074
1023
|
)
|
1075
1024
|
context: DictData = self.parameterize(params)
|
1076
|
-
result.trace.info(f"[WORKFLOW]: Execute: {self.name!r}
|
1025
|
+
result.trace.info(f"[WORKFLOW]: Execute: {self.name!r}")
|
1077
1026
|
if not self.jobs:
|
1078
1027
|
result.trace.warning(f"[WORKFLOW]: {self.name!r} does not set jobs")
|
1079
1028
|
return result.catch(status=SUCCESS, context=context)
|
@@ -1087,7 +1036,9 @@ class Workflow(BaseModel):
|
|
1087
1036
|
"max_job_exec_timeout", f=timeout, extras=self.extras
|
1088
1037
|
)
|
1089
1038
|
event: Event = event or Event()
|
1090
|
-
result.trace.debug(
|
1039
|
+
result.trace.debug(
|
1040
|
+
f"[WORKFLOW]: ... Run {self.name!r} with non-threading."
|
1041
|
+
)
|
1091
1042
|
max_job_parallel: int = dynamic(
|
1092
1043
|
"max_job_parallel", f=max_job_parallel, extras=self.extras
|
1093
1044
|
)
|
@@ -1127,7 +1078,7 @@ class Workflow(BaseModel):
|
|
1127
1078
|
futures.append(
|
1128
1079
|
executor.submit(
|
1129
1080
|
self.execute_job,
|
1130
|
-
|
1081
|
+
job=job,
|
1131
1082
|
params=context,
|
1132
1083
|
result=result,
|
1133
1084
|
event=event,
|
@@ -1140,7 +1091,7 @@ class Workflow(BaseModel):
|
|
1140
1091
|
futures.append(
|
1141
1092
|
executor.submit(
|
1142
1093
|
self.execute_job,
|
1143
|
-
|
1094
|
+
job=job,
|
1144
1095
|
params=context,
|
1145
1096
|
result=result,
|
1146
1097
|
event=event,
|
@@ -1148,9 +1099,6 @@ class Workflow(BaseModel):
|
|
1148
1099
|
)
|
1149
1100
|
time.sleep(0.025)
|
1150
1101
|
elif (future := futures.pop(0)).done() or future.cancelled():
|
1151
|
-
if e := future.exception():
|
1152
|
-
result.trace.error(f"[WORKFLOW]: {e}")
|
1153
|
-
raise WorkflowException(str(e))
|
1154
1102
|
job_queue.put(job_id)
|
1155
1103
|
elif future.running() or "state=pending" in str(future):
|
1156
1104
|
time.sleep(0.075)
|
@@ -1159,8 +1107,10 @@ class Workflow(BaseModel):
|
|
1159
1107
|
else: # pragma: no cov
|
1160
1108
|
job_queue.put(job_id)
|
1161
1109
|
futures.insert(0, future)
|
1110
|
+
time.sleep(0.025)
|
1162
1111
|
result.trace.warning(
|
1163
|
-
f"... Execution non-threading not
|
1112
|
+
f"[WORKFLOW]: ... Execution non-threading not "
|
1113
|
+
f"handle: {future}."
|
1164
1114
|
)
|
1165
1115
|
|
1166
1116
|
job_queue.task_done()
|
@@ -1168,16 +1118,7 @@ class Workflow(BaseModel):
|
|
1168
1118
|
if not_timeout_flag:
|
1169
1119
|
job_queue.join()
|
1170
1120
|
for future in as_completed(futures):
|
1171
|
-
|
1172
|
-
future.result()
|
1173
|
-
except WorkflowException as e:
|
1174
|
-
result.trace.error(f"[WORKFLOW]: Handler:{NEWLINE}{e}")
|
1175
|
-
return result.catch(
|
1176
|
-
status=FAILED,
|
1177
|
-
context={
|
1178
|
-
"errors": WorkflowException(str(e)).to_dict()
|
1179
|
-
},
|
1180
|
-
)
|
1121
|
+
future.result()
|
1181
1122
|
return result.catch(
|
1182
1123
|
status=FAILED if "errors" in result.context else SUCCESS,
|
1183
1124
|
context=context,
|