ddeutil-workflow 0.0.40__py3-none-any.whl → 0.0.42__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ddeutil/workflow/__about__.py +1 -1
- ddeutil/workflow/__init__.py +21 -25
- ddeutil/workflow/api/api.py +8 -8
- ddeutil/workflow/api/routes/logs.py +1 -2
- ddeutil/workflow/api/routes/schedules.py +5 -5
- ddeutil/workflow/api/routes/workflows.py +3 -3
- ddeutil/workflow/conf.py +50 -29
- ddeutil/workflow/cron.py +12 -13
- ddeutil/workflow/job.py +68 -9
- ddeutil/workflow/logs.py +358 -69
- ddeutil/workflow/params.py +1 -0
- ddeutil/workflow/result.py +6 -15
- ddeutil/workflow/{templates.py → reusables.py} +199 -10
- ddeutil/workflow/scheduler.py +27 -29
- ddeutil/workflow/stages.py +423 -64
- ddeutil/workflow/utils.py +10 -0
- ddeutil/workflow/workflow.py +119 -74
- {ddeutil_workflow-0.0.40.dist-info → ddeutil_workflow-0.0.42.dist-info}/METADATA +12 -9
- ddeutil_workflow-0.0.42.dist-info/RECORD +30 -0
- ddeutil/workflow/audit.py +0 -257
- ddeutil/workflow/caller.py +0 -179
- ddeutil/workflow/context.py +0 -61
- ddeutil_workflow-0.0.40.dist-info/RECORD +0 -33
- {ddeutil_workflow-0.0.40.dist-info → ddeutil_workflow-0.0.42.dist-info}/WHEEL +0 -0
- {ddeutil_workflow-0.0.40.dist-info → ddeutil_workflow-0.0.42.dist-info}/licenses/LICENSE +0 -0
- {ddeutil_workflow-0.0.40.dist-info → ddeutil_workflow-0.0.42.dist-info}/top_level.txt +0 -0
ddeutil/workflow/utils.py
CHANGED
@@ -3,6 +3,7 @@
|
|
3
3
|
# Licensed under the MIT License. See LICENSE in the project root for
|
4
4
|
# license information.
|
5
5
|
# ------------------------------------------------------------------------------
|
6
|
+
"""Utility function model."""
|
6
7
|
from __future__ import annotations
|
7
8
|
|
8
9
|
import stat
|
@@ -154,6 +155,15 @@ def gen_id(
|
|
154
155
|
).hexdigest()
|
155
156
|
|
156
157
|
|
158
|
+
def default_gen_id() -> str:
|
159
|
+
"""Return running ID which use for making default ID for the Result model if
|
160
|
+
a run_id field initializes at the first time.
|
161
|
+
|
162
|
+
:rtype: str
|
163
|
+
"""
|
164
|
+
return gen_id("manual", unique=True)
|
165
|
+
|
166
|
+
|
157
167
|
def make_exec(path: str | Path) -> None:
|
158
168
|
"""Change mode of file to be executable file.
|
159
169
|
|
ddeutil/workflow/workflow.py
CHANGED
@@ -3,6 +3,7 @@
|
|
3
3
|
# Licensed under the MIT License. See LICENSE in the project root for
|
4
4
|
# license information.
|
5
5
|
# ------------------------------------------------------------------------------
|
6
|
+
# [x] Use dynamic config`
|
6
7
|
"""A Workflow module that is the core model of this package."""
|
7
8
|
from __future__ import annotations
|
8
9
|
|
@@ -24,21 +25,21 @@ from textwrap import dedent
|
|
24
25
|
from threading import Event
|
25
26
|
from typing import Optional
|
26
27
|
|
27
|
-
from pydantic import BaseModel, ConfigDict, Field
|
28
|
+
from pydantic import BaseModel, ConfigDict, Field, ValidationInfo
|
28
29
|
from pydantic.dataclasses import dataclass
|
29
30
|
from pydantic.functional_validators import field_validator, model_validator
|
30
31
|
from typing_extensions import Self
|
31
32
|
|
32
33
|
from .__cron import CronJob, CronRunner
|
33
34
|
from .__types import DictData, TupleStr
|
34
|
-
from .
|
35
|
-
from .conf import Loader, SimLoad, config, get_logger
|
35
|
+
from .conf import Loader, SimLoad, dynamic, get_logger
|
36
36
|
from .cron import On
|
37
37
|
from .exceptions import JobException, WorkflowException
|
38
38
|
from .job import Job, TriggerState
|
39
|
+
from .logs import Audit, get_audit
|
39
40
|
from .params import Param
|
40
41
|
from .result import Result, Status
|
41
|
-
from .
|
42
|
+
from .reusables import has_template, param2template
|
42
43
|
from .utils import (
|
43
44
|
gen_id,
|
44
45
|
get_dt_now,
|
@@ -92,11 +93,15 @@ class Release:
|
|
92
93
|
return f"{self.date:%Y-%m-%d %H:%M:%S}"
|
93
94
|
|
94
95
|
@classmethod
|
95
|
-
def from_dt(
|
96
|
+
def from_dt(
|
97
|
+
cls, dt: datetime | str, *, externals: Optional[DictData] = None
|
98
|
+
) -> Self:
|
96
99
|
"""Construct Release via datetime object only.
|
97
100
|
|
98
101
|
:param dt: (datetime | str) A datetime object or string that want to
|
99
102
|
construct to the Release object.
|
103
|
+
:param externals: An external parameters that want to pass to override
|
104
|
+
config.
|
100
105
|
|
101
106
|
:raise TypeError: If the type of the dt argument does not valid with
|
102
107
|
datetime or str object.
|
@@ -115,7 +120,9 @@ class Release:
|
|
115
120
|
date=dt,
|
116
121
|
offset=0,
|
117
122
|
end_date=dt + timedelta(days=1),
|
118
|
-
runner=CronJob("* * * * *").schedule(
|
123
|
+
runner=CronJob("* * * * *").schedule(
|
124
|
+
dt.replace(tzinfo=dynamic("tz", extras=externals))
|
125
|
+
),
|
119
126
|
)
|
120
127
|
|
121
128
|
def __eq__(self, other: Release | datetime) -> bool:
|
@@ -150,6 +157,10 @@ class ReleaseQueue:
|
|
150
157
|
queue: list[Release] = field(default_factory=list)
|
151
158
|
running: list[Release] = field(default_factory=list)
|
152
159
|
complete: list[Release] = field(default_factory=list)
|
160
|
+
extras: DictData = Field(
|
161
|
+
default_factory=dict,
|
162
|
+
description="An extra override config values.",
|
163
|
+
)
|
153
164
|
|
154
165
|
@classmethod
|
155
166
|
def from_list(
|
@@ -232,8 +243,8 @@ class ReleaseQueue:
|
|
232
243
|
|
233
244
|
# NOTE: Remove complete queue on workflow that keep more than the
|
234
245
|
# maximum config.
|
235
|
-
num_complete_delete: int = (
|
236
|
-
|
246
|
+
num_complete_delete: int = len(self.complete) - dynamic(
|
247
|
+
"max_queue_complete_hist", extras=self.extras
|
237
248
|
)
|
238
249
|
|
239
250
|
if num_complete_delete > 0:
|
@@ -252,6 +263,11 @@ class Workflow(BaseModel):
|
|
252
263
|
execute method on it.
|
253
264
|
"""
|
254
265
|
|
266
|
+
extras: DictData = Field(
|
267
|
+
default_factory=dict,
|
268
|
+
description="An extra override config values.",
|
269
|
+
)
|
270
|
+
|
255
271
|
name: str = Field(description="A workflow name.")
|
256
272
|
desc: Optional[str] = Field(
|
257
273
|
default=None,
|
@@ -273,36 +289,35 @@ class Workflow(BaseModel):
|
|
273
289
|
)
|
274
290
|
|
275
291
|
@classmethod
|
276
|
-
def
|
292
|
+
def from_conf(
|
277
293
|
cls,
|
278
294
|
name: str,
|
279
|
-
|
295
|
+
extras: DictData | None = None,
|
280
296
|
) -> Self:
|
281
297
|
"""Create Workflow instance from the Loader object that only receive
|
282
298
|
an input workflow name. The loader object will use this workflow name to
|
283
299
|
searching configuration data of this workflow model in conf path.
|
284
300
|
|
285
301
|
:param name: A workflow name that want to pass to Loader object.
|
286
|
-
:param
|
302
|
+
:param extras: An extra parameters that want to pass to Loader
|
287
303
|
object.
|
288
304
|
|
289
305
|
:raise ValueError: If the type does not match with current object.
|
290
306
|
|
291
307
|
:rtype: Self
|
292
308
|
"""
|
293
|
-
loader: Loader = Loader(name, externals=(
|
309
|
+
loader: Loader = Loader(name, externals=(extras or {}))
|
294
310
|
|
295
311
|
# NOTE: Validate the config type match with current connection model
|
296
312
|
if loader.type != cls.__name__:
|
297
313
|
raise ValueError(f"Type {loader.type} does not match with {cls}")
|
298
314
|
|
299
315
|
loader_data: DictData = copy.deepcopy(loader.data)
|
300
|
-
|
301
|
-
# NOTE: Add name to loader data
|
302
316
|
loader_data["name"] = name.replace(" ", "_")
|
303
|
-
|
304
|
-
loader_data
|
305
|
-
|
317
|
+
if extras: # pragma: no cov
|
318
|
+
loader_data["extras"] = extras
|
319
|
+
|
320
|
+
cls.__bypass_on__(loader_data, path=loader.conf_path, extras=extras)
|
306
321
|
return cls.model_validate(obj=loader_data)
|
307
322
|
|
308
323
|
@classmethod
|
@@ -310,7 +325,7 @@ class Workflow(BaseModel):
|
|
310
325
|
cls,
|
311
326
|
name: str,
|
312
327
|
path: Path,
|
313
|
-
|
328
|
+
extras: DictData | None = None,
|
314
329
|
) -> Self:
|
315
330
|
"""Create Workflow instance from the specific path. The loader object
|
316
331
|
will use this workflow name and path to searching configuration data of
|
@@ -318,7 +333,7 @@ class Workflow(BaseModel):
|
|
318
333
|
|
319
334
|
:param name: (str) A workflow name that want to pass to Loader object.
|
320
335
|
:param path: (Path) A config path that want to search.
|
321
|
-
:param
|
336
|
+
:param extras: An extra parameters that want to pass to Loader
|
322
337
|
object.
|
323
338
|
|
324
339
|
:raise ValueError: If the type does not match with current object.
|
@@ -326,17 +341,18 @@ class Workflow(BaseModel):
|
|
326
341
|
:rtype: Self
|
327
342
|
"""
|
328
343
|
loader: SimLoad = SimLoad(
|
329
|
-
name, conf_path=path, externals=(
|
344
|
+
name, conf_path=path, externals=(extras or {})
|
330
345
|
)
|
331
346
|
# NOTE: Validate the config type match with current connection model
|
332
347
|
if loader.type != cls.__name__:
|
333
348
|
raise ValueError(f"Type {loader.type} does not match with {cls}")
|
334
349
|
|
335
350
|
loader_data: DictData = copy.deepcopy(loader.data)
|
336
|
-
|
337
|
-
# NOTE: Add name to loader data
|
338
351
|
loader_data["name"] = name.replace(" ", "_")
|
339
|
-
|
352
|
+
if extras: # pragma: no cov
|
353
|
+
loader_data["extras"] = extras
|
354
|
+
|
355
|
+
cls.__bypass_on__(loader_data, path=path, extras=extras)
|
340
356
|
return cls.model_validate(obj=loader_data)
|
341
357
|
|
342
358
|
@classmethod
|
@@ -344,14 +360,15 @@ class Workflow(BaseModel):
|
|
344
360
|
cls,
|
345
361
|
data: DictData,
|
346
362
|
path: Path,
|
347
|
-
|
363
|
+
extras: DictData | None = None,
|
348
364
|
) -> DictData:
|
349
365
|
"""Bypass the on data to loaded config data.
|
350
366
|
|
351
367
|
:param data: A data to construct to this Workflow model.
|
352
368
|
:param path: A config path.
|
353
|
-
:param
|
369
|
+
:param extras: An extra parameters that want to pass to SimLoad
|
354
370
|
object.
|
371
|
+
|
355
372
|
:rtype: DictData
|
356
373
|
"""
|
357
374
|
if on := data.pop("on", []):
|
@@ -364,7 +381,7 @@ class Workflow(BaseModel):
|
|
364
381
|
# field.
|
365
382
|
data["on"] = [
|
366
383
|
(
|
367
|
-
SimLoad(n, conf_path=path, externals=(
|
384
|
+
SimLoad(n, conf_path=path, externals=(extras or {})).data
|
368
385
|
if isinstance(n, str)
|
369
386
|
else n
|
370
387
|
)
|
@@ -397,7 +414,11 @@ class Workflow(BaseModel):
|
|
397
414
|
return dedent(value)
|
398
415
|
|
399
416
|
@field_validator("on", mode="after")
|
400
|
-
def __on_no_dup_and_reach_limit__(
|
417
|
+
def __on_no_dup_and_reach_limit__(
|
418
|
+
cls,
|
419
|
+
value: list[On],
|
420
|
+
info: ValidationInfo,
|
421
|
+
) -> list[On]:
|
401
422
|
"""Validate the on fields should not contain duplicate values and if it
|
402
423
|
contains the every minute value more than one value, it will remove to
|
403
424
|
only one value.
|
@@ -421,10 +442,12 @@ class Workflow(BaseModel):
|
|
421
442
|
# "only one value in the on field."
|
422
443
|
# )
|
423
444
|
|
424
|
-
|
445
|
+
extras: DictData = info.data.get("extras", {})
|
446
|
+
if len(set_ons) > (
|
447
|
+
conf := dynamic("max_on_per_workflow", extras=extras)
|
448
|
+
):
|
425
449
|
raise ValueError(
|
426
|
-
f"The number of the on should not more than "
|
427
|
-
f"{config.max_on_per_workflow} crontab."
|
450
|
+
f"The number of the on should not more than {conf} crontabs."
|
428
451
|
)
|
429
452
|
return value
|
430
453
|
|
@@ -446,7 +469,6 @@ class Workflow(BaseModel):
|
|
446
469
|
f"{self.name!r}."
|
447
470
|
)
|
448
471
|
|
449
|
-
# NOTE: update a job id with its job id from workflow template
|
450
472
|
self.jobs[job].id = job
|
451
473
|
|
452
474
|
# VALIDATE: Validate workflow name should not dynamic with params
|
@@ -476,7 +498,10 @@ class Workflow(BaseModel):
|
|
476
498
|
f"A Job {name!r} does not exists in this workflow, "
|
477
499
|
f"{self.name!r}"
|
478
500
|
)
|
479
|
-
|
501
|
+
job: Job = self.jobs[name]
|
502
|
+
if self.extras:
|
503
|
+
job.extras = self.extras
|
504
|
+
return job
|
480
505
|
|
481
506
|
def parameterize(self, params: DictData) -> DictData:
|
482
507
|
"""Prepare a passing parameters before use it in execution process.
|
@@ -596,9 +621,11 @@ class Workflow(BaseModel):
|
|
596
621
|
release_params: DictData = {
|
597
622
|
"release": {
|
598
623
|
"logical_date": release.date,
|
599
|
-
"execute_date": datetime.now(
|
624
|
+
"execute_date": datetime.now(
|
625
|
+
tz=dynamic("tz", extras=self.extras)
|
626
|
+
),
|
600
627
|
"run_id": result.run_id,
|
601
|
-
"timezone":
|
628
|
+
"timezone": dynamic("tz", extras=self.extras),
|
602
629
|
}
|
603
630
|
}
|
604
631
|
|
@@ -608,7 +635,7 @@ class Workflow(BaseModel):
|
|
608
635
|
# ... {"params": ..., "jobs": ...}
|
609
636
|
#
|
610
637
|
self.execute(
|
611
|
-
params=param2template(params, release_params),
|
638
|
+
params=param2template(params, release_params, extras=self.extras),
|
612
639
|
result=result,
|
613
640
|
parent_run_id=result.parent_run_id,
|
614
641
|
)
|
@@ -648,7 +675,6 @@ class Workflow(BaseModel):
|
|
648
675
|
return result.catch(
|
649
676
|
status=Status.SUCCESS,
|
650
677
|
context={
|
651
|
-
# NOTE: Update the real params that pass in this method.
|
652
678
|
"params": params,
|
653
679
|
"release": {
|
654
680
|
"type": release.type,
|
@@ -692,10 +718,11 @@ class Workflow(BaseModel):
|
|
692
718
|
for on in self.on:
|
693
719
|
|
694
720
|
runner: CronRunner = on.next(
|
695
|
-
get_dt_now(
|
721
|
+
get_dt_now(
|
722
|
+
tz=dynamic("tz", extras=self.extras), offset=offset
|
723
|
+
).replace(microsecond=0)
|
696
724
|
)
|
697
725
|
|
698
|
-
# NOTE: Skip this runner date if it more than the end date.
|
699
726
|
if runner.date > end_date:
|
700
727
|
continue
|
701
728
|
|
@@ -722,7 +749,6 @@ class Workflow(BaseModel):
|
|
722
749
|
if runner.date > end_date:
|
723
750
|
continue
|
724
751
|
|
725
|
-
# NOTE: Push the Release object to queue.
|
726
752
|
heappush(queue.queue, workflow_release)
|
727
753
|
|
728
754
|
return queue
|
@@ -737,6 +763,7 @@ class Workflow(BaseModel):
|
|
737
763
|
audit: Audit | None = None,
|
738
764
|
force_run: bool = False,
|
739
765
|
timeout: int = 1800,
|
766
|
+
max_poking_pool_worker: int = 4,
|
740
767
|
) -> Result:
|
741
768
|
"""Poke function with a start datetime value that will pass to its
|
742
769
|
`on` field on the threading executor pool for execute the `release`
|
@@ -757,6 +784,7 @@ class Workflow(BaseModel):
|
|
757
784
|
that release was pointed.
|
758
785
|
:param timeout: A second value for timeout while waiting all futures
|
759
786
|
run completely.
|
787
|
+
:param max_poking_pool_worker: The maximum poking pool worker.
|
760
788
|
|
761
789
|
:rtype: Result
|
762
790
|
:return: A list of all results that return from `self.release` method.
|
@@ -772,8 +800,6 @@ class Workflow(BaseModel):
|
|
772
800
|
"The period of poking should be int and grater or equal than 1."
|
773
801
|
)
|
774
802
|
|
775
|
-
# NOTE: If this workflow does not set the on schedule, it will return
|
776
|
-
# empty result.
|
777
803
|
if len(self.on) == 0:
|
778
804
|
result.trace.info(
|
779
805
|
f"[POKING]: {self.name!r} does not have any schedule to run."
|
@@ -781,15 +807,15 @@ class Workflow(BaseModel):
|
|
781
807
|
return result.catch(status=Status.SUCCESS, context={"outputs": []})
|
782
808
|
|
783
809
|
# NOTE: Create the current date that change microsecond to 0
|
784
|
-
current_date: datetime = datetime.now(
|
785
|
-
|
786
|
-
)
|
810
|
+
current_date: datetime = datetime.now(
|
811
|
+
tz=dynamic("tz", extras=self.extras)
|
812
|
+
).replace(microsecond=0)
|
787
813
|
|
788
814
|
# NOTE: Create start_date and offset variables.
|
789
815
|
if start_date and start_date <= current_date:
|
790
|
-
start_date = start_date.replace(
|
791
|
-
|
792
|
-
)
|
816
|
+
start_date = start_date.replace(
|
817
|
+
tzinfo=dynamic("tz", extras=self.extras)
|
818
|
+
).replace(microsecond=0)
|
793
819
|
offset: float = (current_date - start_date).total_seconds()
|
794
820
|
else:
|
795
821
|
# NOTE: Force change start date if it gathers than the current date,
|
@@ -829,7 +855,11 @@ class Workflow(BaseModel):
|
|
829
855
|
# NOTE: Start create the thread pool executor for running this poke
|
830
856
|
# process.
|
831
857
|
with ThreadPoolExecutor(
|
832
|
-
max_workers=
|
858
|
+
max_workers=dynamic(
|
859
|
+
"max_poking_pool_worker",
|
860
|
+
f=max_poking_pool_worker,
|
861
|
+
extras=self.extras,
|
862
|
+
),
|
833
863
|
thread_name_prefix="wf_poking_",
|
834
864
|
) as executor:
|
835
865
|
|
@@ -840,14 +870,22 @@ class Workflow(BaseModel):
|
|
840
870
|
# NOTE: Pop the latest Release object from the release queue.
|
841
871
|
release: Release = heappop(q.queue)
|
842
872
|
|
843
|
-
if reach_next_minute(
|
873
|
+
if reach_next_minute(
|
874
|
+
release.date,
|
875
|
+
tz=dynamic("tz", extras=self.extras),
|
876
|
+
offset=offset,
|
877
|
+
):
|
844
878
|
result.trace.debug(
|
845
879
|
f"[POKING]: The latest release, "
|
846
880
|
f"{release.date:%Y-%m-%d %H:%M:%S}, is not able to run "
|
847
881
|
f"on this minute"
|
848
882
|
)
|
849
883
|
heappush(q.queue, release)
|
850
|
-
wait_to_next_minute(
|
884
|
+
wait_to_next_minute(
|
885
|
+
get_dt_now(
|
886
|
+
tz=dynamic("tz", extras=self.extras), offset=offset
|
887
|
+
)
|
888
|
+
)
|
851
889
|
|
852
890
|
# WARNING: I already call queue poking again because issue
|
853
891
|
# about the every minute crontab.
|
@@ -927,12 +965,6 @@ class Workflow(BaseModel):
|
|
927
965
|
"job execution."
|
928
966
|
)
|
929
967
|
|
930
|
-
# IMPORTANT:
|
931
|
-
# This execution change all job running IDs to the current workflow
|
932
|
-
# running ID, but it still trac log to the same parent running ID
|
933
|
-
# (with passing `run_id` and `parent_run_id` to the job execution
|
934
|
-
# arguments).
|
935
|
-
#
|
936
968
|
try:
|
937
969
|
job: Job = self.jobs[job_id]
|
938
970
|
if job.is_skipped(params=params):
|
@@ -967,8 +999,10 @@ class Workflow(BaseModel):
|
|
967
999
|
*,
|
968
1000
|
run_id: str | None = None,
|
969
1001
|
parent_run_id: str | None = None,
|
970
|
-
timeout: int =
|
1002
|
+
timeout: int = 600,
|
971
1003
|
result: Result | None = None,
|
1004
|
+
max_job_parallel: int = 2,
|
1005
|
+
event: Event | None = None,
|
972
1006
|
) -> Result:
|
973
1007
|
"""Execute workflow with passing a dynamic parameters to all jobs that
|
974
1008
|
included in this workflow model with ``jobs`` field.
|
@@ -993,6 +1027,9 @@ class Workflow(BaseModel):
|
|
993
1027
|
limit time. (default: 0)
|
994
1028
|
:param result: (Result) A result object for keeping context and status
|
995
1029
|
data.
|
1030
|
+
:param max_job_parallel: (int) The maximum threads of job execution.
|
1031
|
+
:param event: (Event) An event manager that pass to the
|
1032
|
+
PoolThreadExecutor.
|
996
1033
|
|
997
1034
|
:rtype: Result
|
998
1035
|
"""
|
@@ -1030,13 +1067,19 @@ class Workflow(BaseModel):
|
|
1030
1067
|
context: DictData = self.parameterize(params)
|
1031
1068
|
status: Status = Status.SUCCESS
|
1032
1069
|
try:
|
1033
|
-
if
|
1070
|
+
if (
|
1071
|
+
dynamic(
|
1072
|
+
"max_job_parallel", f=max_job_parallel, extras=self.extras
|
1073
|
+
)
|
1074
|
+
== 1
|
1075
|
+
):
|
1034
1076
|
self.__exec_non_threading(
|
1035
1077
|
result=result,
|
1036
1078
|
context=context,
|
1037
1079
|
ts=ts,
|
1038
1080
|
job_queue=jq,
|
1039
1081
|
timeout=timeout,
|
1082
|
+
event=event,
|
1040
1083
|
)
|
1041
1084
|
else:
|
1042
1085
|
self.__exec_threading(
|
@@ -1045,6 +1088,7 @@ class Workflow(BaseModel):
|
|
1045
1088
|
ts=ts,
|
1046
1089
|
job_queue=jq,
|
1047
1090
|
timeout=timeout,
|
1091
|
+
event=event,
|
1048
1092
|
)
|
1049
1093
|
except WorkflowException as err:
|
1050
1094
|
status = Status.FAILED
|
@@ -1059,8 +1103,9 @@ class Workflow(BaseModel):
|
|
1059
1103
|
ts: float,
|
1060
1104
|
job_queue: Queue,
|
1061
1105
|
*,
|
1062
|
-
timeout: int =
|
1106
|
+
timeout: int = 600,
|
1063
1107
|
thread_timeout: int = 1800,
|
1108
|
+
event: Event | None = None,
|
1064
1109
|
) -> DictData:
|
1065
1110
|
"""Workflow execution by threading strategy that use multithreading.
|
1066
1111
|
|
@@ -1074,18 +1119,19 @@ class Workflow(BaseModel):
|
|
1074
1119
|
:param job_queue: (Queue) A job queue object.
|
1075
1120
|
:param timeout: (int) A second value unit that bounding running time.
|
1076
1121
|
:param thread_timeout: A timeout to waiting all futures complete.
|
1122
|
+
:param event: (Event) An event manager that pass to the
|
1123
|
+
PoolThreadExecutor.
|
1077
1124
|
|
1078
1125
|
:rtype: DictData
|
1079
1126
|
"""
|
1080
1127
|
not_timeout_flag: bool = True
|
1081
|
-
timeout: int =
|
1082
|
-
|
1128
|
+
timeout: int = dynamic(
|
1129
|
+
"max_job_exec_timeout", f=timeout, extras=self.extras
|
1130
|
+
)
|
1131
|
+
event: Event = event or Event()
|
1083
1132
|
result.trace.debug(f"[WORKFLOW]: Run {self.name!r} with threading.")
|
1084
|
-
|
1085
|
-
# IMPORTANT: The job execution can run parallel and waiting by
|
1086
|
-
# needed.
|
1087
1133
|
with ThreadPoolExecutor(
|
1088
|
-
max_workers=
|
1134
|
+
max_workers=dynamic("max_job_parallel", extras=self.extras),
|
1089
1135
|
thread_name_prefix="wf_exec_threading_",
|
1090
1136
|
) as executor:
|
1091
1137
|
futures: list[Future] = []
|
@@ -1145,7 +1191,6 @@ class Workflow(BaseModel):
|
|
1145
1191
|
result.trace.error(f"[WORKFLOW]: {err}")
|
1146
1192
|
raise WorkflowException(str(err))
|
1147
1193
|
|
1148
|
-
# NOTE: This getting result does not do anything.
|
1149
1194
|
future.result()
|
1150
1195
|
|
1151
1196
|
return context
|
@@ -1166,7 +1211,8 @@ class Workflow(BaseModel):
|
|
1166
1211
|
ts: float,
|
1167
1212
|
job_queue: Queue,
|
1168
1213
|
*,
|
1169
|
-
timeout: int =
|
1214
|
+
timeout: int = 600,
|
1215
|
+
event: Event | None = None,
|
1170
1216
|
) -> DictData:
|
1171
1217
|
"""Workflow execution with non-threading strategy that use sequential
|
1172
1218
|
job running and waiting previous job was run successful.
|
@@ -1179,12 +1225,16 @@ class Workflow(BaseModel):
|
|
1179
1225
|
:param ts: (float) A start timestamp that use for checking execute time
|
1180
1226
|
should time out.
|
1181
1227
|
:param timeout: (int) A second value unit that bounding running time.
|
1228
|
+
:param event: (Event) An event manager that pass to the
|
1229
|
+
PoolThreadExecutor.
|
1182
1230
|
|
1183
1231
|
:rtype: DictData
|
1184
1232
|
"""
|
1185
1233
|
not_timeout_flag: bool = True
|
1186
|
-
timeout: int =
|
1187
|
-
|
1234
|
+
timeout: int = dynamic(
|
1235
|
+
"max_job_exec_timeout", f=timeout, extras=self.extras
|
1236
|
+
)
|
1237
|
+
event: Event = event or Event()
|
1188
1238
|
future: Future | None = None
|
1189
1239
|
result.trace.debug(f"[WORKFLOW]: Run {self.name!r} with non-threading.")
|
1190
1240
|
|
@@ -1199,7 +1249,6 @@ class Workflow(BaseModel):
|
|
1199
1249
|
job_id: str = job_queue.get()
|
1200
1250
|
job: Job = self.jobs[job_id]
|
1201
1251
|
|
1202
|
-
# NOTE: Waiting dependency job run successful before release.
|
1203
1252
|
if (check := job.check_needs(context["jobs"])).is_waiting():
|
1204
1253
|
job_queue.task_done()
|
1205
1254
|
job_queue.put(job_id)
|
@@ -1248,13 +1297,9 @@ class Workflow(BaseModel):
|
|
1248
1297
|
f"that not running."
|
1249
1298
|
)
|
1250
1299
|
|
1251
|
-
# NOTE: Mark this job queue done.
|
1252
1300
|
job_queue.task_done()
|
1253
1301
|
|
1254
1302
|
if not_timeout_flag:
|
1255
|
-
|
1256
|
-
# NOTE: Wait for all items to finish processing by `task_done()`
|
1257
|
-
# method.
|
1258
1303
|
job_queue.join()
|
1259
1304
|
executor.shutdown()
|
1260
1305
|
return context
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: ddeutil-workflow
|
3
|
-
Version: 0.0.
|
3
|
+
Version: 0.0.42
|
4
4
|
Summary: Lightweight workflow orchestration
|
5
5
|
Author-email: ddeutils <korawich.anu@gmail.com>
|
6
6
|
License: MIT
|
@@ -27,6 +27,12 @@ Requires-Dist: ddeutil-io[toml,yaml]>=0.2.10
|
|
27
27
|
Requires-Dist: pydantic==2.11.1
|
28
28
|
Requires-Dist: python-dotenv==1.1.0
|
29
29
|
Requires-Dist: schedule<2.0.0,==1.2.2
|
30
|
+
Provides-Extra: all
|
31
|
+
Requires-Dist: fastapi<1.0.0,>=0.115.0; extra == "all"
|
32
|
+
Requires-Dist: httpx; extra == "all"
|
33
|
+
Requires-Dist: ujson; extra == "all"
|
34
|
+
Requires-Dist: aiofiles; extra == "all"
|
35
|
+
Requires-Dist: aiohttp; extra == "all"
|
30
36
|
Provides-Extra: api
|
31
37
|
Requires-Dist: fastapi<1.0.0,>=0.115.0; extra == "api"
|
32
38
|
Requires-Dist: httpx; extra == "api"
|
@@ -258,7 +264,7 @@ it will use default value and do not raise any error to you.
|
|
258
264
|
| Name | Component | Default | Description |
|
259
265
|
|:-----------------------------|:---------:|:--------------------------------------------------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------|
|
260
266
|
| **ROOT_PATH** | Core | `.` | The root path of the workflow application. |
|
261
|
-
| **
|
267
|
+
| **REGISTRY_CALLER** | Core | `.` | List of importable string for the call stage. |
|
262
268
|
| **REGISTRY_FILTER** | Core | `ddeutil.workflow.templates` | List of importable string for the filter template. |
|
263
269
|
| **CONF_PATH** | Core | `conf` | The config path that keep all template `.yaml` files. |
|
264
270
|
| **TIMEZONE** | Core | `Asia/Bangkok` | A Timezone string value that will pass to `ZoneInfo` object. |
|
@@ -266,20 +272,17 @@ it will use default value and do not raise any error to you.
|
|
266
272
|
| **STAGE_RAISE_ERROR** | Core | `false` | A flag that all stage raise StageException from stage execution. |
|
267
273
|
| **JOB_DEFAULT_ID** | Core | `false` | A flag that enable default job ID that use for catch an execution output. The ID that use will be sequence number. |
|
268
274
|
| **JOB_RAISE_ERROR** | Core | `true` | A flag that all job raise JobException from job strategy execution. |
|
269
|
-
| **MAX_NUM_POKING** | Core | `4` | . |
|
270
|
-
| **MAX_JOB_PARALLEL** | Core | `2` | The maximum job number that able to run parallel in workflow executor. |
|
271
|
-
| **MAX_JOB_EXEC_TIMEOUT** | Core | `600` | |
|
272
275
|
| **MAX_CRON_PER_WORKFLOW** | Core | `5` | |
|
273
276
|
| **MAX_QUEUE_COMPLETE_HIST** | Core | `16` | |
|
274
277
|
| **GENERATE_ID_SIMPLE_MODE** | Core | `true` | A flog that enable generating ID with `md5` algorithm. |
|
275
|
-
| **
|
278
|
+
| **TRACE_PATH** | Log | `./logs` | The log path of the workflow saving log. |
|
276
279
|
| **DEBUG_MODE** | Log | `true` | A flag that enable logging with debug level mode. |
|
277
280
|
| **FORMAT** | Log | `%(asctime)s.%(msecs)03d (%(name)-10s, %(process)-5d,%(thread)-5d) [%(levelname)-7s] %(message)-120s (%(filename)s:%(lineno)s)` | |
|
278
281
|
| **FORMAT_FILE** | Log | `{datetime} ({process:5d}, {thread:5d}) {message:120s} ({filename}:{lineno})` | |
|
279
282
|
| **DATETIME_FORMAT** | Log | `%Y-%m-%d %H:%M:%S` | |
|
280
|
-
| **
|
281
|
-
| **
|
282
|
-
| **
|
283
|
+
| **TRACE_ENABLE_WRITE** | Log | `false` | |
|
284
|
+
| **AUDIT_PATH** | Log | `./audits` | |
|
285
|
+
| **AUDIT_ENABLE_WRITE** | Log | `true` | A flag that enable logging object saving log to its destination. |
|
283
286
|
| **MAX_PROCESS** | App | `2` | The maximum process worker number that run in scheduler app module. |
|
284
287
|
| **MAX_SCHEDULE_PER_PROCESS** | App | `100` | A schedule per process that run parallel. |
|
285
288
|
| **STOP_BOUNDARY_DELTA** | App | `'{"minutes": 5, "seconds": 20}'` | A time delta value that use to stop scheduler app in json string format. |
|
@@ -0,0 +1,30 @@
|
|
1
|
+
ddeutil/workflow/__about__.py,sha256=B3KiwXw9ATZmMG1vER6qdPImMLkQPPjYkRvYepuIhF4,28
|
2
|
+
ddeutil/workflow/__cron.py,sha256=h8rLeIUAAEB2SdZ4Jhch7LU1Yl3bbJ-iNNJ3tQ0eYVM,28095
|
3
|
+
ddeutil/workflow/__init__.py,sha256=cYWwG2utpsYvdwqvkFSRWi_Q6gylDgNQBcIWcF5NFs4,1861
|
4
|
+
ddeutil/workflow/__types.py,sha256=8jBdbfb3aZSetjz0mvNrpGHwwxJff7mK8_4v41cLqlc,4316
|
5
|
+
ddeutil/workflow/conf.py,sha256=lDzWiVSNlNAhTzxbNIhIbQAIF1ggbmetAp0yn2fgnsc,12385
|
6
|
+
ddeutil/workflow/cron.py,sha256=80SijzMdDOBxTWRsiF-Fmuz7Ym7leY0XT2lzRAPGdXc,8781
|
7
|
+
ddeutil/workflow/exceptions.py,sha256=fO37f9p7lOjIJgVOpKE_1X44yJTwBepyukZV9a7NNm4,1241
|
8
|
+
ddeutil/workflow/job.py,sha256=vsayKMzwKDpjchgYQnbshZHnp-vuM9CobpFWhUJETRU,30315
|
9
|
+
ddeutil/workflow/logs.py,sha256=RkM5o_JPoWhFY7NrbYAARZQWjLC62YB_FYzTTcyDp8U,19816
|
10
|
+
ddeutil/workflow/params.py,sha256=Mv-D2DY5inm1ug0lsgCPDkO5wT_AUhc5XEF5jxgDx6U,8036
|
11
|
+
ddeutil/workflow/result.py,sha256=ynZB0g_vEEXn24034J-hatjNWDBmRAj38S8SqGRM-8I,4029
|
12
|
+
ddeutil/workflow/reusables.py,sha256=AtZO83HDFu1uK_azUinv5d8jsA36f2i3n_tqMrolbvc,17529
|
13
|
+
ddeutil/workflow/scheduler.py,sha256=wFEgcnxtgF-8y5otv8RqT1MuBttZl7mu-bBu5ffwV_Y,27534
|
14
|
+
ddeutil/workflow/stages.py,sha256=prw1-za1zwYehbrjeAnoJ79GxpfTqdKLsI2PY0OuSlY,48417
|
15
|
+
ddeutil/workflow/utils.py,sha256=sblje9qOtejCHVt8EVrbC0KY98vKqvxccaR5HIkRiTA,7363
|
16
|
+
ddeutil/workflow/workflow.py,sha256=Y1D5arh2KSobkIZGJ1fWSTe15heURi9OhhdfIr0jHyo,50591
|
17
|
+
ddeutil/workflow/api/__init__.py,sha256=F53NMBWtb9IKaDWkPU5KvybGGfKAcbehgn6TLBwHuuM,21
|
18
|
+
ddeutil/workflow/api/api.py,sha256=b-bMg0aRsEqt8Qb2hNUtamEt2Fq2CgNotF2oXSAdDu8,5226
|
19
|
+
ddeutil/workflow/api/log.py,sha256=NMTnOnsBrDB5129329xF2myLdrb-z9k1MQrmrP7qXJw,1818
|
20
|
+
ddeutil/workflow/api/repeat.py,sha256=cycd1-91j-4v6uY1SkrZHd9l95e-YgVC4UCSNNFuGJ8,5277
|
21
|
+
ddeutil/workflow/api/routes/__init__.py,sha256=qoGtOMyVgQ5nTUc8J8wH27A8isaxl3IFCX8qoyibeCY,484
|
22
|
+
ddeutil/workflow/api/routes/job.py,sha256=YVta083i8vU8-o4WdKFwDpfdC9vN1dZ6goZSmNlQXHA,1954
|
23
|
+
ddeutil/workflow/api/routes/logs.py,sha256=TeRDrEelbKS2Hu_EovgLh0bOdmSv9mfnrIZsrE7uPD4,5353
|
24
|
+
ddeutil/workflow/api/routes/schedules.py,sha256=rUWBm5RgLS1PNBHSWwWXJ0l-c5mYWfl9os0BA9_OTEw,4810
|
25
|
+
ddeutil/workflow/api/routes/workflows.py,sha256=ctgQGxXfpIV6bHFDM9IQ1_qaQHT6n5-HjJ1-D4GKWpc,4527
|
26
|
+
ddeutil_workflow-0.0.42.dist-info/licenses/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
|
27
|
+
ddeutil_workflow-0.0.42.dist-info/METADATA,sha256=TJp1M40eLXYOInkTpl_XhFOWGHLd0hIHktXQXiFsmEw,18853
|
28
|
+
ddeutil_workflow-0.0.42.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
|
29
|
+
ddeutil_workflow-0.0.42.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
|
30
|
+
ddeutil_workflow-0.0.42.dist-info/RECORD,,
|