ddeutil-workflow 0.0.26.post1__py3-none-any.whl → 0.0.27__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ddeutil/workflow/__about__.py +1 -1
- ddeutil/workflow/__init__.py +19 -14
- ddeutil/workflow/conf.py +29 -12
- ddeutil/workflow/exceptions.py +0 -3
- ddeutil/workflow/hook.py +153 -0
- ddeutil/workflow/job.py +1 -1
- ddeutil/workflow/stage.py +3 -55
- ddeutil/workflow/templates.py +334 -0
- ddeutil/workflow/utils.py +3 -391
- ddeutil/workflow/workflow.py +2 -3
- {ddeutil_workflow-0.0.26.post1.dist-info → ddeutil_workflow-0.0.27.dist-info}/METADATA +24 -24
- ddeutil_workflow-0.0.27.dist-info/RECORD +25 -0
- ddeutil_workflow-0.0.26.post1.dist-info/RECORD +0 -23
- {ddeutil_workflow-0.0.26.post1.dist-info → ddeutil_workflow-0.0.27.dist-info}/LICENSE +0 -0
- {ddeutil_workflow-0.0.26.post1.dist-info → ddeutil_workflow-0.0.27.dist-info}/WHEEL +0 -0
- {ddeutil_workflow-0.0.26.post1.dist-info → ddeutil_workflow-0.0.27.dist-info}/top_level.txt +0 -0
ddeutil/workflow/__about__.py
CHANGED
@@ -1 +1 @@
|
|
1
|
-
__version__: str = "0.0.
|
1
|
+
__version__: str = "0.0.27"
|
ddeutil/workflow/__init__.py
CHANGED
@@ -8,6 +8,7 @@ from .conf import (
|
|
8
8
|
Config,
|
9
9
|
Loader,
|
10
10
|
Log,
|
11
|
+
config,
|
11
12
|
env,
|
12
13
|
get_log,
|
13
14
|
get_logger,
|
@@ -24,6 +25,13 @@ from .exceptions import (
|
|
24
25
|
UtilException,
|
25
26
|
WorkflowException,
|
26
27
|
)
|
28
|
+
from .hook import (
|
29
|
+
ReturnTagFunc,
|
30
|
+
TagFunc,
|
31
|
+
extract_hook,
|
32
|
+
make_registry,
|
33
|
+
tag,
|
34
|
+
)
|
27
35
|
from .job import (
|
28
36
|
Job,
|
29
37
|
Strategy,
|
@@ -48,33 +56,30 @@ from .stage import (
|
|
48
56
|
PyStage,
|
49
57
|
Stage,
|
50
58
|
TriggerStage,
|
51
|
-
extract_hook,
|
52
59
|
)
|
53
|
-
from .
|
60
|
+
from .templates import (
|
54
61
|
FILTERS,
|
55
62
|
FilterFunc,
|
56
63
|
FilterRegistry,
|
57
|
-
|
58
|
-
|
64
|
+
custom_filter,
|
65
|
+
get_args_const,
|
66
|
+
has_template,
|
67
|
+
make_filter_registry,
|
68
|
+
map_post_filter,
|
69
|
+
not_in_template,
|
70
|
+
param2template,
|
71
|
+
str2template,
|
72
|
+
)
|
73
|
+
from .utils import (
|
59
74
|
batch,
|
60
75
|
cross_product,
|
61
|
-
custom_filter,
|
62
76
|
dash2underscore,
|
63
77
|
delay,
|
64
78
|
filter_func,
|
65
79
|
gen_id,
|
66
|
-
get_args_const,
|
67
80
|
get_diff_sec,
|
68
81
|
get_dt_now,
|
69
|
-
has_template,
|
70
82
|
make_exec,
|
71
|
-
make_filter_registry,
|
72
|
-
make_registry,
|
73
|
-
map_post_filter,
|
74
|
-
not_in_template,
|
75
|
-
param2template,
|
76
|
-
str2template,
|
77
|
-
tag,
|
78
83
|
)
|
79
84
|
from .workflow import (
|
80
85
|
Workflow,
|
ddeutil/workflow/conf.py
CHANGED
@@ -108,15 +108,13 @@ class Config: # pragma: no cov
|
|
108
108
|
# NOTE: Register
|
109
109
|
@property
|
110
110
|
def regis_hook(self) -> list[str]:
|
111
|
-
regis_hook_str: str = env(
|
112
|
-
"CORE_REGISTRY", "src,src.ddeutil.workflow,tests,tests.utils"
|
113
|
-
)
|
111
|
+
regis_hook_str: str = env("CORE_REGISTRY", "src")
|
114
112
|
return [r.strip() for r in regis_hook_str.split(",")]
|
115
113
|
|
116
114
|
@property
|
117
115
|
def regis_filter(self) -> list[str]:
|
118
116
|
regis_filter_str: str = env(
|
119
|
-
"CORE_REGISTRY_FILTER", "ddeutil.workflow.
|
117
|
+
"CORE_REGISTRY_FILTER", "ddeutil.workflow.templates"
|
120
118
|
)
|
121
119
|
return [r.strip() for r in regis_filter_str.split(",")]
|
122
120
|
|
@@ -312,6 +310,10 @@ class SimLoad:
|
|
312
310
|
)
|
313
311
|
|
314
312
|
|
313
|
+
config = Config()
|
314
|
+
logger = get_logger("ddeutil.workflow")
|
315
|
+
|
316
|
+
|
315
317
|
class Loader(SimLoad):
|
316
318
|
"""Loader Object that get the config `yaml` file from current path.
|
317
319
|
|
@@ -337,15 +339,11 @@ class Loader(SimLoad):
|
|
337
339
|
:rtype: Iterator[tuple[str, DictData]]
|
338
340
|
"""
|
339
341
|
return super().finds(
|
340
|
-
obj=obj, conf=
|
342
|
+
obj=obj, conf=config, included=included, excluded=excluded
|
341
343
|
)
|
342
344
|
|
343
345
|
def __init__(self, name: str, externals: DictData) -> None:
|
344
|
-
super().__init__(name, conf=
|
345
|
-
|
346
|
-
|
347
|
-
config = Config()
|
348
|
-
logger = get_logger("ddeutil.workflow")
|
346
|
+
super().__init__(name, conf=config, externals=externals)
|
349
347
|
|
350
348
|
|
351
349
|
class BaseLog(BaseModel, ABC):
|
@@ -427,8 +425,8 @@ class FileLog(BaseLog):
|
|
427
425
|
workflow name and release values. If a release does not pass to an input
|
428
426
|
argument, it will return the latest release from the current log path.
|
429
427
|
|
430
|
-
:param name:
|
431
|
-
:param release:
|
428
|
+
:param name: A workflow name that want to search log.
|
429
|
+
:param release: A release datetime that want to search log.
|
432
430
|
|
433
431
|
:raise FileNotFoundError:
|
434
432
|
:raise NotImplementedError:
|
@@ -492,8 +490,14 @@ class FileLog(BaseLog):
|
|
492
490
|
|
493
491
|
:rtype: Self
|
494
492
|
"""
|
493
|
+
from .utils import cut_id
|
494
|
+
|
495
495
|
# NOTE: Check environ variable was set for real writing.
|
496
496
|
if not config.enable_write_log:
|
497
|
+
logger.debug(
|
498
|
+
f"({cut_id(self.run_id)}) [LOG]: Skip writing log cause "
|
499
|
+
f"config was set"
|
500
|
+
)
|
497
501
|
return self
|
498
502
|
|
499
503
|
log_file: Path = self.pointer() / f"{self.run_id}.log"
|
@@ -523,6 +527,19 @@ class SQLiteLog(BaseLog): # pragma: no cov
|
|
523
527
|
"""
|
524
528
|
|
525
529
|
def save(self, excluded: list[str] | None) -> None:
|
530
|
+
"""Save logging data that receive a context data from a workflow
|
531
|
+
execution result.
|
532
|
+
"""
|
533
|
+
from .utils import cut_id
|
534
|
+
|
535
|
+
# NOTE: Check environ variable was set for real writing.
|
536
|
+
if not config.enable_write_log:
|
537
|
+
logger.debug(
|
538
|
+
f"({cut_id(self.run_id)}) [LOG]: Skip writing log cause "
|
539
|
+
f"config was set"
|
540
|
+
)
|
541
|
+
return self
|
542
|
+
|
526
543
|
raise NotImplementedError("SQLiteLog does not implement yet.")
|
527
544
|
|
528
545
|
|
ddeutil/workflow/exceptions.py
CHANGED
ddeutil/workflow/hook.py
ADDED
@@ -0,0 +1,153 @@
|
|
1
|
+
# ------------------------------------------------------------------------------
|
2
|
+
# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
|
3
|
+
# Licensed under the MIT License. See LICENSE in the project root for
|
4
|
+
# license information.
|
5
|
+
# ------------------------------------------------------------------------------
|
6
|
+
from __future__ import annotations
|
7
|
+
|
8
|
+
import inspect
|
9
|
+
import logging
|
10
|
+
from dataclasses import dataclass
|
11
|
+
from functools import wraps
|
12
|
+
from importlib import import_module
|
13
|
+
from typing import Any, Callable, Protocol, TypeVar
|
14
|
+
|
15
|
+
try:
|
16
|
+
from typing import ParamSpec
|
17
|
+
except ImportError:
|
18
|
+
from typing_extensions import ParamSpec
|
19
|
+
|
20
|
+
from ddeutil.core import lazy
|
21
|
+
|
22
|
+
from .__types import Re
|
23
|
+
from .conf import config
|
24
|
+
|
25
|
+
T = TypeVar("T")
|
26
|
+
P = ParamSpec("P")
|
27
|
+
|
28
|
+
logger = logging.getLogger("ddeutil.workflow")
|
29
|
+
|
30
|
+
|
31
|
+
class TagFunc(Protocol):
|
32
|
+
"""Tag Function Protocol"""
|
33
|
+
|
34
|
+
name: str
|
35
|
+
tag: str
|
36
|
+
|
37
|
+
def __call__(self, *args, **kwargs): ... # pragma: no cov
|
38
|
+
|
39
|
+
|
40
|
+
ReturnTagFunc = Callable[P, TagFunc]
|
41
|
+
DecoratorTagFunc = Callable[[Callable[[...], Any]], ReturnTagFunc]
|
42
|
+
|
43
|
+
|
44
|
+
def tag(
|
45
|
+
name: str, alias: str | None = None
|
46
|
+
) -> DecoratorTagFunc: # pragma: no cov
|
47
|
+
"""Tag decorator function that set function attributes, ``tag`` and ``name``
|
48
|
+
for making registries variable.
|
49
|
+
|
50
|
+
:param: name: A tag name for make different use-case of a function.
|
51
|
+
:param: alias: A alias function name that keeping in registries. If this
|
52
|
+
value does not supply, it will use original function name from __name__.
|
53
|
+
:rtype: Callable[P, TagFunc]
|
54
|
+
"""
|
55
|
+
|
56
|
+
def func_internal(func: Callable[[...], Any]) -> ReturnTagFunc:
|
57
|
+
func.tag = name
|
58
|
+
func.name = alias or func.__name__.replace("_", "-")
|
59
|
+
|
60
|
+
@wraps(func)
|
61
|
+
def wrapped(*args, **kwargs):
|
62
|
+
# NOTE: Able to do anything before calling hook function.
|
63
|
+
return func(*args, **kwargs)
|
64
|
+
|
65
|
+
return wrapped
|
66
|
+
|
67
|
+
return func_internal
|
68
|
+
|
69
|
+
|
70
|
+
Registry = dict[str, Callable[[], TagFunc]]
|
71
|
+
|
72
|
+
|
73
|
+
def make_registry(submodule: str) -> dict[str, Registry]:
|
74
|
+
"""Return registries of all functions that able to called with task.
|
75
|
+
|
76
|
+
:param submodule: A module prefix that want to import registry.
|
77
|
+
:rtype: dict[str, Registry]
|
78
|
+
"""
|
79
|
+
rs: dict[str, Registry] = {}
|
80
|
+
for module in config.regis_hook:
|
81
|
+
# NOTE: try to sequential import task functions
|
82
|
+
try:
|
83
|
+
importer = import_module(f"{module}.{submodule}")
|
84
|
+
except ModuleNotFoundError:
|
85
|
+
continue
|
86
|
+
|
87
|
+
for fstr, func in inspect.getmembers(importer, inspect.isfunction):
|
88
|
+
# NOTE: check function attribute that already set tag by
|
89
|
+
# ``utils.tag`` decorator.
|
90
|
+
if not hasattr(func, "tag"):
|
91
|
+
continue
|
92
|
+
|
93
|
+
# NOTE: Create new register name if it not exists
|
94
|
+
if func.name not in rs:
|
95
|
+
rs[func.name] = {func.tag: lazy(f"{module}.{submodule}.{fstr}")}
|
96
|
+
continue
|
97
|
+
|
98
|
+
if func.tag in rs[func.name]:
|
99
|
+
raise ValueError(
|
100
|
+
f"The tag {func.tag!r} already exists on "
|
101
|
+
f"{module}.{submodule}, you should change this tag name or "
|
102
|
+
f"change it func name."
|
103
|
+
)
|
104
|
+
rs[func.name][func.tag] = lazy(f"{module}.{submodule}.{fstr}")
|
105
|
+
|
106
|
+
return rs
|
107
|
+
|
108
|
+
|
109
|
+
@dataclass(frozen=True)
|
110
|
+
class HookSearchData:
|
111
|
+
"""Hook Search dataclass that use for receive regular expression grouping
|
112
|
+
dict from searching hook string value.
|
113
|
+
"""
|
114
|
+
|
115
|
+
path: str
|
116
|
+
func: str
|
117
|
+
tag: str
|
118
|
+
|
119
|
+
|
120
|
+
def extract_hook(hook: str) -> Callable[[], TagFunc]:
|
121
|
+
"""Extract Hook function from string value to hook partial function that
|
122
|
+
does run it at runtime.
|
123
|
+
|
124
|
+
:raise NotImplementedError: When the searching hook's function result does
|
125
|
+
not exist in the registry.
|
126
|
+
:raise NotImplementedError: When the searching hook's tag result does not
|
127
|
+
exists in the registry with its function key.
|
128
|
+
|
129
|
+
:param hook: A hook value that able to match with Task regex.
|
130
|
+
:rtype: Callable[[], TagFunc]
|
131
|
+
"""
|
132
|
+
if not (found := Re.RE_TASK_FMT.search(hook)):
|
133
|
+
raise ValueError(
|
134
|
+
f"Hook {hook!r} does not match with hook format regex."
|
135
|
+
)
|
136
|
+
|
137
|
+
# NOTE: Pass the searching hook string to `path`, `func`, and `tag`.
|
138
|
+
hook: HookSearchData = HookSearchData(**found.groupdict())
|
139
|
+
|
140
|
+
# NOTE: Registry object should implement on this package only.
|
141
|
+
rgt: dict[str, Registry] = make_registry(f"{hook.path}")
|
142
|
+
if hook.func not in rgt:
|
143
|
+
raise NotImplementedError(
|
144
|
+
f"``REGISTER-MODULES.{hook.path}.registries`` does not "
|
145
|
+
f"implement registry: {hook.func!r}."
|
146
|
+
)
|
147
|
+
|
148
|
+
if hook.tag not in rgt[hook.func]:
|
149
|
+
raise NotImplementedError(
|
150
|
+
f"tag: {hook.tag!r} does not found on registry func: "
|
151
|
+
f"``REGISTER-MODULES.{hook.path}.registries.{hook.func}``"
|
152
|
+
)
|
153
|
+
return rgt[hook.func][hook.tag]
|
ddeutil/workflow/job.py
CHANGED
@@ -38,13 +38,13 @@ from .exceptions import (
|
|
38
38
|
)
|
39
39
|
from .result import Result
|
40
40
|
from .stage import Stage
|
41
|
+
from .templates import has_template
|
41
42
|
from .utils import (
|
42
43
|
cross_product,
|
43
44
|
cut_id,
|
44
45
|
dash2underscore,
|
45
46
|
filter_func,
|
46
47
|
gen_id,
|
47
|
-
has_template,
|
48
48
|
)
|
49
49
|
|
50
50
|
logger = get_logger("ddeutil.workflow")
|
ddeutil/workflow/stage.py
CHANGED
@@ -31,7 +31,6 @@ import time
|
|
31
31
|
import uuid
|
32
32
|
from abc import ABC, abstractmethod
|
33
33
|
from collections.abc import Iterator
|
34
|
-
from dataclasses import dataclass
|
35
34
|
from functools import wraps
|
36
35
|
from inspect import Parameter
|
37
36
|
from pathlib import Path
|
@@ -48,19 +47,16 @@ from pydantic import BaseModel, Field
|
|
48
47
|
from pydantic.functional_validators import model_validator
|
49
48
|
from typing_extensions import Self
|
50
49
|
|
51
|
-
from .__types import DictData, DictStr,
|
50
|
+
from .__types import DictData, DictStr, TupleStr
|
52
51
|
from .conf import config, get_logger
|
53
52
|
from .exceptions import StageException
|
53
|
+
from .hook import TagFunc, extract_hook
|
54
54
|
from .result import Result
|
55
|
+
from .templates import not_in_template, param2template
|
55
56
|
from .utils import (
|
56
|
-
Registry,
|
57
|
-
TagFunc,
|
58
57
|
cut_id,
|
59
58
|
gen_id,
|
60
59
|
make_exec,
|
61
|
-
make_registry,
|
62
|
-
not_in_template,
|
63
|
-
param2template,
|
64
60
|
)
|
65
61
|
|
66
62
|
P = ParamSpec("P")
|
@@ -76,7 +72,6 @@ __all__: TupleStr = (
|
|
76
72
|
"HookStage",
|
77
73
|
"TriggerStage",
|
78
74
|
"Stage",
|
79
|
-
"extract_hook",
|
80
75
|
)
|
81
76
|
|
82
77
|
|
@@ -558,53 +553,6 @@ class PyStage(BaseStage):
|
|
558
553
|
)
|
559
554
|
|
560
555
|
|
561
|
-
@dataclass(frozen=True)
|
562
|
-
class HookSearchData:
|
563
|
-
"""Hook Search dataclass that use for receive regular expression grouping
|
564
|
-
dict from searching hook string value.
|
565
|
-
"""
|
566
|
-
|
567
|
-
path: str
|
568
|
-
func: str
|
569
|
-
tag: str
|
570
|
-
|
571
|
-
|
572
|
-
def extract_hook(hook: str) -> Callable[[], TagFunc]:
|
573
|
-
"""Extract Hook function from string value to hook partial function that
|
574
|
-
does run it at runtime.
|
575
|
-
|
576
|
-
:raise NotImplementedError: When the searching hook's function result does
|
577
|
-
not exist in the registry.
|
578
|
-
:raise NotImplementedError: When the searching hook's tag result does not
|
579
|
-
exists in the registry with its function key.
|
580
|
-
|
581
|
-
:param hook: A hook value that able to match with Task regex.
|
582
|
-
:rtype: Callable[[], TagFunc]
|
583
|
-
"""
|
584
|
-
if not (found := Re.RE_TASK_FMT.search(hook)):
|
585
|
-
raise ValueError(
|
586
|
-
f"Hook {hook!r} does not match with hook format regex."
|
587
|
-
)
|
588
|
-
|
589
|
-
# NOTE: Pass the searching hook string to `path`, `func`, and `tag`.
|
590
|
-
hook: HookSearchData = HookSearchData(**found.groupdict())
|
591
|
-
|
592
|
-
# NOTE: Registry object should implement on this package only.
|
593
|
-
rgt: dict[str, Registry] = make_registry(f"{hook.path}")
|
594
|
-
if hook.func not in rgt:
|
595
|
-
raise NotImplementedError(
|
596
|
-
f"``REGISTER-MODULES.{hook.path}.registries`` does not "
|
597
|
-
f"implement registry: {hook.func!r}."
|
598
|
-
)
|
599
|
-
|
600
|
-
if hook.tag not in rgt[hook.func]:
|
601
|
-
raise NotImplementedError(
|
602
|
-
f"tag: {hook.tag!r} does not found on registry func: "
|
603
|
-
f"``REGISTER-MODULES.{hook.path}.registries.{hook.func}``"
|
604
|
-
)
|
605
|
-
return rgt[hook.func][hook.tag]
|
606
|
-
|
607
|
-
|
608
556
|
class HookStage(BaseStage):
|
609
557
|
"""Hook executor that hook the Python function from registry with tag
|
610
558
|
decorator function in ``utils`` module and run it with input arguments.
|
@@ -0,0 +1,334 @@
|
|
1
|
+
# ------------------------------------------------------------------------------
|
2
|
+
# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
|
3
|
+
# Licensed under the MIT License. See LICENSE in the project root for
|
4
|
+
# license information.
|
5
|
+
# ------------------------------------------------------------------------------
|
6
|
+
from __future__ import annotations
|
7
|
+
|
8
|
+
import inspect
|
9
|
+
import logging
|
10
|
+
from ast import Call, Constant, Expr, Module, Name, parse
|
11
|
+
from datetime import datetime
|
12
|
+
from functools import wraps
|
13
|
+
from importlib import import_module
|
14
|
+
from typing import Any, Callable, Protocol, TypeVar, Union
|
15
|
+
|
16
|
+
try:
|
17
|
+
from typing import ParamSpec
|
18
|
+
except ImportError:
|
19
|
+
from typing_extensions import ParamSpec
|
20
|
+
|
21
|
+
from ddeutil.core import getdot, hasdot, import_string
|
22
|
+
from ddeutil.io import search_env_replace
|
23
|
+
|
24
|
+
from .__types import DictData, Re
|
25
|
+
from .conf import config
|
26
|
+
from .exceptions import UtilException
|
27
|
+
|
28
|
+
T = TypeVar("T")
|
29
|
+
P = ParamSpec("P")
|
30
|
+
|
31
|
+
logger = logging.getLogger("ddeutil.workflow")
|
32
|
+
|
33
|
+
|
34
|
+
FILTERS: dict[str, callable] = { # pragma: no cov
|
35
|
+
"abs": abs,
|
36
|
+
"str": str,
|
37
|
+
"int": int,
|
38
|
+
"title": lambda x: x.title(),
|
39
|
+
"upper": lambda x: x.upper(),
|
40
|
+
"lower": lambda x: x.lower(),
|
41
|
+
"rstr": [str, repr],
|
42
|
+
}
|
43
|
+
|
44
|
+
|
45
|
+
class FilterFunc(Protocol):
|
46
|
+
"""Tag Function Protocol. This protocol that use to represent any callable
|
47
|
+
object that able to access the name attribute.
|
48
|
+
"""
|
49
|
+
|
50
|
+
name: str
|
51
|
+
|
52
|
+
def __call__(self, *args, **kwargs): ... # pragma: no cov
|
53
|
+
|
54
|
+
|
55
|
+
FilterRegistry = Union[FilterFunc, Callable[[...], Any]]
|
56
|
+
|
57
|
+
|
58
|
+
def custom_filter(name: str) -> Callable[P, FilterFunc]:
|
59
|
+
"""Custom filter decorator function that set function attributes, ``filter``
|
60
|
+
for making filter registries variable.
|
61
|
+
|
62
|
+
:param: name: A filter name for make different use-case of a function.
|
63
|
+
:rtype: Callable[P, FilterFunc]
|
64
|
+
"""
|
65
|
+
|
66
|
+
def func_internal(func: Callable[[...], Any]) -> FilterFunc:
|
67
|
+
func.filter = name
|
68
|
+
|
69
|
+
@wraps(func)
|
70
|
+
def wrapped(*args, **kwargs):
|
71
|
+
# NOTE: Able to do anything before calling custom filter function.
|
72
|
+
return func(*args, **kwargs)
|
73
|
+
|
74
|
+
return wrapped
|
75
|
+
|
76
|
+
return func_internal
|
77
|
+
|
78
|
+
|
79
|
+
def make_filter_registry() -> dict[str, FilterRegistry]:
|
80
|
+
"""Return registries of all functions that able to called with task.
|
81
|
+
|
82
|
+
:rtype: dict[str, Registry]
|
83
|
+
"""
|
84
|
+
rs: dict[str, FilterRegistry] = {}
|
85
|
+
for module in config.regis_filter:
|
86
|
+
# NOTE: try to sequential import task functions
|
87
|
+
try:
|
88
|
+
importer = import_module(module)
|
89
|
+
except ModuleNotFoundError:
|
90
|
+
continue
|
91
|
+
|
92
|
+
for fstr, func in inspect.getmembers(importer, inspect.isfunction):
|
93
|
+
# NOTE: check function attribute that already set tag by
|
94
|
+
# ``utils.tag`` decorator.
|
95
|
+
if not hasattr(func, "filter"):
|
96
|
+
continue
|
97
|
+
|
98
|
+
rs[func.filter] = import_string(f"{module}.{fstr}")
|
99
|
+
|
100
|
+
rs.update(FILTERS)
|
101
|
+
return rs
|
102
|
+
|
103
|
+
|
104
|
+
def get_args_const(
|
105
|
+
expr: str,
|
106
|
+
) -> tuple[str, list[Constant], dict[str, Constant]]:
|
107
|
+
"""Get arguments and keyword-arguments from function calling string.
|
108
|
+
|
109
|
+
:rtype: tuple[str, list[Constant], dict[str, Constant]]
|
110
|
+
"""
|
111
|
+
try:
|
112
|
+
mod: Module = parse(expr)
|
113
|
+
except SyntaxError:
|
114
|
+
raise UtilException(
|
115
|
+
f"Post-filter: {expr} does not valid because it raise syntax error."
|
116
|
+
) from None
|
117
|
+
|
118
|
+
body: list[Expr] = mod.body
|
119
|
+
if len(body) > 1:
|
120
|
+
raise UtilException(
|
121
|
+
"Post-filter function should be only one calling per workflow."
|
122
|
+
)
|
123
|
+
|
124
|
+
caller: Union[Name, Call]
|
125
|
+
if isinstance((caller := body[0].value), Name):
|
126
|
+
return caller.id, [], {}
|
127
|
+
elif not isinstance(caller, Call):
|
128
|
+
raise UtilException(
|
129
|
+
f"Get arguments does not support for caller type: {type(caller)}"
|
130
|
+
)
|
131
|
+
|
132
|
+
name: Name = caller.func
|
133
|
+
args: list[Constant] = caller.args
|
134
|
+
keywords: dict[str, Constant] = {k.arg: k.value for k in caller.keywords}
|
135
|
+
|
136
|
+
if any(not isinstance(i, Constant) for i in args):
|
137
|
+
raise UtilException(f"Argument of {expr} should be constant.")
|
138
|
+
|
139
|
+
if any(not isinstance(i, Constant) for i in keywords.values()):
|
140
|
+
raise UtilException(f"Keyword argument of {expr} should be constant.")
|
141
|
+
|
142
|
+
return name.id, args, keywords
|
143
|
+
|
144
|
+
|
145
|
+
def get_args_from_filter(
|
146
|
+
ft: str,
|
147
|
+
filters: dict[str, FilterRegistry],
|
148
|
+
) -> tuple[str, FilterRegistry, list[Any], dict[Any, Any]]: # pragma: no cov
|
149
|
+
"""Get arguments and keyword-arguments from filter function calling string.
|
150
|
+
and validate it with the filter functions mapping dict.
|
151
|
+
"""
|
152
|
+
func_name, _args, _kwargs = get_args_const(ft)
|
153
|
+
args: list[Any] = [arg.value for arg in _args]
|
154
|
+
kwargs: dict[Any, Any] = {k: v.value for k, v in _kwargs.items()}
|
155
|
+
|
156
|
+
if func_name not in filters:
|
157
|
+
raise UtilException(
|
158
|
+
f"The post-filter: {func_name!r} does not support yet."
|
159
|
+
)
|
160
|
+
|
161
|
+
if isinstance((f_func := filters[func_name]), list) and (args or kwargs):
|
162
|
+
raise UtilException(
|
163
|
+
"Chain filter function does not support for passing arguments."
|
164
|
+
)
|
165
|
+
|
166
|
+
return func_name, f_func, args, kwargs
|
167
|
+
|
168
|
+
|
169
|
+
def map_post_filter(
|
170
|
+
value: T,
|
171
|
+
post_filter: list[str],
|
172
|
+
filters: dict[str, FilterRegistry],
|
173
|
+
) -> T:
|
174
|
+
"""Mapping post-filter to value with sequence list of filter function name
|
175
|
+
that will get from the filter registry.
|
176
|
+
|
177
|
+
:param value: A string value that want to mapped with filter function.
|
178
|
+
:param post_filter: A list of post-filter function name.
|
179
|
+
:param filters: A filter registry.
|
180
|
+
|
181
|
+
:rtype: T
|
182
|
+
"""
|
183
|
+
for ft in post_filter:
|
184
|
+
func_name, f_func, args, kwargs = get_args_from_filter(ft, filters)
|
185
|
+
try:
|
186
|
+
if isinstance(f_func, list):
|
187
|
+
for func in f_func:
|
188
|
+
value: T = func(value)
|
189
|
+
else:
|
190
|
+
value: T = f_func(value, *args, **kwargs)
|
191
|
+
except UtilException as err:
|
192
|
+
logger.warning(str(err))
|
193
|
+
raise
|
194
|
+
except Exception as err:
|
195
|
+
logger.warning(str(err))
|
196
|
+
raise UtilException(
|
197
|
+
f"The post-filter function: {func_name} does not fit with "
|
198
|
+
f"{value} (type: {type(value).__name__})."
|
199
|
+
) from None
|
200
|
+
return value
|
201
|
+
|
202
|
+
|
203
|
+
def not_in_template(value: Any, *, not_in: str = "matrix.") -> bool:
|
204
|
+
"""Check value should not pass template with not_in value prefix.
|
205
|
+
|
206
|
+
:param value: A value that want to find parameter template prefix.
|
207
|
+
:param not_in: The not in string that use in the `.startswith` function.
|
208
|
+
|
209
|
+
:rtype: bool
|
210
|
+
"""
|
211
|
+
if isinstance(value, dict):
|
212
|
+
return any(not_in_template(value[k], not_in=not_in) for k in value)
|
213
|
+
elif isinstance(value, (list, tuple, set)):
|
214
|
+
return any(not_in_template(i, not_in=not_in) for i in value)
|
215
|
+
elif not isinstance(value, str):
|
216
|
+
return False
|
217
|
+
return any(
|
218
|
+
(not found.caller.strip().startswith(not_in))
|
219
|
+
for found in Re.finditer_caller(value.strip())
|
220
|
+
)
|
221
|
+
|
222
|
+
|
223
|
+
def has_template(value: Any) -> bool:
|
224
|
+
"""Check value include templating string.
|
225
|
+
|
226
|
+
:param value: A value that want to find parameter template.
|
227
|
+
|
228
|
+
:rtype: bool
|
229
|
+
"""
|
230
|
+
if isinstance(value, dict):
|
231
|
+
return any(has_template(value[k]) for k in value)
|
232
|
+
elif isinstance(value, (list, tuple, set)):
|
233
|
+
return any(has_template(i) for i in value)
|
234
|
+
elif not isinstance(value, str):
|
235
|
+
return False
|
236
|
+
return bool(Re.RE_CALLER.findall(value.strip()))
|
237
|
+
|
238
|
+
|
239
|
+
def str2template(
|
240
|
+
value: str,
|
241
|
+
params: DictData,
|
242
|
+
*,
|
243
|
+
filters: dict[str, FilterRegistry] | None = None,
|
244
|
+
) -> Any:
|
245
|
+
"""(Sub-function) Pass param to template string that can search by
|
246
|
+
``RE_CALLER`` regular expression.
|
247
|
+
|
248
|
+
The getter value that map a template should have typing support align
|
249
|
+
with the workflow parameter types that is `str`, `int`, `datetime`, and
|
250
|
+
`list`.
|
251
|
+
|
252
|
+
:param value: A string value that want to mapped with an params
|
253
|
+
:param params: A parameter value that getting with matched regular
|
254
|
+
expression.
|
255
|
+
:param filters:
|
256
|
+
"""
|
257
|
+
filters: dict[str, FilterRegistry] = filters or make_filter_registry()
|
258
|
+
|
259
|
+
# NOTE: remove space before and after this string value.
|
260
|
+
value: str = value.strip()
|
261
|
+
for found in Re.finditer_caller(value):
|
262
|
+
# NOTE:
|
263
|
+
# Get caller and filter values that setting inside;
|
264
|
+
#
|
265
|
+
# ... ``${{ <caller-value> [ | <filter-value>] ... }}``
|
266
|
+
#
|
267
|
+
caller: str = found.caller
|
268
|
+
pfilter: list[str] = [
|
269
|
+
i.strip()
|
270
|
+
for i in (found.post_filters.strip().removeprefix("|").split("|"))
|
271
|
+
if i != ""
|
272
|
+
]
|
273
|
+
if not hasdot(caller, params):
|
274
|
+
raise UtilException(f"The params does not set caller: {caller!r}.")
|
275
|
+
|
276
|
+
# NOTE: from validate step, it guarantee that caller exists in params.
|
277
|
+
getter: Any = getdot(caller, params)
|
278
|
+
|
279
|
+
# NOTE:
|
280
|
+
# If type of getter caller is not string type and it does not use to
|
281
|
+
# concat other string value, it will return origin value from the
|
282
|
+
# ``getdot`` function.
|
283
|
+
if value.replace(found.full, "", 1) == "":
|
284
|
+
return map_post_filter(getter, pfilter, filters=filters)
|
285
|
+
|
286
|
+
# NOTE: map post-filter function.
|
287
|
+
getter: Any = map_post_filter(getter, pfilter, filters=filters)
|
288
|
+
if not isinstance(getter, str):
|
289
|
+
getter: str = str(getter)
|
290
|
+
|
291
|
+
value: str = value.replace(found.full, getter, 1)
|
292
|
+
|
293
|
+
return search_env_replace(value)
|
294
|
+
|
295
|
+
|
296
|
+
def param2template(
|
297
|
+
value: Any,
|
298
|
+
params: DictData,
|
299
|
+
) -> Any:
|
300
|
+
"""Pass param to template string that can search by ``RE_CALLER`` regular
|
301
|
+
expression.
|
302
|
+
|
303
|
+
:param value: A value that want to mapped with an params
|
304
|
+
:param params: A parameter value that getting with matched regular
|
305
|
+
expression.
|
306
|
+
|
307
|
+
:rtype: Any
|
308
|
+
:returns: An any getter value from the params input.
|
309
|
+
"""
|
310
|
+
filters: dict[str, FilterRegistry] = make_filter_registry()
|
311
|
+
if isinstance(value, dict):
|
312
|
+
return {k: param2template(value[k], params) for k in value}
|
313
|
+
elif isinstance(value, (list, tuple, set)):
|
314
|
+
return type(value)([param2template(i, params) for i in value])
|
315
|
+
elif not isinstance(value, str):
|
316
|
+
return value
|
317
|
+
return str2template(value, params, filters=filters)
|
318
|
+
|
319
|
+
|
320
|
+
@custom_filter("fmt") # pragma: no cov
|
321
|
+
def datetime_format(value: datetime, fmt: str = "%Y-%m-%d %H:%M:%S") -> str:
|
322
|
+
"""Format datetime object to string with the format.
|
323
|
+
|
324
|
+
:param value: A datetime value that want to format to string value.
|
325
|
+
:param fmt: A format string pattern that passing to the `dt.strftime`
|
326
|
+
method.
|
327
|
+
|
328
|
+
:rtype: str
|
329
|
+
"""
|
330
|
+
if isinstance(value, datetime):
|
331
|
+
return value.strftime(fmt)
|
332
|
+
raise UtilException(
|
333
|
+
"This custom function should pass input value with datetime type."
|
334
|
+
)
|
ddeutil/workflow/utils.py
CHANGED
@@ -5,21 +5,17 @@
|
|
5
5
|
# ------------------------------------------------------------------------------
|
6
6
|
from __future__ import annotations
|
7
7
|
|
8
|
-
import inspect
|
9
8
|
import logging
|
10
9
|
import stat
|
11
10
|
import time
|
12
|
-
from ast import Call, Constant, Expr, Module, Name, parse
|
13
11
|
from collections.abc import Iterator
|
14
12
|
from datetime import datetime, timedelta
|
15
|
-
from functools import wraps
|
16
13
|
from hashlib import md5
|
17
|
-
from importlib import import_module
|
18
14
|
from inspect import isfunction
|
19
15
|
from itertools import chain, islice, product
|
20
16
|
from pathlib import Path
|
21
17
|
from random import randrange
|
22
|
-
from typing import Any,
|
18
|
+
from typing import Any, TypeVar
|
23
19
|
from zoneinfo import ZoneInfo
|
24
20
|
|
25
21
|
try:
|
@@ -27,18 +23,13 @@ try:
|
|
27
23
|
except ImportError:
|
28
24
|
from typing_extensions import ParamSpec
|
29
25
|
|
30
|
-
from ddeutil.core import
|
31
|
-
from ddeutil.io import search_env_replace
|
32
|
-
from pydantic import BaseModel
|
26
|
+
from ddeutil.core import hash_str
|
33
27
|
|
34
|
-
from .__types import DictData, Matrix
|
28
|
+
from .__types import DictData, Matrix
|
35
29
|
from .conf import config
|
36
|
-
from .exceptions import UtilException
|
37
30
|
|
38
31
|
T = TypeVar("T")
|
39
32
|
P = ParamSpec("P")
|
40
|
-
AnyModel = TypeVar("AnyModel", bound=BaseModel)
|
41
|
-
AnyModelType = type[AnyModel]
|
42
33
|
|
43
34
|
logger = logging.getLogger("ddeutil.workflow")
|
44
35
|
|
@@ -121,84 +112,6 @@ def gen_id(
|
|
121
112
|
).hexdigest()
|
122
113
|
|
123
114
|
|
124
|
-
class TagFunc(Protocol):
|
125
|
-
"""Tag Function Protocol"""
|
126
|
-
|
127
|
-
name: str
|
128
|
-
tag: str
|
129
|
-
|
130
|
-
def __call__(self, *args, **kwargs): ... # pragma: no cov
|
131
|
-
|
132
|
-
|
133
|
-
ReturnTagFunc = Callable[P, TagFunc]
|
134
|
-
DecoratorTagFunc = Callable[[Callable[[...], Any]], ReturnTagFunc]
|
135
|
-
|
136
|
-
|
137
|
-
def tag(
|
138
|
-
name: str, alias: str | None = None
|
139
|
-
) -> DecoratorTagFunc: # pragma: no cov
|
140
|
-
"""Tag decorator function that set function attributes, ``tag`` and ``name``
|
141
|
-
for making registries variable.
|
142
|
-
|
143
|
-
:param: name: A tag name for make different use-case of a function.
|
144
|
-
:param: alias: A alias function name that keeping in registries. If this
|
145
|
-
value does not supply, it will use original function name from __name__.
|
146
|
-
:rtype: Callable[P, TagFunc]
|
147
|
-
"""
|
148
|
-
|
149
|
-
def func_internal(func: Callable[[...], Any]) -> ReturnTagFunc:
|
150
|
-
func.tag = name
|
151
|
-
func.name = alias or func.__name__.replace("_", "-")
|
152
|
-
|
153
|
-
@wraps(func)
|
154
|
-
def wrapped(*args, **kwargs):
|
155
|
-
# NOTE: Able to do anything before calling hook function.
|
156
|
-
return func(*args, **kwargs)
|
157
|
-
|
158
|
-
return wrapped
|
159
|
-
|
160
|
-
return func_internal
|
161
|
-
|
162
|
-
|
163
|
-
Registry = dict[str, Callable[[], TagFunc]]
|
164
|
-
|
165
|
-
|
166
|
-
def make_registry(submodule: str) -> dict[str, Registry]:
|
167
|
-
"""Return registries of all functions that able to called with task.
|
168
|
-
|
169
|
-
:param submodule: A module prefix that want to import registry.
|
170
|
-
:rtype: dict[str, Registry]
|
171
|
-
"""
|
172
|
-
rs: dict[str, Registry] = {}
|
173
|
-
for module in config.regis_hook:
|
174
|
-
# NOTE: try to sequential import task functions
|
175
|
-
try:
|
176
|
-
importer = import_module(f"{module}.{submodule}")
|
177
|
-
except ModuleNotFoundError:
|
178
|
-
continue
|
179
|
-
|
180
|
-
for fstr, func in inspect.getmembers(importer, inspect.isfunction):
|
181
|
-
# NOTE: check function attribute that already set tag by
|
182
|
-
# ``utils.tag`` decorator.
|
183
|
-
if not hasattr(func, "tag"):
|
184
|
-
continue
|
185
|
-
|
186
|
-
# NOTE: Create new register name if it not exists
|
187
|
-
if func.name not in rs:
|
188
|
-
rs[func.name] = {func.tag: lazy(f"{module}.{submodule}.{fstr}")}
|
189
|
-
continue
|
190
|
-
|
191
|
-
if func.tag in rs[func.name]:
|
192
|
-
raise ValueError(
|
193
|
-
f"The tag {func.tag!r} already exists on "
|
194
|
-
f"{module}.{submodule}, you should change this tag name or "
|
195
|
-
f"change it func name."
|
196
|
-
)
|
197
|
-
rs[func.name][func.tag] = lazy(f"{module}.{submodule}.{fstr}")
|
198
|
-
|
199
|
-
return rs
|
200
|
-
|
201
|
-
|
202
115
|
def make_exec(path: str | Path) -> None:
|
203
116
|
"""Change mode of file to be executable file.
|
204
117
|
|
@@ -208,307 +121,6 @@ def make_exec(path: str | Path) -> None:
|
|
208
121
|
f.chmod(f.stat().st_mode | stat.S_IEXEC)
|
209
122
|
|
210
123
|
|
211
|
-
FILTERS: dict[str, callable] = { # pragma: no cov
|
212
|
-
"abs": abs,
|
213
|
-
"str": str,
|
214
|
-
"int": int,
|
215
|
-
"title": lambda x: x.title(),
|
216
|
-
"upper": lambda x: x.upper(),
|
217
|
-
"lower": lambda x: x.lower(),
|
218
|
-
"rstr": [str, repr],
|
219
|
-
}
|
220
|
-
|
221
|
-
|
222
|
-
class FilterFunc(Protocol):
|
223
|
-
"""Tag Function Protocol. This protocol that use to represent any callable
|
224
|
-
object that able to access the name attribute.
|
225
|
-
"""
|
226
|
-
|
227
|
-
name: str
|
228
|
-
|
229
|
-
def __call__(self, *args, **kwargs): ... # pragma: no cov
|
230
|
-
|
231
|
-
|
232
|
-
def custom_filter(name: str) -> Callable[P, FilterFunc]:
|
233
|
-
"""Custom filter decorator function that set function attributes, ``filter``
|
234
|
-
for making filter registries variable.
|
235
|
-
|
236
|
-
:param: name: A filter name for make different use-case of a function.
|
237
|
-
:rtype: Callable[P, FilterFunc]
|
238
|
-
"""
|
239
|
-
|
240
|
-
def func_internal(func: Callable[[...], Any]) -> FilterFunc:
|
241
|
-
func.filter = name
|
242
|
-
|
243
|
-
@wraps(func)
|
244
|
-
def wrapped(*args, **kwargs):
|
245
|
-
# NOTE: Able to do anything before calling custom filter function.
|
246
|
-
return func(*args, **kwargs)
|
247
|
-
|
248
|
-
return wrapped
|
249
|
-
|
250
|
-
return func_internal
|
251
|
-
|
252
|
-
|
253
|
-
FilterRegistry = Union[FilterFunc, Callable[[...], Any]]
|
254
|
-
|
255
|
-
|
256
|
-
def make_filter_registry() -> dict[str, FilterRegistry]:
|
257
|
-
"""Return registries of all functions that able to called with task.
|
258
|
-
|
259
|
-
:rtype: dict[str, Registry]
|
260
|
-
"""
|
261
|
-
rs: dict[str, Registry] = {}
|
262
|
-
for module in config.regis_filter:
|
263
|
-
# NOTE: try to sequential import task functions
|
264
|
-
try:
|
265
|
-
importer = import_module(module)
|
266
|
-
except ModuleNotFoundError:
|
267
|
-
continue
|
268
|
-
|
269
|
-
for fstr, func in inspect.getmembers(importer, inspect.isfunction):
|
270
|
-
# NOTE: check function attribute that already set tag by
|
271
|
-
# ``utils.tag`` decorator.
|
272
|
-
if not hasattr(func, "filter"):
|
273
|
-
continue
|
274
|
-
|
275
|
-
rs[func.filter] = import_string(f"{module}.{fstr}")
|
276
|
-
|
277
|
-
rs.update(FILTERS)
|
278
|
-
return rs
|
279
|
-
|
280
|
-
|
281
|
-
def get_args_const(
|
282
|
-
expr: str,
|
283
|
-
) -> tuple[str, list[Constant], dict[str, Constant]]:
|
284
|
-
"""Get arguments and keyword-arguments from function calling string.
|
285
|
-
|
286
|
-
:rtype: tuple[str, list[Constant], dict[str, Constant]]
|
287
|
-
"""
|
288
|
-
try:
|
289
|
-
mod: Module = parse(expr)
|
290
|
-
except SyntaxError:
|
291
|
-
raise UtilException(
|
292
|
-
f"Post-filter: {expr} does not valid because it raise syntax error."
|
293
|
-
) from None
|
294
|
-
|
295
|
-
body: list[Expr] = mod.body
|
296
|
-
if len(body) > 1:
|
297
|
-
raise UtilException(
|
298
|
-
"Post-filter function should be only one calling per workflow."
|
299
|
-
)
|
300
|
-
|
301
|
-
caller: Union[Name, Call]
|
302
|
-
if isinstance((caller := body[0].value), Name):
|
303
|
-
return caller.id, [], {}
|
304
|
-
elif not isinstance(caller, Call):
|
305
|
-
raise UtilException(
|
306
|
-
f"Get arguments does not support for caller type: {type(caller)}"
|
307
|
-
)
|
308
|
-
|
309
|
-
name: Name = caller.func
|
310
|
-
args: list[Constant] = caller.args
|
311
|
-
keywords: dict[str, Constant] = {k.arg: k.value for k in caller.keywords}
|
312
|
-
|
313
|
-
if any(not isinstance(i, Constant) for i in args):
|
314
|
-
raise UtilException(f"Argument of {expr} should be constant.")
|
315
|
-
|
316
|
-
if any(not isinstance(i, Constant) for i in keywords.values()):
|
317
|
-
raise UtilException(f"Keyword argument of {expr} should be constant.")
|
318
|
-
|
319
|
-
return name.id, args, keywords
|
320
|
-
|
321
|
-
|
322
|
-
def get_args_from_filter(
|
323
|
-
ft: str,
|
324
|
-
filters: dict[str, FilterRegistry],
|
325
|
-
) -> tuple[str, FilterRegistry, list[Any], dict[Any, Any]]: # pragma: no cov
|
326
|
-
"""Get arguments and keyword-arguments from filter function calling string.
|
327
|
-
and validate it with the filter functions mapping dict.
|
328
|
-
"""
|
329
|
-
func_name, _args, _kwargs = get_args_const(ft)
|
330
|
-
args: list[Any] = [arg.value for arg in _args]
|
331
|
-
kwargs: dict[Any, Any] = {k: v.value for k, v in _kwargs.items()}
|
332
|
-
|
333
|
-
if func_name not in filters:
|
334
|
-
raise UtilException(
|
335
|
-
f"The post-filter: {func_name!r} does not support yet."
|
336
|
-
)
|
337
|
-
|
338
|
-
if isinstance((f_func := filters[func_name]), list) and (args or kwargs):
|
339
|
-
raise UtilException(
|
340
|
-
"Chain filter function does not support for passing arguments."
|
341
|
-
)
|
342
|
-
|
343
|
-
return func_name, f_func, args, kwargs
|
344
|
-
|
345
|
-
|
346
|
-
@custom_filter("fmt") # pragma: no cov
|
347
|
-
def datetime_format(value: datetime, fmt: str = "%Y-%m-%d %H:%M:%S") -> str:
|
348
|
-
"""Format datetime object to string with the format.
|
349
|
-
|
350
|
-
:param value: A datetime value that want to format to string value.
|
351
|
-
:param fmt: A format string pattern that passing to the `dt.strftime`
|
352
|
-
method.
|
353
|
-
|
354
|
-
:rtype: str
|
355
|
-
"""
|
356
|
-
if isinstance(value, datetime):
|
357
|
-
return value.strftime(fmt)
|
358
|
-
raise UtilException(
|
359
|
-
"This custom function should pass input value with datetime type."
|
360
|
-
)
|
361
|
-
|
362
|
-
|
363
|
-
def map_post_filter(
|
364
|
-
value: T,
|
365
|
-
post_filter: list[str],
|
366
|
-
filters: dict[str, FilterRegistry],
|
367
|
-
) -> T:
|
368
|
-
"""Mapping post-filter to value with sequence list of filter function name
|
369
|
-
that will get from the filter registry.
|
370
|
-
|
371
|
-
:param value: A string value that want to mapped with filter function.
|
372
|
-
:param post_filter: A list of post-filter function name.
|
373
|
-
:param filters: A filter registry.
|
374
|
-
|
375
|
-
:rtype: T
|
376
|
-
"""
|
377
|
-
for ft in post_filter:
|
378
|
-
func_name, f_func, args, kwargs = get_args_from_filter(ft, filters)
|
379
|
-
try:
|
380
|
-
if isinstance(f_func, list):
|
381
|
-
for func in f_func:
|
382
|
-
value: T = func(value)
|
383
|
-
else:
|
384
|
-
value: T = f_func(value, *args, **kwargs)
|
385
|
-
except UtilException as err:
|
386
|
-
logger.warning(str(err))
|
387
|
-
raise
|
388
|
-
except Exception as err:
|
389
|
-
logger.warning(str(err))
|
390
|
-
raise UtilException(
|
391
|
-
f"The post-filter function: {func_name} does not fit with "
|
392
|
-
f"{value} (type: {type(value).__name__})."
|
393
|
-
) from None
|
394
|
-
return value
|
395
|
-
|
396
|
-
|
397
|
-
def not_in_template(value: Any, *, not_in: str = "matrix.") -> bool:
|
398
|
-
"""Check value should not pass template with not_in value prefix.
|
399
|
-
|
400
|
-
:param value: A value that want to find parameter template prefix.
|
401
|
-
:param not_in: The not in string that use in the `.startswith` function.
|
402
|
-
:rtype: bool
|
403
|
-
"""
|
404
|
-
if isinstance(value, dict):
|
405
|
-
return any(not_in_template(value[k], not_in=not_in) for k in value)
|
406
|
-
elif isinstance(value, (list, tuple, set)):
|
407
|
-
return any(not_in_template(i, not_in=not_in) for i in value)
|
408
|
-
elif not isinstance(value, str):
|
409
|
-
return False
|
410
|
-
return any(
|
411
|
-
(not found.caller.strip().startswith(not_in))
|
412
|
-
for found in Re.finditer_caller(value.strip())
|
413
|
-
)
|
414
|
-
|
415
|
-
|
416
|
-
def has_template(value: Any) -> bool:
|
417
|
-
"""Check value include templating string.
|
418
|
-
|
419
|
-
:param value: A value that want to find parameter template.
|
420
|
-
:rtype: bool
|
421
|
-
"""
|
422
|
-
if isinstance(value, dict):
|
423
|
-
return any(has_template(value[k]) for k in value)
|
424
|
-
elif isinstance(value, (list, tuple, set)):
|
425
|
-
return any(has_template(i) for i in value)
|
426
|
-
elif not isinstance(value, str):
|
427
|
-
return False
|
428
|
-
return bool(Re.RE_CALLER.findall(value.strip()))
|
429
|
-
|
430
|
-
|
431
|
-
def str2template(
|
432
|
-
value: str,
|
433
|
-
params: DictData,
|
434
|
-
*,
|
435
|
-
filters: dict[str, FilterRegistry] | None = None,
|
436
|
-
) -> Any:
|
437
|
-
"""(Sub-function) Pass param to template string that can search by
|
438
|
-
``RE_CALLER`` regular expression.
|
439
|
-
|
440
|
-
The getter value that map a template should have typing support align
|
441
|
-
with the workflow parameter types that is `str`, `int`, `datetime`, and
|
442
|
-
`list`.
|
443
|
-
|
444
|
-
:param value: A string value that want to mapped with an params
|
445
|
-
:param params: A parameter value that getting with matched regular
|
446
|
-
expression.
|
447
|
-
:param filters:
|
448
|
-
"""
|
449
|
-
filters: dict[str, FilterRegistry] = filters or make_filter_registry()
|
450
|
-
|
451
|
-
# NOTE: remove space before and after this string value.
|
452
|
-
value: str = value.strip()
|
453
|
-
for found in Re.finditer_caller(value):
|
454
|
-
# NOTE:
|
455
|
-
# Get caller and filter values that setting inside;
|
456
|
-
#
|
457
|
-
# ... ``${{ <caller-value> [ | <filter-value>] ... }}``
|
458
|
-
#
|
459
|
-
caller: str = found.caller
|
460
|
-
pfilter: list[str] = [
|
461
|
-
i.strip()
|
462
|
-
for i in (found.post_filters.strip().removeprefix("|").split("|"))
|
463
|
-
if i != ""
|
464
|
-
]
|
465
|
-
if not hasdot(caller, params):
|
466
|
-
raise UtilException(f"The params does not set caller: {caller!r}.")
|
467
|
-
|
468
|
-
# NOTE: from validate step, it guarantee that caller exists in params.
|
469
|
-
getter: Any = getdot(caller, params)
|
470
|
-
|
471
|
-
# NOTE:
|
472
|
-
# If type of getter caller is not string type and it does not use to
|
473
|
-
# concat other string value, it will return origin value from the
|
474
|
-
# ``getdot`` function.
|
475
|
-
if value.replace(found.full, "", 1) == "":
|
476
|
-
return map_post_filter(getter, pfilter, filters=filters)
|
477
|
-
|
478
|
-
# NOTE: map post-filter function.
|
479
|
-
getter: Any = map_post_filter(getter, pfilter, filters=filters)
|
480
|
-
if not isinstance(getter, str):
|
481
|
-
getter: str = str(getter)
|
482
|
-
|
483
|
-
value: str = value.replace(found.full, getter, 1)
|
484
|
-
|
485
|
-
return search_env_replace(value)
|
486
|
-
|
487
|
-
|
488
|
-
def param2template(
|
489
|
-
value: Any,
|
490
|
-
params: DictData,
|
491
|
-
) -> Any:
|
492
|
-
"""Pass param to template string that can search by ``RE_CALLER`` regular
|
493
|
-
expression.
|
494
|
-
|
495
|
-
:param value: A value that want to mapped with an params
|
496
|
-
:param params: A parameter value that getting with matched regular
|
497
|
-
expression.
|
498
|
-
|
499
|
-
:rtype: Any
|
500
|
-
:returns: An any getter value from the params input.
|
501
|
-
"""
|
502
|
-
filters: dict[str, FilterRegistry] = make_filter_registry()
|
503
|
-
if isinstance(value, dict):
|
504
|
-
return {k: param2template(value[k], params) for k in value}
|
505
|
-
elif isinstance(value, (list, tuple, set)):
|
506
|
-
return type(value)([param2template(i, params) for i in value])
|
507
|
-
elif not isinstance(value, str):
|
508
|
-
return value
|
509
|
-
return str2template(value, params, filters=filters)
|
510
|
-
|
511
|
-
|
512
124
|
def filter_func(value: Any) -> Any:
|
513
125
|
"""Filter out an own created function of any value of mapping context by
|
514
126
|
replacing it to its function name. If it is built-in function, it does not
|
ddeutil/workflow/workflow.py
CHANGED
@@ -48,12 +48,11 @@ from .exceptions import JobException, WorkflowException
|
|
48
48
|
from .job import Job
|
49
49
|
from .params import Param
|
50
50
|
from .result import Result
|
51
|
+
from .templates import has_template, param2template
|
51
52
|
from .utils import (
|
52
53
|
cut_id,
|
53
54
|
gen_id,
|
54
55
|
get_dt_now,
|
55
|
-
has_template,
|
56
|
-
param2template,
|
57
56
|
wait_a_minute,
|
58
57
|
)
|
59
58
|
|
@@ -486,7 +485,7 @@ class Workflow(BaseModel):
|
|
486
485
|
- Initialize WorkflowQueue and WorkflowRelease if they do not pass.
|
487
486
|
- Create release data for pass to parameter templating function.
|
488
487
|
- Execute this workflow with mapping release data to its parameters.
|
489
|
-
- Writing log
|
488
|
+
- Writing result log
|
490
489
|
- Remove this release on the running queue
|
491
490
|
- Push this release to complete queue
|
492
491
|
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: ddeutil-workflow
|
3
|
-
Version: 0.0.
|
3
|
+
Version: 0.0.27
|
4
4
|
Summary: Lightweight workflow orchestration with less dependencies
|
5
5
|
Author-email: ddeutils <korawich.anu@gmail.com>
|
6
6
|
License: MIT
|
@@ -168,29 +168,29 @@ The main configuration that use to dynamic changing with your propose of this
|
|
168
168
|
application. If any configuration values do not set yet, it will use default value
|
169
169
|
and do not raise any error to you.
|
170
170
|
|
171
|
-
| Environment | Component | Default
|
172
|
-
|
173
|
-
| **WORKFLOW_ROOT_PATH** | Core | `.`
|
174
|
-
| **WORKFLOW_CORE_REGISTRY** | Core | `src
|
175
|
-
| **WORKFLOW_CORE_REGISTRY_FILTER** | Core | `
|
176
|
-
| **WORKFLOW_CORE_PATH_CONF** | Core | `conf`
|
177
|
-
| **WORKFLOW_CORE_TIMEZONE** | Core | `Asia/Bangkok`
|
178
|
-
| **WORKFLOW_CORE_STAGE_DEFAULT_ID** | Core | `true`
|
179
|
-
| **WORKFLOW_CORE_STAGE_RAISE_ERROR** | Core | `false`
|
180
|
-
| **WORKFLOW_CORE_JOB_DEFAULT_ID** | Core | `false`
|
181
|
-
| **WORKFLOW_CORE_JOB_RAISE_ERROR** | Core | `true`
|
182
|
-
| **WORKFLOW_CORE_MAX_NUM_POKING** | Core | `4`
|
183
|
-
| **WORKFLOW_CORE_MAX_JOB_PARALLEL** | Core | `2`
|
184
|
-
| **WORKFLOW_CORE_MAX_JOB_EXEC_TIMEOUT** | Core | `600`
|
185
|
-
| **WORKFLOW_CORE_MAX_CRON_PER_WORKFLOW** | Core | `5`
|
186
|
-
| **WORKFLOW_CORE_MAX_QUEUE_COMPLETE_HIST** | Core | `16`
|
187
|
-
| **WORKFLOW_CORE_GENERATE_ID_SIMPLE_MODE** | Core | `true`
|
188
|
-
| **WORKFLOW_LOG_PATH** | Log | `./logs`
|
189
|
-
| **WORKFLOW_LOG_DEBUG_MODE** | Log | `true`
|
190
|
-
| **WORKFLOW_LOG_ENABLE_WRITE** | Log | `true`
|
191
|
-
| **WORKFLOW_APP_MAX_PROCESS** | Schedule | `2`
|
192
|
-
| **WORKFLOW_APP_MAX_SCHEDULE_PER_PROCESS** | Schedule | `100`
|
193
|
-
| **WORKFLOW_APP_STOP_BOUNDARY_DELTA** | Schedule | `'{"minutes": 5, "seconds": 20}'`
|
171
|
+
| Environment | Component | Default | Description | Remark |
|
172
|
+
|:-------------------------------------------|:---------:|:----------------------------------|:-------------------------------------------------------------------------------------------------------------------|--------|
|
173
|
+
| **WORKFLOW_ROOT_PATH** | Core | `.` | The root path of the workflow application. | |
|
174
|
+
| **WORKFLOW_CORE_REGISTRY** | Core | `src` | List of importable string for the hook stage. | |
|
175
|
+
| **WORKFLOW_CORE_REGISTRY_FILTER** | Core | `ddeutil.workflow.utils` | List of importable string for the filter template. | |
|
176
|
+
| **WORKFLOW_CORE_PATH_CONF** | Core | `conf` | The config path that keep all template `.yaml` files. | |
|
177
|
+
| **WORKFLOW_CORE_TIMEZONE** | Core | `Asia/Bangkok` | A Timezone string value that will pass to `ZoneInfo` object. | |
|
178
|
+
| **WORKFLOW_CORE_STAGE_DEFAULT_ID** | Core | `true` | A flag that enable default stage ID that use for catch an execution output. | |
|
179
|
+
| **WORKFLOW_CORE_STAGE_RAISE_ERROR** | Core | `false` | A flag that all stage raise StageException from stage execution. | |
|
180
|
+
| **WORKFLOW_CORE_JOB_DEFAULT_ID** | Core | `false` | A flag that enable default job ID that use for catch an execution output. The ID that use will be sequence number. | |
|
181
|
+
| **WORKFLOW_CORE_JOB_RAISE_ERROR** | Core | `true` | A flag that all job raise JobException from job strategy execution. | |
|
182
|
+
| **WORKFLOW_CORE_MAX_NUM_POKING** | Core | `4` | . | |
|
183
|
+
| **WORKFLOW_CORE_MAX_JOB_PARALLEL** | Core | `2` | The maximum job number that able to run parallel in workflow executor. | |
|
184
|
+
| **WORKFLOW_CORE_MAX_JOB_EXEC_TIMEOUT** | Core | `600` | | |
|
185
|
+
| **WORKFLOW_CORE_MAX_CRON_PER_WORKFLOW** | Core | `5` | | |
|
186
|
+
| **WORKFLOW_CORE_MAX_QUEUE_COMPLETE_HIST** | Core | `16` | | |
|
187
|
+
| **WORKFLOW_CORE_GENERATE_ID_SIMPLE_MODE** | Core | `true` | A flog that enable generating ID with `md5` algorithm. | |
|
188
|
+
| **WORKFLOW_LOG_PATH** | Log | `./logs` | The log path of the workflow saving log. | |
|
189
|
+
| **WORKFLOW_LOG_DEBUG_MODE** | Log | `true` | A flag that enable logging with debug level mode. | |
|
190
|
+
| **WORKFLOW_LOG_ENABLE_WRITE** | Log | `true` | A flag that enable logging object saving log to its destination. | |
|
191
|
+
| **WORKFLOW_APP_MAX_PROCESS** | Schedule | `2` | The maximum process worker number that run in scheduler app module. | |
|
192
|
+
| **WORKFLOW_APP_MAX_SCHEDULE_PER_PROCESS** | Schedule | `100` | A schedule per process that run parallel. | |
|
193
|
+
| **WORKFLOW_APP_STOP_BOUNDARY_DELTA** | Schedule | `'{"minutes": 5, "seconds": 20}'` | A time delta value that use to stop scheduler app in json string format. | |
|
194
194
|
|
195
195
|
**API Application**:
|
196
196
|
|
@@ -0,0 +1,25 @@
|
|
1
|
+
ddeutil/workflow/__about__.py,sha256=Tb1KYKrlWsfQNnf5zwGCHzEkcwD78iHOrO3PQIyUgTY,28
|
2
|
+
ddeutil/workflow/__cron.py,sha256=uA8XcbY_GwA9rJSHaHUaXaJyGDObJN0ZeYlJSinL8y8,26880
|
3
|
+
ddeutil/workflow/__init__.py,sha256=ATYXzGtLyq4LWCtJ-Odz36QSrLL7dKymVs8ziThOVOk,1582
|
4
|
+
ddeutil/workflow/__types.py,sha256=Ia7f38kvL3NibwmRKi0wQ1ud_45Z-SojYGhNJwIqcu8,3713
|
5
|
+
ddeutil/workflow/conf.py,sha256=jr7KPnt3vd7icuXTLGcJt_kT9tlmN1Cu5QBDMHUrm94,16819
|
6
|
+
ddeutil/workflow/cron.py,sha256=75A0hqevvouziKoLALncLJspVAeki9qCH3zniAJaxzY,7513
|
7
|
+
ddeutil/workflow/exceptions.py,sha256=NqnQJP52S59XIYMeXbTDbr4xH2UZ5EA3ejpU5Z4g6cQ,894
|
8
|
+
ddeutil/workflow/hook.py,sha256=yXhpr9E6ZzPb9_9ed79rWiRWDLnwkbRRg3TPmLvqoEI,4899
|
9
|
+
ddeutil/workflow/job.py,sha256=JJ4vSpuhQnY7LOMf9xq6N8pBZQ1oAxqYQFbKHn_HjdQ,24237
|
10
|
+
ddeutil/workflow/params.py,sha256=uPGkZx18E-iZ8BteqQ2ONgg0frhF3ZmP5cOyfK2j59U,5280
|
11
|
+
ddeutil/workflow/result.py,sha256=WIC8MsnfLiWNpZomT6jS4YCdYhlbIVVBjtGGe2dkoKk,3404
|
12
|
+
ddeutil/workflow/scheduler.py,sha256=BbY_3Y3QOdNwDfdvnRa7grGC2_a0Hn1KJbZKAscchk8,20454
|
13
|
+
ddeutil/workflow/stage.py,sha256=JJDuObNzpw803-bECmDzeGuaxQW2DnR0Ps8Tl0uJZnw,25033
|
14
|
+
ddeutil/workflow/templates.py,sha256=X-s5IZjwYpSD7UY3jaQiqbQBBG_Z3cWJDkzEIpicldg,10797
|
15
|
+
ddeutil/workflow/utils.py,sha256=jg0ZsbglrrF3bAakQ3rSna9KTMq2Qf_NPLnlORHf3J0,6039
|
16
|
+
ddeutil/workflow/workflow.py,sha256=QoSljQakGcumrx8l-W9yWuQZUTCrhArwAYktsj_L_9s,42204
|
17
|
+
ddeutil/workflow/api/__init__.py,sha256=F53NMBWtb9IKaDWkPU5KvybGGfKAcbehgn6TLBwHuuM,21
|
18
|
+
ddeutil/workflow/api/api.py,sha256=Md1cz3Edc7_uz63s_L_i-R3IE4mkO3aTADrX8GOGU-Y,5644
|
19
|
+
ddeutil/workflow/api/repeat.py,sha256=zyvsrXKk-3-_N8ZRZSki0Mueshugum2jtqctEOp9QSc,4927
|
20
|
+
ddeutil/workflow/api/route.py,sha256=v96jNbgjM1cJ2MpVSRWs2kgRqF8DQElEBdRZrVFEpEw,8578
|
21
|
+
ddeutil_workflow-0.0.27.dist-info/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
|
22
|
+
ddeutil_workflow-0.0.27.dist-info/METADATA,sha256=SC2dWZTZ0eruriXwbnhuCHhOg-0Z6wT1t026hJvsLnw,13921
|
23
|
+
ddeutil_workflow-0.0.27.dist-info/WHEEL,sha256=A3WOREP4zgxI0fKrHUG8DC8013e3dK3n7a6HDbcEIwE,91
|
24
|
+
ddeutil_workflow-0.0.27.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
|
25
|
+
ddeutil_workflow-0.0.27.dist-info/RECORD,,
|
@@ -1,23 +0,0 @@
|
|
1
|
-
ddeutil/workflow/__about__.py,sha256=jU_KFZf1uiZIWhuownbhRsjIL3oHGR_URL-jKTEnMKo,34
|
2
|
-
ddeutil/workflow/__cron.py,sha256=uA8XcbY_GwA9rJSHaHUaXaJyGDObJN0ZeYlJSinL8y8,26880
|
3
|
-
ddeutil/workflow/__init__.py,sha256=ozadVrqfqFRuukjv_zXUcgLANdiSrC6wrKkyVjdGg3w,1521
|
4
|
-
ddeutil/workflow/__types.py,sha256=Ia7f38kvL3NibwmRKi0wQ1ud_45Z-SojYGhNJwIqcu8,3713
|
5
|
-
ddeutil/workflow/conf.py,sha256=AU3GKTaxFFGDN-Sg8BGb08xj7vRBCTTjwk0FaORLJIk,16188
|
6
|
-
ddeutil/workflow/cron.py,sha256=75A0hqevvouziKoLALncLJspVAeki9qCH3zniAJaxzY,7513
|
7
|
-
ddeutil/workflow/exceptions.py,sha256=P56K7VD3etGm9y-k_GXrzEyqsTCaz9EJazTIshZDf9g,943
|
8
|
-
ddeutil/workflow/job.py,sha256=cvSLMdc1sMl1MeU7so7Oe2SdRYxQwt6hm55mLV1iP-Y,24219
|
9
|
-
ddeutil/workflow/params.py,sha256=uPGkZx18E-iZ8BteqQ2ONgg0frhF3ZmP5cOyfK2j59U,5280
|
10
|
-
ddeutil/workflow/result.py,sha256=WIC8MsnfLiWNpZomT6jS4YCdYhlbIVVBjtGGe2dkoKk,3404
|
11
|
-
ddeutil/workflow/scheduler.py,sha256=BbY_3Y3QOdNwDfdvnRa7grGC2_a0Hn1KJbZKAscchk8,20454
|
12
|
-
ddeutil/workflow/stage.py,sha256=a2sngzs9DkP6GU2pgAD3QvGoijyBQTR_pOhyJUIuWAo,26692
|
13
|
-
ddeutil/workflow/utils.py,sha256=pucRnCi9aLJDptXhzzReHZd5d-S0o5oZif5tr6H4iy8,18736
|
14
|
-
ddeutil/workflow/workflow.py,sha256=s6E-mKzSVQPTSV0biIAu5lFjslo6blKA-WTAjeOfLuw,42183
|
15
|
-
ddeutil/workflow/api/__init__.py,sha256=F53NMBWtb9IKaDWkPU5KvybGGfKAcbehgn6TLBwHuuM,21
|
16
|
-
ddeutil/workflow/api/api.py,sha256=Md1cz3Edc7_uz63s_L_i-R3IE4mkO3aTADrX8GOGU-Y,5644
|
17
|
-
ddeutil/workflow/api/repeat.py,sha256=zyvsrXKk-3-_N8ZRZSki0Mueshugum2jtqctEOp9QSc,4927
|
18
|
-
ddeutil/workflow/api/route.py,sha256=v96jNbgjM1cJ2MpVSRWs2kgRqF8DQElEBdRZrVFEpEw,8578
|
19
|
-
ddeutil_workflow-0.0.26.post1.dist-info/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
|
20
|
-
ddeutil_workflow-0.0.26.post1.dist-info/METADATA,sha256=B95z9M1Z9DWiKXQr1VoRvtlYcB6eX11RGktlAwn4MvI,14364
|
21
|
-
ddeutil_workflow-0.0.26.post1.dist-info/WHEEL,sha256=A3WOREP4zgxI0fKrHUG8DC8013e3dK3n7a6HDbcEIwE,91
|
22
|
-
ddeutil_workflow-0.0.26.post1.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
|
23
|
-
ddeutil_workflow-0.0.26.post1.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|