ddeutil-workflow 0.0.6__py3-none-any.whl → 0.0.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ddeutil/workflow/__about__.py +1 -1
- ddeutil/workflow/__init__.py +26 -4
- ddeutil/workflow/__types.py +11 -1
- ddeutil/workflow/api.py +120 -0
- ddeutil/workflow/app.py +41 -0
- ddeutil/workflow/exceptions.py +3 -0
- ddeutil/workflow/log.py +30 -0
- ddeutil/workflow/pipeline.py +341 -105
- ddeutil/workflow/repeat.py +134 -0
- ddeutil/workflow/route.py +78 -0
- ddeutil/workflow/stage.py +41 -12
- ddeutil/workflow/utils.py +280 -56
- {ddeutil_workflow-0.0.6.dist-info → ddeutil_workflow-0.0.7.dist-info}/METADATA +61 -14
- ddeutil_workflow-0.0.7.dist-info/RECORD +20 -0
- {ddeutil_workflow-0.0.6.dist-info → ddeutil_workflow-0.0.7.dist-info}/WHEEL +1 -1
- ddeutil_workflow-0.0.6.dist-info/RECORD +0 -15
- {ddeutil_workflow-0.0.6.dist-info → ddeutil_workflow-0.0.7.dist-info}/LICENSE +0 -0
- {ddeutil_workflow-0.0.6.dist-info → ddeutil_workflow-0.0.7.dist-info}/top_level.txt +0 -0
ddeutil/workflow/utils.py
CHANGED
@@ -6,9 +6,11 @@
|
|
6
6
|
from __future__ import annotations
|
7
7
|
|
8
8
|
import inspect
|
9
|
+
import logging
|
9
10
|
import os
|
10
11
|
import stat
|
11
12
|
from abc import ABC, abstractmethod
|
13
|
+
from ast import Call, Constant, Expr, Module, Name, parse
|
12
14
|
from collections.abc import Iterator
|
13
15
|
from dataclasses import dataclass, field
|
14
16
|
from datetime import date, datetime
|
@@ -20,14 +22,24 @@ from pathlib import Path
|
|
20
22
|
from typing import Any, Callable, Literal, Optional, Protocol, Union
|
21
23
|
from zoneinfo import ZoneInfo
|
22
24
|
|
23
|
-
from ddeutil.core import getdot, hasdot, lazy
|
24
|
-
from ddeutil.io import PathData
|
25
|
+
from ddeutil.core import getdot, hasdot, import_string, lazy
|
26
|
+
from ddeutil.io import PathData, search_env_replace
|
25
27
|
from ddeutil.io.models.lineage import dt_now
|
26
|
-
from pydantic import BaseModel, Field
|
28
|
+
from pydantic import BaseModel, ConfigDict, Field
|
27
29
|
from pydantic.functional_validators import model_validator
|
28
30
|
from typing_extensions import Self
|
29
31
|
|
30
32
|
from .__types import DictData, Matrix, Re
|
33
|
+
from .exceptions import ParamValueException, UtilException
|
34
|
+
|
35
|
+
|
36
|
+
def get_diff_sec(dt: datetime, tz: ZoneInfo | None = None) -> int:
|
37
|
+
"""Return second value that come from diff of an input datetime and the
|
38
|
+
current datetime with specific timezone.
|
39
|
+
"""
|
40
|
+
return round(
|
41
|
+
(dt - datetime.now(tz=(tz or ZoneInfo("UTC")))).total_seconds()
|
42
|
+
)
|
31
43
|
|
32
44
|
|
33
45
|
class Engine(BaseModel):
|
@@ -35,9 +47,10 @@ class Engine(BaseModel):
|
|
35
47
|
|
36
48
|
paths: PathData = Field(default_factory=PathData)
|
37
49
|
registry: list[str] = Field(
|
38
|
-
default_factory=lambda: [
|
39
|
-
|
40
|
-
|
50
|
+
default_factory=lambda: ["ddeutil.workflow"],
|
51
|
+
)
|
52
|
+
registry_filter: list[str] = Field(
|
53
|
+
default=lambda: ["ddeutil.workflow.utils"]
|
41
54
|
)
|
42
55
|
|
43
56
|
@model_validator(mode="before")
|
@@ -47,9 +60,21 @@ class Engine(BaseModel):
|
|
47
60
|
"""
|
48
61
|
if (_regis := values.get("registry")) and isinstance(_regis, str):
|
49
62
|
values["registry"] = [_regis]
|
63
|
+
if (_regis_filter := values.get("registry_filter")) and isinstance(
|
64
|
+
_regis, str
|
65
|
+
):
|
66
|
+
values["registry_filter"] = [_regis_filter]
|
50
67
|
return values
|
51
68
|
|
52
69
|
|
70
|
+
class CoreConf(BaseModel):
|
71
|
+
"""Core Config Model"""
|
72
|
+
|
73
|
+
model_config = ConfigDict(arbitrary_types_allowed=True)
|
74
|
+
|
75
|
+
tz: ZoneInfo = Field(default_factory=lambda: ZoneInfo("UTC"))
|
76
|
+
|
77
|
+
|
53
78
|
class ConfParams(BaseModel):
|
54
79
|
"""Params Model"""
|
55
80
|
|
@@ -57,16 +82,24 @@ class ConfParams(BaseModel):
|
|
57
82
|
default_factory=Engine,
|
58
83
|
description="A engine mapping values.",
|
59
84
|
)
|
85
|
+
core: CoreConf = Field(
|
86
|
+
default_factory=CoreConf,
|
87
|
+
description="A core config value",
|
88
|
+
)
|
60
89
|
|
61
90
|
|
62
91
|
def config() -> ConfParams:
|
63
92
|
"""Load Config data from ``workflows-conf.yaml`` file."""
|
64
93
|
root_path: str = os.getenv("WORKFLOW_ROOT_PATH", ".")
|
65
94
|
|
66
|
-
regis: list[str] = []
|
95
|
+
regis: list[str] = ["ddeutil.workflow"]
|
67
96
|
if regis_env := os.getenv("WORKFLOW_CORE_REGISTRY"):
|
68
97
|
regis = [r.strip() for r in regis_env.split(",")]
|
69
98
|
|
99
|
+
regis_filter: list[str] = ["ddeutil.workflow.utils"]
|
100
|
+
if regis_filter_env := os.getenv("WORKFLOW_CORE_REGISTRY_FILTER"):
|
101
|
+
regis_filter = [r.strip() for r in regis_filter_env.split(",")]
|
102
|
+
|
70
103
|
conf_path: str = (
|
71
104
|
f"{root_path}/{conf_env}"
|
72
105
|
if (conf_env := os.getenv("WORKFLOW_CORE_PATH_CONF"))
|
@@ -76,6 +109,7 @@ def config() -> ConfParams:
|
|
76
109
|
obj={
|
77
110
|
"engine": {
|
78
111
|
"registry": regis,
|
112
|
+
"registry_filter": regis_filter,
|
79
113
|
"paths": {
|
80
114
|
"root": root_path,
|
81
115
|
"conf": conf_path,
|
@@ -115,24 +149,24 @@ class TagFunc(Protocol):
|
|
115
149
|
def __call__(self, *args, **kwargs): ...
|
116
150
|
|
117
151
|
|
118
|
-
def tag(
|
152
|
+
def tag(name: str, alias: str | None = None):
|
119
153
|
"""Tag decorator function that set function attributes, ``tag`` and ``name``
|
120
154
|
for making registries variable.
|
121
155
|
|
122
|
-
:param:
|
123
|
-
:param:
|
156
|
+
:param: name: A tag value for make different use-case of a function.
|
157
|
+
:param: alias: A alias function name that keeping in registries. If this
|
158
|
+
value does not supply, it will use original function name from __name__.
|
124
159
|
"""
|
125
160
|
|
126
|
-
def func_internal(func:
|
127
|
-
func.tag =
|
128
|
-
func.name =
|
161
|
+
def func_internal(func: Callable[[...], Any]) -> TagFunc:
|
162
|
+
func.tag = name
|
163
|
+
func.name = alias or func.__name__.replace("_", "-")
|
129
164
|
|
130
165
|
@wraps(func)
|
131
166
|
def wrapped(*args, **kwargs):
|
167
|
+
# NOTE: Able to do anything before calling hook function.
|
132
168
|
return func(*args, **kwargs)
|
133
169
|
|
134
|
-
# TODO: pass result from a wrapped to Result model
|
135
|
-
# >>> return Result.model_validate(obj=wrapped)
|
136
170
|
return wrapped
|
137
171
|
|
138
172
|
return func_internal
|
@@ -145,6 +179,7 @@ def make_registry(submodule: str) -> dict[str, Registry]:
|
|
145
179
|
"""Return registries of all functions that able to called with task.
|
146
180
|
|
147
181
|
:param submodule: A module prefix that want to import registry.
|
182
|
+
:rtype: dict[str, Registry]
|
148
183
|
"""
|
149
184
|
rs: dict[str, Registry] = {}
|
150
185
|
for module in config().engine.registry:
|
@@ -185,7 +220,7 @@ class BaseParam(BaseModel, ABC):
|
|
185
220
|
|
186
221
|
@abstractmethod
|
187
222
|
def receive(self, value: Optional[Any] = None) -> Any:
|
188
|
-
raise
|
223
|
+
raise NotImplementedError(
|
189
224
|
"Receive value and validate typing before return valid value."
|
190
225
|
)
|
191
226
|
|
@@ -197,14 +232,14 @@ class DefaultParam(BaseParam):
|
|
197
232
|
|
198
233
|
@abstractmethod
|
199
234
|
def receive(self, value: Optional[Any] = None) -> Any:
|
200
|
-
raise
|
235
|
+
raise NotImplementedError(
|
201
236
|
"Receive value and validate typing before return valid value."
|
202
237
|
)
|
203
238
|
|
204
239
|
@model_validator(mode="after")
|
205
240
|
def check_default(self) -> Self:
|
206
241
|
if not self.required and self.default is None:
|
207
|
-
raise
|
242
|
+
raise ParamValueException(
|
208
243
|
"Default should set when this parameter does not required."
|
209
244
|
)
|
210
245
|
return self
|
@@ -218,6 +253,7 @@ class DatetimeParam(DefaultParam):
|
|
218
253
|
default: datetime = Field(default_factory=dt_now)
|
219
254
|
|
220
255
|
def receive(self, value: str | datetime | date | None = None) -> datetime:
|
256
|
+
"""Receive value that match with datetime."""
|
221
257
|
if value is None:
|
222
258
|
return self.default
|
223
259
|
|
@@ -226,7 +262,7 @@ class DatetimeParam(DefaultParam):
|
|
226
262
|
elif isinstance(value, date):
|
227
263
|
return datetime(value.year, value.month, value.day)
|
228
264
|
elif not isinstance(value, str):
|
229
|
-
raise
|
265
|
+
raise ParamValueException(
|
230
266
|
f"Value that want to convert to datetime does not support for "
|
231
267
|
f"type: {type(value)}"
|
232
268
|
)
|
@@ -239,6 +275,7 @@ class StrParam(DefaultParam):
|
|
239
275
|
type: Literal["str"] = "str"
|
240
276
|
|
241
277
|
def receive(self, value: Optional[str] = None) -> str | None:
|
278
|
+
"""Receive value that match with str."""
|
242
279
|
if value is None:
|
243
280
|
return self.default
|
244
281
|
return str(value)
|
@@ -250,13 +287,14 @@ class IntParam(DefaultParam):
|
|
250
287
|
type: Literal["int"] = "int"
|
251
288
|
|
252
289
|
def receive(self, value: Optional[int] = None) -> int | None:
|
290
|
+
"""Receive value that match with int."""
|
253
291
|
if value is None:
|
254
292
|
return self.default
|
255
293
|
if not isinstance(value, int):
|
256
294
|
try:
|
257
295
|
return int(str(value))
|
258
296
|
except TypeError as err:
|
259
|
-
raise
|
297
|
+
raise ParamValueException(
|
260
298
|
f"Value that want to convert to integer does not support "
|
261
299
|
f"for type: {type(value)}"
|
262
300
|
) from err
|
@@ -264,6 +302,8 @@ class IntParam(DefaultParam):
|
|
264
302
|
|
265
303
|
|
266
304
|
class ChoiceParam(BaseParam):
|
305
|
+
"""Choice parameter."""
|
306
|
+
|
267
307
|
type: Literal["choice"] = "choice"
|
268
308
|
options: list[str]
|
269
309
|
|
@@ -274,13 +314,16 @@ class ChoiceParam(BaseParam):
|
|
274
314
|
if value is None:
|
275
315
|
return self.options[0]
|
276
316
|
if any(value not in self.options):
|
277
|
-
raise
|
317
|
+
raise ParamValueException(
|
318
|
+
f"{value!r} does not match any value in choice options."
|
319
|
+
)
|
278
320
|
return value
|
279
321
|
|
280
322
|
|
281
323
|
Param = Union[
|
282
324
|
ChoiceParam,
|
283
325
|
DatetimeParam,
|
326
|
+
IntParam,
|
284
327
|
StrParam,
|
285
328
|
]
|
286
329
|
|
@@ -291,6 +334,11 @@ class Result:
|
|
291
334
|
the pipeline execution.
|
292
335
|
"""
|
293
336
|
|
337
|
+
# TODO: Add running ID to this result dataclass.
|
338
|
+
# ---
|
339
|
+
# parent_run_id: str
|
340
|
+
# run_id: str
|
341
|
+
#
|
294
342
|
status: int = field(default=2)
|
295
343
|
context: DictData = field(default_factory=dict)
|
296
344
|
|
@@ -301,11 +349,216 @@ def make_exec(path: str | Path):
|
|
301
349
|
f.chmod(f.stat().st_mode | stat.S_IEXEC)
|
302
350
|
|
303
351
|
|
304
|
-
|
352
|
+
FILTERS: dict[str, callable] = {
|
353
|
+
"abs": abs,
|
354
|
+
"str": str,
|
355
|
+
"int": int,
|
356
|
+
"upper": lambda x: x.upper(),
|
357
|
+
"lower": lambda x: x.lower(),
|
358
|
+
"rstr": [str, repr],
|
359
|
+
}
|
360
|
+
|
361
|
+
|
362
|
+
class FilterFunc(Protocol):
|
363
|
+
"""Tag Function Protocol"""
|
364
|
+
|
365
|
+
name: str
|
366
|
+
|
367
|
+
def __call__(self, *args, **kwargs): ...
|
368
|
+
|
369
|
+
|
370
|
+
def custom_filter(name: str):
|
371
|
+
"""Custom filter decorator function that set function attributes, ``filter``
|
372
|
+
for making filter registries variable.
|
373
|
+
|
374
|
+
:param: name: A filter name for make different use-case of a function.
|
375
|
+
"""
|
376
|
+
|
377
|
+
def func_internal(func: Callable[[...], Any]) -> TagFunc:
|
378
|
+
func.filter = name
|
379
|
+
|
380
|
+
@wraps(func)
|
381
|
+
def wrapped(*args, **kwargs):
|
382
|
+
# NOTE: Able to do anything before calling custom filter function.
|
383
|
+
return func(*args, **kwargs)
|
384
|
+
|
385
|
+
return wrapped
|
386
|
+
|
387
|
+
return func_internal
|
388
|
+
|
389
|
+
|
390
|
+
FilterRegistry = Union[FilterFunc, Callable[[...], Any]]
|
391
|
+
|
392
|
+
|
393
|
+
def make_filter_registry() -> dict[str, FilterRegistry]:
|
394
|
+
"""Return registries of all functions that able to called with task.
|
395
|
+
|
396
|
+
:rtype: dict[str, Registry]
|
397
|
+
"""
|
398
|
+
rs: dict[str, Registry] = {}
|
399
|
+
for module in config().engine.registry_filter:
|
400
|
+
# NOTE: try to sequential import task functions
|
401
|
+
try:
|
402
|
+
importer = import_module(module)
|
403
|
+
except ModuleNotFoundError:
|
404
|
+
continue
|
405
|
+
|
406
|
+
for fstr, func in inspect.getmembers(importer, inspect.isfunction):
|
407
|
+
# NOTE: check function attribute that already set tag by
|
408
|
+
# ``utils.tag`` decorator.
|
409
|
+
if not hasattr(func, "filter"):
|
410
|
+
continue
|
411
|
+
|
412
|
+
rs[func.filter] = import_string(f"{module}.{fstr}")
|
413
|
+
|
414
|
+
rs.update(FILTERS)
|
415
|
+
return rs
|
416
|
+
|
417
|
+
|
418
|
+
def get_args_const(
|
419
|
+
expr: str,
|
420
|
+
) -> tuple[str, list[Constant], dict[str, Constant]]:
|
421
|
+
"""Get arguments and keyword-arguments from function calling string."""
|
422
|
+
try:
|
423
|
+
mod: Module = parse(expr)
|
424
|
+
except SyntaxError:
|
425
|
+
raise UtilException(
|
426
|
+
f"Post-filter: {expr} does not valid because it raise syntax error."
|
427
|
+
) from None
|
428
|
+
body: list[Expr] = mod.body
|
429
|
+
|
430
|
+
if len(body) > 1:
|
431
|
+
raise UtilException(
|
432
|
+
"Post-filter function should be only one calling per pipe"
|
433
|
+
)
|
434
|
+
|
435
|
+
caller: Union[Name, Call]
|
436
|
+
if isinstance((caller := body[0].value), Name):
|
437
|
+
return caller.id, [], {}
|
438
|
+
elif not isinstance(caller, Call):
|
439
|
+
raise UtilException(
|
440
|
+
f"Get arguments does not support for caller type: {type(caller)}"
|
441
|
+
)
|
442
|
+
|
443
|
+
name: Name = caller.func
|
444
|
+
args: list[Constant] = caller.args
|
445
|
+
keywords: dict[str, Constant] = {k.arg: k.value for k in caller.keywords}
|
446
|
+
|
447
|
+
if any(not isinstance(i, Constant) for i in args):
|
448
|
+
raise UtilException("Argument should be constant.")
|
449
|
+
|
450
|
+
return name.id, args, keywords
|
451
|
+
|
452
|
+
|
453
|
+
@custom_filter("fmt")
|
454
|
+
def datetime_format(value: datetime, fmt: str = "%Y-%m-%d %H:%M:%S") -> str:
|
455
|
+
return value.strftime(fmt)
|
456
|
+
|
457
|
+
|
458
|
+
def map_post_filter(
|
305
459
|
value: Any,
|
306
|
-
|
460
|
+
post_filter: list[str],
|
461
|
+
filters: dict[str, FilterRegistry],
|
462
|
+
) -> Any:
|
463
|
+
"""Mapping post-filter to value with sequence list of filter function name
|
464
|
+
that will get from the filter registry.
|
465
|
+
|
466
|
+
:param value: A string value that want to mapped with filter function.
|
467
|
+
:param post_filter: A list of post-filter function name.
|
468
|
+
:param filters: A filter registry.
|
469
|
+
"""
|
470
|
+
for _filter in post_filter:
|
471
|
+
func_name, _args, _kwargs = get_args_const(_filter)
|
472
|
+
args = [arg.value for arg in _args]
|
473
|
+
kwargs = {k: v.value for k, v in _kwargs.items()}
|
474
|
+
|
475
|
+
if func_name not in filters:
|
476
|
+
raise UtilException(
|
477
|
+
f"The post-filter: {func_name} does not support yet."
|
478
|
+
)
|
479
|
+
|
480
|
+
try:
|
481
|
+
if isinstance((f_func := filters[func_name]), list):
|
482
|
+
if args or kwargs:
|
483
|
+
raise UtilException(
|
484
|
+
"Chain filter function does not support for passing "
|
485
|
+
"arguments."
|
486
|
+
)
|
487
|
+
for func in f_func:
|
488
|
+
value: Any = func(value)
|
489
|
+
else:
|
490
|
+
value: Any = f_func(value, *args, **kwargs)
|
491
|
+
except Exception as err:
|
492
|
+
logging.warning(str(err))
|
493
|
+
raise UtilException(
|
494
|
+
f"The post-filter function: {func_name} does not fit with "
|
495
|
+
f"{value} (type: {type(value).__name__})."
|
496
|
+
) from None
|
497
|
+
return value
|
498
|
+
|
499
|
+
|
500
|
+
def str2template(
|
501
|
+
value: str,
|
502
|
+
params: DictData,
|
307
503
|
*,
|
308
|
-
|
504
|
+
filters: dict[str, FilterRegistry] | None = None,
|
505
|
+
) -> Any:
|
506
|
+
"""(Sub-function) Pass param to template string that can search by
|
507
|
+
``RE_CALLER`` regular expression.
|
508
|
+
|
509
|
+
The getter value that map a template should have typing support align
|
510
|
+
with the pipeline parameter types that is `str`, `int`, `datetime`, and
|
511
|
+
`list`.
|
512
|
+
|
513
|
+
:param value: A string value that want to mapped with an params
|
514
|
+
:param params: A parameter value that getting with matched regular
|
515
|
+
expression.
|
516
|
+
:param filters:
|
517
|
+
"""
|
518
|
+
filters: dict[str, FilterRegistry] = filters or make_filter_registry()
|
519
|
+
|
520
|
+
# NOTE: remove space before and after this string value.
|
521
|
+
value: str = value.strip()
|
522
|
+
for found in Re.RE_CALLER.finditer(value):
|
523
|
+
# NOTE:
|
524
|
+
# Get caller and filter values that setting inside;
|
525
|
+
#
|
526
|
+
# ... ``${{ <caller-value> [ | <filter-value>] ... }}``
|
527
|
+
#
|
528
|
+
caller: str = found.group("caller")
|
529
|
+
pfilter: list[str] = [
|
530
|
+
i.strip()
|
531
|
+
for i in (
|
532
|
+
found.group("post_filters").strip().removeprefix("|").split("|")
|
533
|
+
)
|
534
|
+
if i != ""
|
535
|
+
]
|
536
|
+
if not hasdot(caller, params):
|
537
|
+
raise UtilException(f"The params does not set caller: {caller!r}.")
|
538
|
+
|
539
|
+
# NOTE: from validate step, it guarantee that caller exists in params.
|
540
|
+
getter: Any = getdot(caller, params)
|
541
|
+
|
542
|
+
# NOTE:
|
543
|
+
# If type of getter caller is not string type and it does not use to
|
544
|
+
# concat other string value, it will return origin value from the
|
545
|
+
# ``getdot`` function.
|
546
|
+
if value.replace(found.group(0), "", 1) == "":
|
547
|
+
return map_post_filter(getter, pfilter, filters=filters)
|
548
|
+
|
549
|
+
# NOTE: map post-filter function.
|
550
|
+
getter: Any = map_post_filter(getter, pfilter, filters=filters)
|
551
|
+
if not isinstance(getter, str):
|
552
|
+
getter: str = str(getter)
|
553
|
+
|
554
|
+
value: str = value.replace(found.group(0), getter, 1)
|
555
|
+
|
556
|
+
return search_env_replace(value)
|
557
|
+
|
558
|
+
|
559
|
+
def param2template(
|
560
|
+
value: Any,
|
561
|
+
params: DictData,
|
309
562
|
) -> Any:
|
310
563
|
"""Pass param to template string that can search by ``RE_CALLER`` regular
|
311
564
|
expression.
|
@@ -313,47 +566,18 @@ def param2template(
|
|
313
566
|
:param value: A value that want to mapped with an params
|
314
567
|
:param params: A parameter value that getting with matched regular
|
315
568
|
expression.
|
316
|
-
:param repr_flag: A repr flag for using repr instead of str if it set be
|
317
|
-
true.
|
318
569
|
|
319
570
|
:rtype: Any
|
320
571
|
:returns: An any getter value from the params input.
|
321
572
|
"""
|
573
|
+
filters: dict[str, FilterRegistry] = make_filter_registry()
|
322
574
|
if isinstance(value, dict):
|
323
575
|
return {k: param2template(value[k], params) for k in value}
|
324
576
|
elif isinstance(value, (list, tuple, set)):
|
325
577
|
return type(value)([param2template(i, params) for i in value])
|
326
578
|
elif not isinstance(value, str):
|
327
579
|
return value
|
328
|
-
|
329
|
-
if not Re.RE_CALLER.search(value):
|
330
|
-
return value
|
331
|
-
|
332
|
-
for found in Re.RE_CALLER.finditer(value):
|
333
|
-
|
334
|
-
# NOTE: get caller value that setting inside; ``${{ <caller-value> }}``
|
335
|
-
caller: str = found.group("caller")
|
336
|
-
if not hasdot(caller, params):
|
337
|
-
raise ValueError(f"params does not set caller: {caller!r}")
|
338
|
-
|
339
|
-
getter: Any = getdot(caller, params)
|
340
|
-
|
341
|
-
# NOTE: check type of vars
|
342
|
-
if isinstance(getter, (str, int)):
|
343
|
-
value: str = value.replace(
|
344
|
-
found.group(0), (repr(getter) if repr_flag else str(getter)), 1
|
345
|
-
)
|
346
|
-
continue
|
347
|
-
|
348
|
-
# NOTE:
|
349
|
-
# If type of getter caller does not formatting, it will return origin
|
350
|
-
# value from the ``getdot`` function.
|
351
|
-
if value.replace(found.group(0), "", 1) != "":
|
352
|
-
raise ValueError(
|
353
|
-
"Callable variable should not pass other outside ${{ ... }}"
|
354
|
-
)
|
355
|
-
return getter
|
356
|
-
return value
|
580
|
+
return str2template(value, params, filters=filters)
|
357
581
|
|
358
582
|
|
359
583
|
def dash2underscore(
|
@@ -368,7 +592,7 @@ def dash2underscore(
|
|
368
592
|
return values
|
369
593
|
|
370
594
|
|
371
|
-
def cross_product(matrix: Matrix) -> Iterator:
|
595
|
+
def cross_product(matrix: Matrix) -> Iterator[DictData]:
|
372
596
|
"""Iterator of products value from matrix."""
|
373
597
|
yield from (
|
374
598
|
{_k: _v for e in mapped for _k, _v in e.items()}
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: ddeutil-workflow
|
3
|
-
Version: 0.0.
|
3
|
+
Version: 0.0.7
|
4
4
|
Summary: Data Developer & Engineer Workflow Utility Objects
|
5
5
|
Author-email: ddeutils <korawich.anu@gmail.com>
|
6
6
|
License: MIT
|
@@ -24,9 +24,12 @@ License-File: LICENSE
|
|
24
24
|
Requires-Dist: fmtutil
|
25
25
|
Requires-Dist: ddeutil-io
|
26
26
|
Requires-Dist: python-dotenv ==1.0.1
|
27
|
+
Provides-Extra: api
|
28
|
+
Requires-Dist: fastapi[standard] ==0.112.0 ; extra == 'api'
|
29
|
+
Requires-Dist: apscheduler[sqlalchemy] <4.0.0,==3.10.4 ; extra == 'api'
|
30
|
+
Requires-Dist: croniter ==3.0.3 ; extra == 'api'
|
27
31
|
Provides-Extra: app
|
28
|
-
Requires-Dist:
|
29
|
-
Requires-Dist: apscheduler[sqlalchemy] ==3.10.4 ; extra == 'app'
|
32
|
+
Requires-Dist: schedule <2.0.0,==1.2.2 ; extra == 'app'
|
30
33
|
|
31
34
|
# Workflow
|
32
35
|
|
@@ -39,7 +42,6 @@ Requires-Dist: apscheduler[sqlalchemy] ==3.10.4 ; extra == 'app'
|
|
39
42
|
|
40
43
|
- [Installation](#installation)
|
41
44
|
- [Getting Started](#getting-started)
|
42
|
-
- [Core Features](#core-features)
|
43
45
|
- [On](#on)
|
44
46
|
- [Pipeline](#pipeline)
|
45
47
|
- [Usage](#usage)
|
@@ -50,12 +52,14 @@ Requires-Dist: apscheduler[sqlalchemy] ==3.10.4 ; extra == 'app'
|
|
50
52
|
- [Deployment](#deployment)
|
51
53
|
|
52
54
|
This **Workflow** objects was created for easy to make a simple metadata
|
53
|
-
driven pipeline that able to **ETL, T, EL, or
|
55
|
+
driven for data pipeline orchestration that able to use for **ETL, T, EL, or
|
56
|
+
ELT** by a `.yaml` file template.
|
54
57
|
|
55
|
-
I think
|
56
|
-
write
|
57
|
-
|
58
|
-
|
58
|
+
In my opinion, I think it should not create duplicate pipeline codes if I can
|
59
|
+
write with dynamic input parameters on the one template pipeline that just change
|
60
|
+
the input parameters per use-case instead.
|
61
|
+
This way I can handle a lot of logical pipelines in our orgs with only metadata
|
62
|
+
configuration. It called **Metadata Driven Data Pipeline**.
|
59
63
|
|
60
64
|
Next, we should get some monitoring tools for manage logging that return from
|
61
65
|
pipeline running. Because it not show us what is a use-case that running data
|
@@ -79,6 +83,10 @@ this package with application add-ons, you should add `app` in installation;
|
|
79
83
|
pip install ddeutil-workflow[app]
|
80
84
|
```
|
81
85
|
|
86
|
+
```shell
|
87
|
+
pip install ddeutil-workflow[api]
|
88
|
+
```
|
89
|
+
|
82
90
|
## Getting Started
|
83
91
|
|
84
92
|
The first step, you should start create the connections and datasets for In and
|
@@ -240,6 +248,18 @@ pipe_el_pg_to_lake:
|
|
240
248
|
endpoint: "/${{ params.name }}"
|
241
249
|
```
|
242
250
|
|
251
|
+
Implement hook:
|
252
|
+
|
253
|
+
```python
|
254
|
+
from ddeutil.workflow.utils import tag
|
255
|
+
|
256
|
+
@tag('polars', alias='postgres-to-delta')
|
257
|
+
def postgres_to_delta(source, sink):
|
258
|
+
return {
|
259
|
+
"source": source, "sink": sink
|
260
|
+
}
|
261
|
+
```
|
262
|
+
|
243
263
|
### Hook (Transform)
|
244
264
|
|
245
265
|
```yaml
|
@@ -265,12 +285,30 @@ pipeline_hook_mssql_proc:
|
|
265
285
|
target: ${{ params.target_name }}
|
266
286
|
```
|
267
287
|
|
288
|
+
Implement hook:
|
289
|
+
|
290
|
+
```python
|
291
|
+
from ddeutil.workflow.utils import tag
|
292
|
+
|
293
|
+
@tag('odbc', alias='mssql-proc')
|
294
|
+
def odbc_mssql_procedure(_exec: str, params: dict):
|
295
|
+
return {
|
296
|
+
"exec": _exec, "params": params
|
297
|
+
}
|
298
|
+
```
|
299
|
+
|
268
300
|
## Configuration
|
269
301
|
|
270
302
|
```bash
|
271
303
|
export WORKFLOW_ROOT_PATH=.
|
272
304
|
export WORKFLOW_CORE_REGISTRY=ddeutil.workflow,tests.utils
|
305
|
+
export WORKFLOW_CORE_REGISTRY_FILTER=ddeutil.workflow.utils
|
273
306
|
export WORKFLOW_CORE_PATH_CONF=conf
|
307
|
+
export WORKFLOW_CORE_TIMEZONE=Asia/Bangkok
|
308
|
+
export WORKFLOW_CORE_DEFAULT_STAGE_ID=true
|
309
|
+
|
310
|
+
export WORKFLOW_CORE_MAX_PIPELINE_POKING=4
|
311
|
+
export WORKFLOW_CORE_MAX_JOB_PARALLEL=2
|
274
312
|
```
|
275
313
|
|
276
314
|
Application config:
|
@@ -283,12 +321,21 @@ export WORKFLOW_APP_INTERVAL=10
|
|
283
321
|
## Deployment
|
284
322
|
|
285
323
|
This package able to run as a application service for receive manual trigger
|
286
|
-
from the master node via RestAPI
|
324
|
+
from the master node via RestAPI or use to be Scheduler background service
|
325
|
+
like crontab job but via Python API.
|
326
|
+
|
327
|
+
### Schedule Service
|
287
328
|
|
288
|
-
|
289
|
-
|
290
|
-
|
329
|
+
```shell
|
330
|
+
(venv) $ python src.ddeutil.workflow.app
|
331
|
+
```
|
332
|
+
|
333
|
+
### API Server
|
291
334
|
|
292
335
|
```shell
|
293
|
-
(venv) $ workflow
|
336
|
+
(venv) $ uvicorn src.ddeutil.workflow.api:app --host 0.0.0.0 --port 80 --reload
|
294
337
|
```
|
338
|
+
|
339
|
+
> [!NOTE]
|
340
|
+
> If this package already deploy, it able to use
|
341
|
+
> `uvicorn ddeutil.workflow.api:app --host 0.0.0.0 --port 80`
|
@@ -0,0 +1,20 @@
|
|
1
|
+
ddeutil/workflow/__about__.py,sha256=b23XabBwtuoPOLmS_Hj_gSA4LZ0fRfAkACM6c3szVoc,27
|
2
|
+
ddeutil/workflow/__init__.py,sha256=4PEL3RdHmUowK0Dz-tK7fO0wvFX4u9CLd0Up7b3lrAQ,760
|
3
|
+
ddeutil/workflow/__types.py,sha256=SYMoxbENQX8uPsiCZkjtpHAqqHOh8rUrarAFicAJd0E,1773
|
4
|
+
ddeutil/workflow/api.py,sha256=d2Mmv9jTtN3FITIy-2mivyAKdBOGZxtkNWRMPbCLlFI,3341
|
5
|
+
ddeutil/workflow/app.py,sha256=GbdwvUkE8lO2Ze4pZ0-J-7p9mcZAaORfjkHwW_oZIP0,1076
|
6
|
+
ddeutil/workflow/exceptions.py,sha256=BH7COn_3uz3z7oJBZOQGiuo8osBFgeXL8HYymnjCOPQ,671
|
7
|
+
ddeutil/workflow/loader.py,sha256=_ZD-XP5P7VbUeqItrUVPaKIZu6dMUZ2aywbCbReW1hQ,2778
|
8
|
+
ddeutil/workflow/log.py,sha256=_GJEdJr7bqpcQDxZjrqHd-hkiW3NKFaVoR6voE6Ty0o,952
|
9
|
+
ddeutil/workflow/on.py,sha256=YoEqDbzJUwqOA3JRltbvlYr0rNTtxdmb7cWMxl8U19k,6717
|
10
|
+
ddeutil/workflow/pipeline.py,sha256=dKF09TFS_v5TCD-5o8tp1UhB5sGuWIQu4zl_UFtlIC0,25951
|
11
|
+
ddeutil/workflow/repeat.py,sha256=sNoRfbOR4cYm_edrSvlVy9N8Dk_osLIq9FC5GMZz32M,4621
|
12
|
+
ddeutil/workflow/route.py,sha256=Ck_O1xJwI-vKkMJr37El0-1PGKlwKF8__DDNWVQrf0A,2079
|
13
|
+
ddeutil/workflow/scheduler.py,sha256=FqmkvWCqwJ4eRf8aDn5Ce4FcNWqmcvu2aTTfL34lfgs,22184
|
14
|
+
ddeutil/workflow/stage.py,sha256=z05bKk2QFQDXjidSnQYCVOdceSpSO13sHXE0B1UH6XA,14978
|
15
|
+
ddeutil/workflow/utils.py,sha256=pDM2jaYVP-USH0pLd_XmHOguxVPGVzZ76hOh1AZdINU,18495
|
16
|
+
ddeutil_workflow-0.0.7.dist-info/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
|
17
|
+
ddeutil_workflow-0.0.7.dist-info/METADATA,sha256=ba2nH57cpHB2P4ldQCRT8ZWDj3r1OPx9a1dgcB0a2Ws,9702
|
18
|
+
ddeutil_workflow-0.0.7.dist-info/WHEEL,sha256=HiCZjzuy6Dw0hdX5R3LCFPDmFS4BWl8H-8W39XfmgX4,91
|
19
|
+
ddeutil_workflow-0.0.7.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
|
20
|
+
ddeutil_workflow-0.0.7.dist-info/RECORD,,
|