ddeutil-workflow 0.0.61__py3-none-any.whl → 0.0.63__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1 +1 @@
-__version__: str = "0.0.61"
+__version__: str = "0.0.63"

ddeutil/workflow/__cron.py

@@ -18,7 +18,7 @@ from ddeutil.core import (
     isinstance_check,
     must_split,
 )
-from ddeutil.core.dtutils import next_date, replace_date
+from ddeutil.core.dtutils import DatetimeMode, next_date, replace_date
 
 WEEKDAYS: dict[str, int] = {
     "Sun": 0,
@@ -31,7 +31,8 @@ WEEKDAYS: dict[str, int] = {
 }
 
 
-class CronYearLimit(Exception): ...
+class YearReachLimit(Exception):
+    """"""
 
 
 def str2cron(value: str) -> str:  # pragma: no cov
@@ -178,7 +179,7 @@ class CronPart:
     def __init__(
         self,
         unit: Unit,
-        values: str | list[int],
+        values: Union[str, list[int]],
         options: Options,
     ) -> None:
         self.unit: Unit = unit
@@ -229,19 +230,21 @@ class CronPart:
             f"(unit={self.unit}, values={self.__str__()!r})"
         )
 
-    def __lt__(self, other) -> bool:
+    def __lt__(self, other: Union[CronPart, list]) -> bool:
         """Override __lt__ method."""
         if isinstance(other, CronPart):
             return self.values < other.values
         elif isinstance(other, list):
             return self.values < other
+        return NotImplemented
 
-    def __eq__(self, other) -> bool:
+    def __eq__(self, other: Union[CronPart, list]) -> bool:
         """Override __eq__ method."""
         if isinstance(other, CronPart):
             return self.values == other.values
         elif isinstance(other, list):
             return self.values == other
+        return NotImplemented
 
     @property
     def min(self) -> int:
@@ -271,6 +274,7 @@ class CronPart:
             and (step := self.values[1] - self.values[0]) > 1
         ):
             return step
+        return None
 
     @property
     def is_full(self) -> bool:
@@ -355,6 +359,8 @@ class CronPart:
                     f"Invalid interval step value {value_step!r} for "
                     f"{self.unit.name!r}"
                 )
+            elif value_step:
+                value_step: int = int(value_step)
 
             # NOTE: Generate interval that has step
             interval_list.append(self._interval(value_range_list, value_step))
@@ -375,7 +381,9 @@ class CronPart:
                 value: str = value.replace(alt, str(self.unit.min + i))
         return value
 
-    def replace_weekday(self, values: list[int] | Iterator[int]) -> list[int]:
+    def replace_weekday(
+        self, values: Union[list[int], Iterator[int]]
+    ) -> list[int]:
         """Replaces all 7 with 0 as Sunday can be represented by both.
 
         :param values: list or iter of int that want to mode by 7
@@ -433,12 +441,12 @@ class CronPart:
     def _interval(
         self,
         values: list[int],
-        step: int | None = None,
+        step: Optional[int] = None,
     ) -> list[int]:
         """Applies an interval step to a collection of values.
 
         :param values:
-        :param step:
+        :param step: (int) A step
 
         :rtype: list[int]
         """
@@ -515,7 +523,7 @@ class CronPart:
                 start_number: Optional[int] = value
         return multi_dim_values
 
-    def filler(self, value: int) -> int | str:
+    def filler(self, value: int) -> Union[int, str]:
         """Formats weekday and month names as string when the relevant options
         are set.
 
@@ -765,12 +773,12 @@ class CronRunner:
 
     def __init__(
         self,
-        cron: CronJob | CronJobYear,
+        cron: Union[CronJob, CronJobYear],
         date: Optional[datetime] = None,
         *,
-        tz: str | ZoneInfo | None = None,
+        tz: Optional[Union[str, ZoneInfo]] = None,
     ) -> None:
-        self.tz: ZoneInfo | None = None
+        self.tz: Optional[ZoneInfo] = None
         if tz:
             if isinstance(tz, ZoneInfo):
                 self.tz = tz
@@ -810,7 +818,7 @@ class CronRunner:
             )
 
         self.__start_date: datetime = self.date
-        self.cron: CronJob | CronJobYear = cron
+        self.cron: Union[CronJob, CronJobYear] = cron
         self.is_year: bool = isinstance(cron, CronJobYear)
         self.reset_flag: bool = True
 
@@ -863,7 +871,7 @@ class CronRunner:
 
         raise RecursionError("Unable to find execution time for schedule")
 
-    def __shift_date(self, mode: str, reverse: bool = False) -> bool:
+    def __shift_date(self, mode: DatetimeMode, reverse: bool = False) -> bool:
         """Increments the mode of date value ("month", "day", "hour", "minute")
         until matches with the schedule.
 
@@ -897,8 +905,8 @@ class CronRunner:
                 getattr(self.date, mode)
                 > (max_year := max(self.cron.year.values))
             ):
-                raise CronYearLimit(
-                    f"The year is out of limit with this crontab value: "
+                raise YearReachLimit(
+                    f"The year is reach the limit with this crontab setting: "
                     f"{max_year}."
                 )
 
@@ -917,12 +925,3 @@ class CronRunner:
 
         # NOTE: Return False if the date that match with condition.
         return False
-
-
-__all__ = (
-    "CronJob",
-    "CronJobYear",
-    "CronRunner",
-    "Options",
-    "WEEKDAYS",
-)

ddeutil/workflow/__init__.py

@@ -5,12 +5,7 @@
 # ------------------------------------------------------------------------------
 from .__cron import CronJob, CronRunner
 from .__types import DictData, DictStr, Matrix, Re, TupleStr
-from .conf import (
-    Config,
-    FileLoad,
-    config,
-    env,
-)
+from .conf import *
 from .event import *
 from .exceptions import *
 from .job import *
@@ -37,24 +32,7 @@ from .result import (
     Result,
     Status,
 )
-from .reusables import (
-    FILTERS,
-    FilterFunc,
-    FilterRegistry,
-    ReturnTagFunc,
-    TagFunc,
-    custom_filter,
-    extract_call,
-    get_args_const,
-    has_template,
-    make_filter_registry,
-    make_registry,
-    map_post_filter,
-    not_in_template,
-    param2template,
-    str2template,
-    tag,
-)
+from .reusables import *
 from .scheduler import (
     Schedule,
     ScheduleWorkflow,

ddeutil/workflow/conf.py

@@ -18,8 +18,9 @@ from typing import Final, Optional, Protocol, TypeVar, Union
 from zoneinfo import ZoneInfo
 
 from ddeutil.core import str2bool
-from ddeutil.io import YamlFlResolve
+from ddeutil.io import YamlFlResolve, search_env_replace
 from ddeutil.io.paths import glob_files, is_ignored, read_ignore
+from pydantic import SecretStr
 
 from .__types import DictData
 
@@ -321,15 +322,16 @@ class FileLoad(BaseLoad):
         *,
         path: Optional[Path] = None,
         paths: Optional[list[Path]] = None,
-        excluded: list[str] | None = None,
+        excluded: Optional[list[str]] = None,
         extras: Optional[DictData] = None,
     ) -> Iterator[tuple[str, DictData]]:
         """Find all data that match with object type in config path. This class
         method can use include and exclude list of identity name for filter and
         adds-on.
 
-        :param obj: An object that want to validate matching before return.
-        :param path: A config path object.
+        :param obj: (object) An object that want to validate matching before
+            return.
+        :param path: (Path) A config path object.
         :param paths: (list[Path]) A list of config path object.
         :param excluded: An included list of data key that want to filter from
             data.
@@ -474,3 +476,33 @@ class Loader(Protocol):  # pragma: no cov
     def finds(
         cls, obj: object, *args, **kwargs
     ) -> Iterator[tuple[str, DictData]]: ...
+
+
+def pass_env(value: T) -> T:  # pragma: no cov
+    """Passing environment variable to an input value.
+
+    :param value: (Any) A value that want to pass env var searching.
+
+    :rtype: Any
+    """
+    if isinstance(value, dict):
+        return {k: pass_env(value[k]) for k in value}
+    elif isinstance(value, (list, tuple, set)):
+        return type(value)([pass_env(i) for i in value])
+    if not isinstance(value, str):
+        return value
+
+    rs: str = search_env_replace(value)
+    return None if rs == "null" else rs
+
+
+class WorkflowSecret(SecretStr):  # pragma: no cov
+    """Workflow Secret String model."""
+
+    def get_secret_value(self) -> str:
+        """Override get_secret_value by adding pass_env before return the
+        real-value.
+
+        :rtype: str
+        """
+        return pass_env(super().get_secret_value())

ddeutil/workflow/event.py

@@ -17,6 +17,7 @@ from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
 from pydantic import BaseModel, ConfigDict, Field, ValidationInfo
 from pydantic.functional_serializers import field_serializer
 from pydantic.functional_validators import field_validator, model_validator
+from pydantic_extra_types.timezone_name import TimeZoneName
 from typing_extensions import Self
 
 from .__cron import WEEKDAYS, CronJob, CronJobYear, CronRunner, Options
@@ -92,7 +93,7 @@ class Crontab(BaseModel):
         ),
     ]
     tz: Annotated[
-        str,
+        TimeZoneName,
         Field(
             description="A timezone string value",
             alias="timezone",
@@ -316,6 +317,17 @@ class CrontabYear(Crontab):
         )
 
 
+class ReleaseEvent(BaseModel):  # pragma: no cov
+    """Release trigger event."""
+
+    release: list[str] = Field(
+        description=(
+            "A list of workflow name that want to receive event from release"
+            "trigger."
+        )
+    )
+
+
 Event = Annotated[
     Union[
         CronJobYear,

ddeutil/workflow/reusables.py

@@ -4,7 +4,7 @@
 # license information.
 # ------------------------------------------------------------------------------
 # [x] Use dynamic config
-"""Reusables module that keep any templating functions."""
+"""Reusables module that keep any template and template filter functions."""
 from __future__ import annotations
 
 import copy
@@ -14,7 +14,16 @@ from ast import Call, Constant, Expr, Module, Name, parse
 from datetime import datetime
 from functools import wraps
 from importlib import import_module
-from typing import Any, Callable, Optional, Protocol, TypeVar, Union
+from typing import (
+    Any,
+    Callable,
+    Literal,
+    Optional,
+    Protocol,
+    TypeVar,
+    Union,
+    get_type_hints,
+)
 
 try:
     from typing import ParamSpec
@@ -23,6 +32,7 @@ except ImportError:
 
 from ddeutil.core import getdot, import_string, lazy
 from ddeutil.io import search_env_replace
+from pydantic import BaseModel, create_model
 from pydantic.dataclasses import dataclass
 
 from .__types import DictData, Re
@@ -32,7 +42,7 @@ from .exceptions import UtilException
 T = TypeVar("T")
 P = ParamSpec("P")
 
-logger = logging.getLogger("ddeutil.workflow")
+# NOTE: Adjust logging level of the `asyncio` to INFO level.
 logging.getLogger("asyncio").setLevel(logging.INFO)
 
 
@@ -40,12 +50,14 @@ FILTERS: dict[str, Callable] = {  # pragma: no cov
     "abs": abs,
     "str": str,
     "int": int,
+    "list": list,
+    "dict": dict,
     "title": lambda x: x.title(),
     "upper": lambda x: x.upper(),
     "lower": lambda x: x.lower(),
     "rstr": [str, repr],
-    "keys": lambda x: list(x.keys()),
-    "values": lambda x: list(x.values()),
+    "keys": lambda x: x.keys(),
+    "values": lambda x: x.values(),
 }
 
 
@@ -55,6 +67,7 @@ class FilterFunc(Protocol):
     """
 
     filter: str
+    mark: Literal["filter"] = "filter"
 
     def __call__(self, *args, **kwargs): ...  # pragma: no cov
 
@@ -73,6 +86,7 @@ def custom_filter(name: str) -> Callable[P, FilterFunc]:
 
     def func_internal(func: Callable[[...], Any]) -> FilterFunc:
         func.filter = name
+        func.mark = "filter"
 
         @wraps(func)
         def wrapped(*args, **kwargs):
@@ -104,7 +118,10 @@ def make_filter_registry(
         for fstr, func in inspect.getmembers(importer, inspect.isfunction):
             # NOTE: check function attribute that already set tag by
             #   ``utils.tag`` decorator.
-            if not hasattr(func, "filter"):
+            if not (
+                hasattr(func, "filter")
+                and str(getattr(func, "mark", "NOT SET")) == "filter"
+            ):
                 continue
 
             func: FilterFunc
@@ -209,11 +226,9 @@ def map_post_filter(
                     value: T = func(value)
             else:
                 value: T = f_func(value, *args, **kwargs)
-        except UtilException as err:
-            logger.warning(str(err))
+        except UtilException:
             raise
-        except Exception as err:
-            logger.warning(str(err))
+        except Exception:
             raise UtilException(
                 f"The post-filter: {func_name!r} does not fit with {value!r} "
                 f"(type: {type(value).__name__})."
@@ -226,6 +241,7 @@ def not_in_template(value: Any, *, not_in: str = "matrix.") -> bool:
 
     :param value: A value that want to find parameter template prefix.
     :param not_in: The not-in string that use in the `.startswith` function.
+        (Default is `matrix.`)
 
     :rtype: bool
     """
@@ -274,7 +290,7 @@ def str2template(
     :param value: (str) A string value that want to map with params.
     :param params: (DictData) A parameter value that getting with matched
         regular expression.
-    :param filters: A mapping of filter registry.
+    :param filters: (dict[str, FilterRegistry]) A mapping of filter registry.
     :param registers: (Optional[list[str]]) Override list of register.
 
     :rtype: str
@@ -299,12 +315,14 @@ def str2template(
         ]
 
         # NOTE: from validate step, it guarantees that caller exists in params.
+        #   I recommend to avoid logging params context on this case because it
+        #   can include secret value.
         try:
             getter: Any = getdot(caller, params)
-        except ValueError as err:
+        except ValueError:
             raise UtilException(
-                f"Params does not set caller: {caller!r}."
-            ) from err
+                f"Parameters does not get dot with caller: {caller!r}."
+            ) from None
 
         # NOTE:
         #   If type of getter caller is not string type, and it does not use to
@@ -332,17 +350,18 @@ def param2template(
     filters: Optional[dict[str, FilterRegistry]] = None,
     *,
     extras: Optional[DictData] = None,
-) -> T:
+) -> Any:
     """Pass param to template string that can search by ``RE_CALLER`` regular
     expression.
 
-    :param value: A value that want to map with params
-    :param params: A parameter value that getting with matched regular
-        expression.
-    :param filters: A filter mapping for mapping with `map_post_filter` func.
+    :param value: (Any) A value that want to map with params.
+    :param params: (DictData) A parameter value that getting with matched
+        regular expression.
+    :param filters: (dict[str, FilterRegistry]) A filter mapping for mapping
+        with `map_post_filter` func.
     :param extras: (Optional[list[str]]) An Override extras.
 
-    :rtype: T
+    :rtype: Any
     :returns: An any getter value from the params input.
     """
     registers: Optional[list[str]] = (
@@ -369,6 +388,11 @@ def param2template(
 def datetime_format(value: datetime, fmt: str = "%Y-%m-%d %H:%M:%S") -> str:
     """Format datetime object to string with the format.
 
+    Examples:
+
+        >>> "${{ start-date | fmt('%Y%m%d') }}"
+        >>> "${{ start-date | fmt }}"
+
     :param value: (datetime) A datetime value that want to format to string
         value.
     :param fmt: (str) A format string pattern that passing to the `dt.strftime`
@@ -385,15 +409,68 @@ def datetime_format(value: datetime, fmt: str = "%Y-%m-%d %H:%M:%S") -> str:
 
 @custom_filter("coalesce")  # pragma: no cov
 def coalesce(value: Optional[T], default: Any) -> T:
-    """Coalesce with default value if the main value is None."""
+    """Coalesce with default value if the main value is None.
+
+    Examples:
+
+        >>> "${{ value | coalesce('foo') }}"
+
+    :param value: A value that want to check nullable.
+    :param default: A default value that use to returned value if an input
+        value was null.
+    """
     return default if value is None else value
 
 
+@custom_filter("getitem")  # pragma: no cov
+def get_item(
+    value: DictData, key: Union[str, int], default: Optional[Any] = None
+) -> Any:
+    """Get a value with an input specific key.
+
+    Examples:
+
+        >>> "${{ value | getitem('key') }}"
+        >>> "${{ value | getitem('key', 'default') }}"
+
+    """
+    if not isinstance(value, dict):
+        raise UtilException(
+            f"The value that pass to `getitem` filter should be `dict` not "
+            f"`{type(value)}`."
+        )
+    return value.get(key, default)
+
+
+@custom_filter("getindex")  # pragma: no cov
+def get_index(value: list[Any], index: int) -> Any:
+    """Get a value with an input specific index.
+
+    Examples:
+
+        >>> "${{ value | getindex(1) }}"
+
+    """
+    if not isinstance(value, list):
+        raise UtilException(
+            f"The value that pass to `getindex` filter should be `list` not "
+            f"`{type(value)}`."
+        )
+    try:
+        return value[index]
+    except IndexError as e:
+        raise UtilException(
+            f"Index: {index} is out of range of value (The maximum range is "
+            f"{len(value)})."
+        ) from e
+
+
 class TagFunc(Protocol):
     """Tag Function Protocol"""
 
     name: str
     tag: str
+    mark: Literal["tag"] = "tag"
 
     def __call__(self, *args, **kwargs): ...  # pragma: no cov
 
@@ -421,6 +498,7 @@ def tag(
     def func_internal(func: Callable[[...], Any]) -> ReturnTagFunc:
         func.tag = name or "latest"
         func.name = alias or func.__name__.replace("_", "-")
+        func.mark = "tag"
 
         @wraps(func)
         def wrapped(*args: P.args, **kwargs: P.kwargs) -> TagFunc:
@@ -468,7 +546,9 @@ def make_registry(
             # NOTE: check function attribute that already set tag by
             #   ``utils.tag`` decorator.
             if not (
-                hasattr(func, "tag") and hasattr(func, "name")
+                hasattr(func, "tag")
+                and hasattr(func, "name")
+                and str(getattr(func, "mark", "NOT SET")) == "tag"
             ):  # pragma: no cov
                 continue
 
@@ -553,3 +633,32 @@ def extract_call(
             f"`REGISTER.{call.path}.registries.{call.func}`"
         )
     return rgt[call.func][call.tag]
+
+
+def create_model_from_caller(func: Callable) -> BaseModel:  # pragma: no cov
+    """Create model from the caller function. This function will use for
+    validate the caller function argument typed-hint that valid with the args
+    field.
+
+    :param func: A caller function.
+
+    :rtype: BaseModel
+    """
+    sig: inspect.Signature = inspect.signature(func)
+    type_hints: dict[str, Any] = get_type_hints(func)
+    fields: dict[str, Any] = {}
+    for name in sig.parameters:
+        param: inspect.Parameter = sig.parameters[name]
+        if param.kind in (
+            inspect.Parameter.VAR_KEYWORD,
+            inspect.Parameter.VAR_POSITIONAL,
+        ):
+            continue
+        if param.default != inspect.Parameter.empty:
+            fields[name] = (type_hints[name], param.default)
+        else:
+            fields[name] = (type_hints[name], ...)
+
+    return create_model(
+        "".join(i.title() for i in func.__name__.split("_")), **fields
+    )

ddeutil/workflow/stages.py

@@ -60,7 +60,7 @@ from pydantic.functional_validators import model_validator
 from typing_extensions import Self
 
 from .__types import DictData, DictStr, StrOrInt, StrOrNone, TupleStr
-from .conf import dynamic
+from .conf import dynamic, pass_env
 from .exceptions import StageException, to_dict
 from .result import CANCEL, FAILED, SUCCESS, WAIT, Result, Status
 from .reusables import TagFunc, extract_call, not_in_template, param2template
@@ -626,10 +626,10 @@ class BashStage(BaseAsyncStage):
             await f.write(f"#!/bin/{f_shebang}\n\n")
 
             # NOTE: add setting environment variable before bash skip statement.
-            await f.writelines([f"{k}='{env[k]}';\n" for k in env])
+            await f.writelines(pass_env([f"{k}='{env[k]}';\n" for k in env]))
 
             # NOTE: make sure that shell script file does not have `\r` char.
-            await f.write("\n" + bash.replace("\r\n", "\n"))
+            await f.write("\n" + pass_env(bash.replace("\r\n", "\n")))
 
         # NOTE: Make this .sh file able to executable.
         make_exec(f"./{f_name}")
@@ -662,10 +662,10 @@ class BashStage(BaseAsyncStage):
             f.write(f"#!/bin/{f_shebang}\n\n")
 
             # NOTE: add setting environment variable before bash skip statement.
-            f.writelines([f"{k}='{env[k]}';\n" for k in env])
+            f.writelines(pass_env([f"{k}='{env[k]}';\n" for k in env]))
 
             # NOTE: make sure that shell script file does not have `\r` char.
-            f.write("\n" + bash.replace("\r\n", "\n"))
+            f.write("\n" + pass_env(bash.replace("\r\n", "\n")))
 
         # NOTE: Make this .sh file able to executable.
         make_exec(f"./{f_name}")
@@ -895,7 +895,9 @@ class PyStage(BaseAsyncStage):
         # WARNING: The exec build-in function is very dangerous. So, it
         #   should use the re module to validate exec-string before running.
         exec(
-            param2template(dedent(self.run), params, extras=self.extras),
+            pass_env(
+                param2template(dedent(self.run), params, extras=self.extras)
+            ),
             gb,
             lc,
         )
@@ -1060,12 +1062,12 @@ class CallStage(BaseAsyncStage):
         args: DictData = {"result": result} | param2template(
             self.args, params, extras=self.extras
         )
-        ips = inspect.signature(call_func)
+        sig = inspect.signature(call_func)
         necessary_params: list[str] = []
         has_keyword: bool = False
-        for k in ips.parameters:
+        for k in sig.parameters:
             if (
-                v := ips.parameters[k]
+                v := sig.parameters[k]
             ).default == Parameter.empty and v.kind not in (
                 Parameter.VAR_KEYWORD,
                 Parameter.VAR_POSITIONAL,
@@ -1083,7 +1085,7 @@ class CallStage(BaseAsyncStage):
                 f"does not set to args, {list(args.keys())}."
             )
 
-        if "result" not in ips.parameters and not has_keyword:
+        if "result" not in sig.parameters and not has_keyword:
             args.pop("result")
 
         args = self.parse_model_args(call_func, args, result)
@@ -1149,12 +1151,12 @@ class CallStage(BaseAsyncStage):
         args: DictData = {"result": result} | param2template(
             self.args, params, extras=self.extras
         )
-        ips = inspect.signature(call_func)
+        sig = inspect.signature(call_func)
         necessary_params: list[str] = []
         has_keyword: bool = False
-        for k in ips.parameters:
+        for k in sig.parameters:
             if (
-                v := ips.parameters[k]
+                v := sig.parameters[k]
             ).default == Parameter.empty and v.kind not in (
                 Parameter.VAR_KEYWORD,
                 Parameter.VAR_POSITIONAL,
@@ -1172,7 +1174,7 @@ class CallStage(BaseAsyncStage):
                 f"does not set to args, {list(args.keys())}."
             )
 
-        if "result" not in ips.parameters and not has_keyword:
+        if "result" not in sig.parameters and not has_keyword:
             args.pop("result")
 
         args = self.parse_model_args(call_func, args, result)
@@ -1295,7 +1297,7 @@ class TriggerStage(BaseStage):
         _trigger: str = param2template(self.trigger, params, extras=self.extras)
         result.trace.info(f"[STAGE]: Execute Trigger-Stage: {_trigger!r}")
         rs: Result = Workflow.from_conf(
-            name=_trigger,
+            name=pass_env(_trigger),
             extras=self.extras | {"stage_raise_error": True},
         ).execute(
             params=param2template(self.params, params, extras=self.extras),
@@ -2417,9 +2419,11 @@ class DockerStage(BaseStage):  # pragma: no cov
         )
 
         resp = client.api.pull(
-            repository=f"{self.image}",
-            tag=self.tag,
-            auth_config=param2template(self.auth, params, extras=self.extras),
+            repository=pass_env(self.image),
+            tag=pass_env(self.tag),
+            auth_config=pass_env(
+                param2template(self.auth, params, extras=self.extras)
+            ),
             stream=True,
             decode=True,
         )
@@ -2438,10 +2442,10 @@ class DockerStage(BaseStage):  # pragma: no cov
 
         unique_image_name: str = f"{self.image}_{datetime.now():%Y%m%d%H%M%S%f}"
         container = client.containers.run(
-            image=f"{self.image}:{self.tag}",
+            image=pass_env(f"{self.image}:{self.tag}"),
             name=unique_image_name,
-            environment=self.env,
-            volumes=(
+            environment=pass_env(self.env),
+            volumes=pass_env(
                 {
                     Path.cwd()
                     / f".docker.{result.run_id}.logs": {
@@ -2549,8 +2553,10 @@ class VirtualPyStage(PyStage):  # pragma: no cov
         f_name: str = f"{run_id}.py"
         with open(f"./{f_name}", mode="w", newline="\n") as f:
             # NOTE: Create variable mapping that write before running statement.
-            vars_str: str = "\n ".join(
-                f"{var} = {value!r}" for var, value in values.items()
+            vars_str: str = pass_env(
+                "\n ".join(
+                    f"{var} = {value!r}" for var, value in values.items()
+                )
             )
 
             # NOTE: `uv` supports PEP 723 — inline TOML metadata.
@@ -2568,7 +2574,7 @@ class VirtualPyStage(PyStage):  # pragma: no cov
             )
 
             # NOTE: make sure that py script file does not have `\r` char.
-            f.write("\n" + py.replace("\r\n", "\n"))
+            f.write("\n" + pass_env(py.replace("\r\n", "\n")))
 
         # NOTE: Make this .py file able to executable.
         make_exec(f"./{f_name}")

ddeutil/workflow/workflow.py

@@ -774,7 +774,7 @@ class Workflow(BaseModel):
         parent_run_id: Optional[str] = None,
         result: Optional[Result] = None,
         event: Optional[Event] = None,
-        timeout: int = 3600,
+        timeout: float = 3600,
         max_job_parallel: int = 2,
     ) -> Result:
         """Execute workflow with passing a dynamic parameters to all jobs that
@@ -805,10 +805,10 @@ class Workflow(BaseModel):
         :param result: (Result) A Result instance for return context and status.
         :param event: (Event) An Event manager instance that use to cancel this
             execution if it forces stopped by parent execution.
-        :param timeout: (int) A workflow execution time out in second unit that
-            use for limit time of execution and waiting job dependency. This
-            value does not force stop the task that still running more than this
-            limit time. (Default: 60 * 60 seconds)
+        :param timeout: (float) A workflow execution time out in second unit
+            that use for limit time of execution and waiting job dependency.
+            This value does not force stop the task that still running more than
+            this limit time. (Default: 60 * 60 seconds)
         :param max_job_parallel: (int) The maximum workers that use for job
             execution in `PoolThreadExecutor` object. (Default: 2 workers)
 
@@ -840,7 +840,7 @@ class Workflow(BaseModel):
             job_queue.put(job_id)
 
         not_timeout_flag: bool = True
-        timeout: int = dynamic(
+        timeout: float = dynamic(
             "max_job_exec_timeout", f=timeout, extras=self.extras
         )
 

{ddeutil_workflow-0.0.61.dist-info → ddeutil_workflow-0.0.63.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddeutil-workflow
-Version: 0.0.61
+Version: 0.0.63
 Summary: Lightweight workflow orchestration
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT
@@ -25,6 +25,7 @@ License-File: LICENSE
 Requires-Dist: ddeutil[checksum]>=0.4.8
 Requires-Dist: ddeutil-io[toml,yaml]>=0.2.13
 Requires-Dist: pydantic==2.11.4
+Requires-Dist: pydantic-extra-types==2.10.4
 Requires-Dist: python-dotenv==1.1.0
 Requires-Dist: schedule<2.0.0,==1.2.2
 Provides-Extra: all
@@ -215,19 +216,23 @@ registry-caller/
 This function will store as module that will import from `WORKFLOW_CORE_REGISTRY_CALLER`
 value (This config can override by extra parameters with `registry_caller` key).
 
+> [!NOTE]
+> You can use Pydantic Model as argument of your caller function. The core workflow
+> engine will auto use the `model_validate` method before run your caller function.
+
 ```python
-from ddeutil.workflow import Result, tag
+from ddeutil.workflow import Result, WorkflowSecret, tag
 from ddeutil.workflow.exceptions import StageException
-from pydantic import BaseModel, SecretStr
+from pydantic import BaseModel
 
 class AwsCredential(BaseModel):
     path: str
     access_client_id: str
-    access_client_secret: SecretStr
+    access_client_secret: WorkflowSecret
 
 class RestAuth(BaseModel):
     type: str
-    keys: SecretStr
+    keys: WorkflowSecret
 
 @tag("requests", alias="get-api-with-oauth-to-s3")
 def get_api_with_oauth_to_s3(
@@ -243,6 +248,7 @@ def get_api_with_oauth_to_s3(
     result.trace.info(f"... {method}: {url}")
     if method != "post":
        raise StageException(f"RestAPI does not support for {method} action.")
+    # NOTE: If you want to use secret, you can use `auth.keys.get_secret_value()`.
     return {"records": 1000}
 ```
 

{ddeutil_workflow-0.0.61.dist-info → ddeutil_workflow-0.0.63.dist-info}/RECORD

@@ -1,20 +1,20 @@
-ddeutil/workflow/__about__.py,sha256=MeIsfJkRHwfi0J3-dkYVf-U5ewlNB1-grkzIr3wMoa0,28
-ddeutil/workflow/__cron.py,sha256=5DHQKejG-76L_oREW78RcwMzeyKddJxSMmBzYyMAeeY,28536
-ddeutil/workflow/__init__.py,sha256=NXEhjzKFdIGa-jtIq9HXChLCjSXNPd8VJ8ltggxbBO8,1371
+ddeutil/workflow/__about__.py,sha256=xnUKKC8j_qfLMiPiNf3P8eAETFo17HtTgo2_-q1Nkvw,28
+ddeutil/workflow/__cron.py,sha256=BOKQcreiex0SAigrK1gnLxpvOeF3aca_rQwyz9Kfve4,28751
+ddeutil/workflow/__init__.py,sha256=4QIAmK-p6Oex3VPrFCSwo8brhpL7_mEF3AQQlZdTqvs,1030
 ddeutil/workflow/__main__.py,sha256=x-sYedl4T8p6054aySk-EQX6vhytvPR0HvaBNYxMzp0,364
 ddeutil/workflow/__types.py,sha256=uNfoRbVmNK5O37UUMVnqcmoghD9oMS1q9fXC0APnjSI,4584
-ddeutil/workflow/conf.py,sha256=NLvjZ8bpDsn4e0MG3m1vgMdAwtmii5hP1D0STKQyZeo,14907
-ddeutil/workflow/event.py,sha256=ATQhCgx4F3I2SPQesXxLgOREEGtwkX6uc6jjViQ5pQg,10716
+ddeutil/workflow/conf.py,sha256=kjjFZmBK7SzSpQnMJ0KKzB2I0Zcn4oGA6uLdHRRroj0,15828
+ddeutil/workflow/event.py,sha256=S2eJAZZx_V5TuQ0l417hFVCtjWXnfNPZBgSCICzxQ48,11041
 ddeutil/workflow/exceptions.py,sha256=TKHBIlfquz3yEb8_kg6UXpxVLKxstt3QA9a1XYsLPJk,2455
 ddeutil/workflow/job.py,sha256=Php1b3n6c-jddel8PTSa61kAW22QBTetzoLVR4XXM4E,35240
 ddeutil/workflow/logs.py,sha256=iVtyl8i69y7t07tAuWkihc54WlkHCcBy_Ur0WtzJ_lM,31367
 ddeutil/workflow/params.py,sha256=1u8gXs1ZyMq-2eD9H8L7Yjfu5t7b_OzjA0fJvhxdYWY,12505
 ddeutil/workflow/result.py,sha256=4M9VCcveI8Yz6ZrnI-67SZlry-Z8G7e0hziy1k-pklk,5906
-ddeutil/workflow/reusables.py,sha256=JIXuAicRXhGuocQy71C7pjK4BTl2wo9lNK2p-zhEA6M,17743
+ddeutil/workflow/reusables.py,sha256=gbSHUptdEar5HSfBx13ldHJw0IBGF6BShQB3XpPd3Wg,20812
 ddeutil/workflow/scheduler.py,sha256=OsEyj2zscQ-3bDMk2z7UtKlCWLlgoGjaRFt17o1B1ew,27263
-ddeutil/workflow/stages.py,sha256=xH_f7IRohFCnUrtyD-QW86BlJ72p64JmpRTRrLPoF6A,93241
+ddeutil/workflow/stages.py,sha256=4JBcgXxxpDfwUFob30ocWwVqoxuk7t-WoDlj8dh8Dqs,93482
 ddeutil/workflow/utils.py,sha256=rcaDwXaEs4SCdcBKWx4ZCEtpnNfPI8du7Er6b_rg8t4,9569
-ddeutil/workflow/workflow.py,sha256=ty4PUhci7YCPlsNCH8qsaxbzHmBliCVmXczpwlAy_mk,44852
+ddeutil/workflow/workflow.py,sha256=8Z_h8OtNHkaGf8MJixTHNeXsyA4mBlYtHDqj0oEVFBs,44858
 ddeutil/workflow/api/__init__.py,sha256=kY30dL8HPY8tY_GBmm7y_3OdoXzB1-EA2a96PLU0AQw,5278
 ddeutil/workflow/api/logs.py,sha256=NMTnOnsBrDB5129329xF2myLdrb-z9k1MQrmrP7qXJw,1818
 ddeutil/workflow/api/utils.py,sha256=uTtUFVLpiYYahXvCVx8sueRQ03K2Xw1id_gW3IMmX1U,5295
@@ -23,9 +23,9 @@ ddeutil/workflow/api/routes/job.py,sha256=8X5VLDJH6PumyNIY6JGRNBsf2gWN0eG9DzxRPS
 ddeutil/workflow/api/routes/logs.py,sha256=U6vOni3wd-ZTOwd3yVdSOpgyRmNdcgfngU5KlLM3Cww,5383
 ddeutil/workflow/api/routes/schedules.py,sha256=14RnaJKEGMSJtncI1H_QQVZNBe_jDS40PPRO6qFc3i0,4805
 ddeutil/workflow/api/routes/workflows.py,sha256=GJu5PiXEylswrXylEImpncySjeU9chrvrtjhiMCw2RQ,4529
-ddeutil_workflow-0.0.61.dist-info/licenses/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
-ddeutil_workflow-0.0.61.dist-info/METADATA,sha256=LO7R_mpY7OkPg4J6epm2MwWn4qaI2HIMhTqw7T_LEhU,19427
-ddeutil_workflow-0.0.61.dist-info/WHEEL,sha256=DnLRTWE75wApRYVsjgc6wsVswC54sMSJhAEd4xhDpBk,91
-ddeutil_workflow-0.0.61.dist-info/entry_points.txt,sha256=qDTpPSauL0ciO6T4iSVt8bJeYrVEkkoEEw_RlGx6Kgk,63
-ddeutil_workflow-0.0.61.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
-ddeutil_workflow-0.0.61.dist-info/RECORD,,
+ddeutil_workflow-0.0.63.dist-info/licenses/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
+ddeutil_workflow-0.0.63.dist-info/METADATA,sha256=9h5mIv3IdcYoOG9Qaz02PgCm2KOJqjkDCS09WjDCAUo,19748
+ddeutil_workflow-0.0.63.dist-info/WHEEL,sha256=DnLRTWE75wApRYVsjgc6wsVswC54sMSJhAEd4xhDpBk,91
+ddeutil_workflow-0.0.63.dist-info/entry_points.txt,sha256=qDTpPSauL0ciO6T4iSVt8bJeYrVEkkoEEw_RlGx6Kgk,63
+ddeutil_workflow-0.0.63.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
+ddeutil_workflow-0.0.63.dist-info/RECORD,,