ddeutil-workflow 0.0.41__py3-none-any.whl → 0.0.43__py3-none-any.whl

ddeutil/workflow/logs.py

@@ -3,6 +3,7 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
+# [x] Use fix config
 """A Logs module contain TraceLog dataclass and AuditLog model.
 """
 from __future__ import annotations
@@ -160,7 +161,6 @@ class BaseTraceLog(ABC):  # pragma: no cov
         """
         msg: str = self.make_message(message)
 
-        # NOTE: Write file if debug mode was enabled.
         if config.debug:
             self.writer(msg)
 
@@ -206,6 +206,32 @@ class BaseTraceLog(ABC):  # pragma: no cov
         self.writer(msg, is_err=True)
         logger.exception(msg, stacklevel=2)
 
+    async def adebug(self, message: str) -> None:  # pragma: no cov
+        msg: str = self.make_message(message)
+        if config.debug:
+            await self.awriter(msg)
+        logger.info(msg, stacklevel=2)
+
+    async def ainfo(self, message: str) -> None:  # pragma: no cov
+        msg: str = self.make_message(message)
+        await self.awriter(msg)
+        logger.info(msg, stacklevel=2)
+
+    async def awarning(self, message: str) -> None:  # pragma: no cov
+        msg: str = self.make_message(message)
+        await self.awriter(msg)
+        logger.warning(msg, stacklevel=2)
+
+    async def aerror(self, message: str) -> None:  # pragma: no cov
+        msg: str = self.make_message(message)
+        await self.awriter(msg, is_err=True)
+        logger.error(msg, stacklevel=2)
+
+    async def aexception(self, message: str) -> None:  # pragma: no cov
+        msg: str = self.make_message(message)
+        await self.awriter(msg, is_err=True)
+        logger.exception(msg, stacklevel=2)
+
 
 class FileTraceLog(BaseTraceLog):  # pragma: no cov
     """Trace Log object that write file to the local storage."""
@@ -296,11 +322,13 @@ class FileTraceLog(BaseTraceLog):  # pragma: no cov
     async def awriter(
         self, message: str, is_err: bool = False
     ) -> None:  # pragma: no cov
-        """TODO: Use `aiofiles` for make writer method support async."""
         if not config.enable_write_log:
             return
 
-        import aiofiles
+        try:
+            import aiofiles
+        except ImportError as e:
+            raise ImportError("Async mode need aiofiles package") from e
 
         write_file: str = "stderr" if is_err else "stdout"
         trace_meta: TraceMeda = TraceMeda.make(mode=write_file, message=message)
@@ -468,9 +496,8 @@ class FileAudit(BaseAudit):
                 f"release={release:%Y%m%d%H%M%S} does not found."
             )
 
-        with max(pointer.glob("./*.log"), key=os.path.getctime).open(
-            mode="r", encoding="utf-8"
-        ) as f:
+        latest_file: Path = max(pointer.glob("./*.log"), key=os.path.getctime)
+        with latest_file.open(mode="r", encoding="utf-8") as f:
             return cls.model_validate(obj=json.load(f))
 
     @classmethod

ddeutil/workflow/params.py

@@ -3,7 +3,6 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
-# [ ] Use config
 """This module include all Param Pydantic Models that use for parsing an
 incoming parameters that was passed to the Workflow and Schedule objects before
 execution or release methods.
@@ -18,6 +17,7 @@ from abc import ABC, abstractmethod
 from datetime import date, datetime
 from typing import Annotated, Any, Literal, Optional, TypeVar, Union
 
+from ddeutil.core import str2dict, str2list
 from pydantic import BaseModel, Field
 
 from .__types import TupleStr
@@ -31,6 +31,8 @@ __all__: TupleStr = (
     "IntParam",
     "Param",
     "StrParam",
+    "ArrayParam",
+    "MapParam",
 )
 
 T = TypeVar("T")
@@ -42,7 +44,10 @@ class BaseParam(BaseModel, ABC):
     """
 
     desc: Optional[str] = Field(
-        default=None, description="A description of parameter providing."
+        default=None,
+        description=(
+            "A description of this parameter provide to the workflow model."
+        ),
     )
     required: bool = Field(
         default=True,
@@ -52,6 +57,7 @@ class BaseParam(BaseModel, ABC):
 
     @abstractmethod
     def receive(self, value: Optional[T] = None) -> T:
+        """Abstract method receive value to this parameter model."""
         raise NotImplementedError(
             "Receive value and validate typing before return valid value."
         )
@@ -66,13 +72,14 @@ class DefaultParam(BaseParam):
         default=False,
         description="A require flag for the default-able parameter value.",
     )
-    default: Optional[str] = Field(
+    default: Optional[Any] = Field(
         default=None,
         description="A default value if parameter does not pass.",
     )
 
     @abstractmethod
     def receive(self, value: Optional[Any] = None) -> Any:
+        """Abstract method receive value to this parameter model."""
         raise NotImplementedError(
             "Receive value and validate typing before return valid value."
         )
@@ -82,7 +89,10 @@ class DateParam(DefaultParam):  # pragma: no cov
     """Date parameter model."""
 
     type: Literal["date"] = "date"
-    default: date = Field(default_factory=get_d_now)
+    default: date = Field(
+        default_factory=get_d_now,
+        description="A default date that make from the current date func.",
+    )
 
     def receive(self, value: Optional[str | datetime | date] = None) -> date:
         """Receive value that match with date. If an input value pass with
@@ -116,7 +126,12 @@ class DatetimeParam(DefaultParam):
     """Datetime parameter model."""
 
     type: Literal["datetime"] = "datetime"
-    default: datetime = Field(default_factory=get_dt_now)
+    default: datetime = Field(
+        default_factory=get_dt_now,
+        description=(
+            "A default datetime that make from the current datetime func."
+        ),
+    )
 
     def receive(self, value: str | datetime | date | None = None) -> datetime:
         """Receive value that match with datetime. If an input value pass with
@@ -167,10 +182,6 @@ class IntParam(DefaultParam):
     """Integer parameter."""
 
     type: Literal["int"] = "int"
-    default: Optional[int] = Field(
-        default=None,
-        description="A default value if parameter does not pass.",
-    )
 
     def receive(self, value: int | None = None) -> int | None:
         """Receive value that match with int.
@@ -222,36 +233,84 @@ class ChoiceParam(BaseParam):
         return value
 
 
-# TODO: Not implement this parameter yet
 class MapParam(DefaultParam):  # pragma: no cov
+    """Map parameter."""
 
     type: Literal["map"] = "map"
-    default: dict[Any, Any] = Field(default_factory=dict)
+    default: dict[Any, Any] = Field(
+        default_factory=dict,
+        description="A default dict that make from the dict built-in func.",
+    )
+
+    def receive(
+        self,
+        value: Optional[Union[dict[Any, Any], str]] = None,
+    ) -> dict[Any, Any]:
+        """Receive value that match with map type.
 
-    def receive(self, value: Optional[dict[Any, Any]] = None) -> dict[Any, Any]:
+        :param value: A value that want to validate with map parameter type.
+        :rtype: dict[Any, Any]
+        """
         if value is None:
             return self.default
 
+        if isinstance(value, str):
+            try:
+                value: dict[Any, Any] = str2dict(value)
+            except ValueError as e:
+                raise ParamValueException(
+                    f"Value that want to convert to map does not support for "
+                    f"type: {type(value)}"
+                ) from e
+        elif not isinstance(value, dict):
+            raise ParamValueException(
+                f"Value of map param support only string-dict or dict type, "
+                f"not {type(value)}"
+            )
+        return value
+
 
-# TODO: Not implement this parameter yet
 class ArrayParam(DefaultParam):  # pragma: no cov
+    """Array parameter."""
 
     type: Literal["array"] = "array"
-    default: list[Any] = Field(default_factory=list)
+    default: list[Any] = Field(
+        default_factory=list,
+        description="A default list that make from the list built-in func.",
+    )
 
-    def receive(self, value: Optional[list[T]] = None) -> list[T]:
+    def receive(
+        self, value: Optional[Union[list[T], tuple[T, ...], str]] = None
+    ) -> list[T]:
+        """Receive value that match with array type.
+
+        :param value: A value that want to validate with array parameter type.
+        :rtype: list[Any]
+        """
         if value is None:
             return self.default
-        if not isinstance(value, list):
+        if isinstance(value, str):
+            try:
+                value: list[T] = str2list(value)
+            except ValueError as e:
+                raise ParamValueException(
+                    f"Value that want to convert to array does not support for "
+                    f"type: {type(value)}"
+                ) from e
+        elif isinstance(value, (tuple, set)):
+            return list(value)
+        elif not isinstance(value, list):
             raise ParamValueException(
-                f"Value that want to convert to array does not support for "
-                f"type: {type(value)}"
+                f"Value of map param support only string-list or list type, "
+                f"not {type(value)}"
             )
         return value
 
 
 Param = Annotated[
     Union[
+        MapParam,
+        ArrayParam,
         ChoiceParam,
         DatetimeParam,
         DateParam,

ddeutil/workflow/result.py

@@ -3,8 +3,8 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
-"""This is the Result module. It is the data context transfer objects that use
-by all object in this package. This module provide Result dataclass.
+"""Result module. It is the data context transfer objects that use by all object
+in this package. This module provide Status enum object and Result dataclass.
 """
 from __future__ import annotations
 
@@ -23,13 +23,19 @@ from .logs import TraceLog, get_dt_tznow, get_trace
 from .utils import default_gen_id, gen_id
 
 __all__: TupleStr = (
+    "SUCCESS",
+    "FAILED",
+    "WAIT",
+    "SKIP",
     "Result",
     "Status",
 )
 
 
 class Status(IntEnum):
-    """Status Int Enum object."""
+    """Status Int Enum object that use for tracking execution status to the
+    Result dataclass object.
+    """
 
     SUCCESS: int = 0
     FAILED: int = 1
@@ -37,8 +43,17 @@ class Status(IntEnum):
     SKIP: int = 3
 
 
+SUCCESS = Status.SUCCESS
+FAILED = Status.FAILED
+WAIT = Status.WAIT
+SKIP = Status.SKIP
+
+
 @dataclass(
-    config=ConfigDict(arbitrary_types_allowed=True, use_enum_values=True)
+    config=ConfigDict(
+        arbitrary_types_allowed=True,
+        use_enum_values=True,
+    ),
 )
 class Result:
     """Result Pydantic Model for passing and receiving data context from any
@@ -49,8 +64,9 @@ class Result:
     and ``_run_id`` fields to comparing with other result instance.
     """
 
-    status: Status = field(default=Status.WAIT)
+    status: Status = field(default=WAIT)
     context: DictData = field(default_factory=dict)
+    errors: DictData = field(default_factory=dict)
     run_id: Optional[str] = field(default_factory=default_gen_id)
     parent_run_id: Optional[str] = field(default=None, compare=False)
     ts: datetime = field(default_factory=get_dt_tznow, compare=False)
@@ -64,9 +80,16 @@ class Result:
         run_id: str | None = None,
         parent_run_id: str | None = None,
         id_logic: str | None = None,
-    ) -> Self:  # pragma: no cov
+    ) -> Self:
         """Create the Result object or set parent running id if passing Result
         object.
+
+        :param result:
+        :param run_id:
+        :param parent_run_id:
+        :param id_logic:
+
+        :rtype: Self
         """
         if result is None:
             result: Result = cls(
@@ -101,18 +124,23 @@ class Result:
         self,
         status: int | Status,
         context: DictData | None = None,
+        error: DictData | None = None,
     ) -> Self:
         """Catch the status and context to this Result object. This method will
         use between a child execution return a result, and it wants to pass
         status and context to this object.
 
-        :param status:
-        :param context:
+        :param status: A status enum object.
+        :param context: A context data that will update to the current context.
+        :param error: An error data that will update to the current errors.
+
+        :rtype: Self
         """
         self.__dict__["status"] = (
             Status(status) if isinstance(status, int) else status
         )
         self.__dict__["context"].update(context or {})
+        self.__dict__["errors"].update(error or {})
         return self
 
     def alive_time(self) -> float:  # pragma: no cov

ddeutil/workflow/reusables.py

@@ -3,7 +3,7 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
-# [x] Use config
+# [x] Use dynamic config
 """Reusables module that keep any templating functions."""
 from __future__ import annotations
 
@@ -25,7 +25,7 @@ from ddeutil.core import getdot, import_string, lazy
 from ddeutil.io import search_env_replace
 
 from .__types import DictData, Re
-from .conf import config
+from .conf import dynamic
 from .exceptions import UtilException
 
 T = TypeVar("T")
@@ -91,7 +91,7 @@ def make_filter_registry(
     :rtype: dict[str, FilterRegistry]
     """
     rs: dict[str, FilterRegistry] = {}
-    for module in registers or config.regis_filter:
+    for module in dynamic("regis_filter", f=registers):
         # NOTE: try to sequential import task functions
         try:
             importer = import_module(module)
@@ -277,7 +277,7 @@ def str2template(
     :rtype: str
     """
     filters: dict[str, FilterRegistry] = filters or make_filter_registry(
-        registers
+        registers=registers
     )
 
     # NOTE: remove space before and after this string value.
@@ -328,7 +328,7 @@ def param2template(
     params: DictData,
     filters: dict[str, FilterRegistry] | None = None,
     *,
-    registers: Optional[list[str]] = None,
+    extras: Optional[DictData] = None,
 ) -> T:
     """Pass param to template string that can search by ``RE_CALLER`` regular
     expression.
@@ -337,25 +337,25 @@ def param2template(
     :param params: A parameter value that getting with matched regular
         expression.
     :param filters: A filter mapping for mapping with `map_post_filter` func.
-    :param registers: (Optional[list[str]]) Override list of register.
+    :param extras: (Optional[list[str]]) An Override extras.
 
     :rtype: T
     :returns: An any getter value from the params input.
     """
+    registers: Optional[list[str]] = (
+        extras.get("regis_filter") if extras else None
+    )
     filters: dict[str, FilterRegistry] = filters or make_filter_registry(
-        registers
+        registers=registers
     )
     if isinstance(value, dict):
         return {
-            k: param2template(value[k], params, filters, registers=registers)
+            k: param2template(value[k], params, filters, extras=extras)
             for k in value
         }
     elif isinstance(value, (list, tuple, set)):
         return type(value)(
-            [
-                param2template(i, params, filters, registers=registers)
-                for i in value
-            ]
+            [param2template(i, params, filters, extras=extras) for i in value]
         )
     elif not isinstance(value, str):
         return value
@@ -447,8 +447,11 @@ def make_registry(
     :rtype: dict[str, Registry]
     """
     rs: dict[str, Registry] = {}
-    regis_calls: list[str] = registries or config.regis_call  # pragma: no cov
+    regis_calls: list[str] = dynamic(
+        "regis_call", f=registries
+    )  # pragma: no cov
     regis_calls.extend(["ddeutil.vendors"])
+
     for module in regis_calls:
         # NOTE: try to sequential import task functions
         try:

ddeutil/workflow/scheduler.py

@@ -3,8 +3,8 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
-"""
-The main schedule running is `schedule_runner` function that trigger the
+# [x] Use fix config
+"""The main schedule running is `schedule_runner` function that trigger the
 multiprocess of `schedule_control` function for listing schedules on the
 config by `Loader.finds(Schedule)`.
 
@@ -56,14 +56,13 @@ from .conf import Loader, SimLoad, config, get_logger
 from .cron import On
 from .exceptions import ScheduleException, WorkflowException
 from .logs import Audit, get_audit
-from .result import Result, Status
+from .result import SUCCESS, Result
 from .utils import batch, delay
 from .workflow import Release, ReleaseQueue, Workflow, WorkflowTask
 
 P = ParamSpec("P")
-logger = get_logger("ddeutil.workflow")
 
-# NOTE: Adjust logging level on the `schedule` package.
+logger = get_logger("ddeutil.workflow")
 logging.getLogger("schedule").setLevel(logging.INFO)
 
 
@@ -86,9 +85,9 @@ class ScheduleWorkflow(BaseModel):
     model.
 
         This on field does not equal to the on field of Workflow model, but it
-    uses same logic to generate running release date with crontab object. It use
-    for override the on field if the schedule time was change but you do not
-    want to change on the workflow model.
+    uses same logic to generate running release date with crontab object. It
+    uses for override the on field if the schedule time was change, but you do
+    not want to change on the workflow model.
     """
 
     alias: Optional[str] = Field(
@@ -177,7 +176,7 @@ class ScheduleWorkflow(BaseModel):
         start_date: datetime,
         queue: dict[str, ReleaseQueue],
         *,
-        externals: DictData | None = None,
+        extras: DictData | None = None,
     ) -> list[WorkflowTask]:
         """Return the list of WorkflowTask object from the specific input
         datetime that mapping with the on field.
@@ -187,17 +186,17 @@ class ScheduleWorkflow(BaseModel):
 
         :param start_date: A start date that get from the workflow schedule.
         :param queue: A mapping of name and list of datetime for queue.
-        :param externals: An external parameters that pass to the Loader object.
+        :param extras: An extra parameters that pass to the Loader object.
 
         :rtype: list[WorkflowTask]
         :return: Return the list of WorkflowTask object from the specific
             input datetime that mapping with the on field.
         """
         workflow_tasks: list[WorkflowTask] = []
-        extras: DictData = externals or {}
+        extras: DictData = extras or {}
 
         # NOTE: Loading workflow model from the name of workflow.
-        wf: Workflow = Workflow.from_loader(self.name, externals=extras)
+        wf: Workflow = Workflow.from_conf(self.name, extras=extras)
         wf_queue: ReleaseQueue = queue[self.alias]
 
         # IMPORTANT: Create the default 'on' value if it does not pass the `on`
@@ -254,24 +253,24 @@ class Schedule(BaseModel):
         return dedent(value)
 
     @classmethod
-    def from_loader(
+    def from_conf(
         cls,
         name: str,
-        externals: DictData | None = None,
+        extras: DictData | None = None,
     ) -> Self:
         """Create Schedule instance from the Loader object that only receive
         an input schedule name. The loader object will use this schedule name to
         searching configuration data of this schedule model in conf path.
 
         :param name: (str) A schedule name that want to pass to Loader object.
-        :param externals: An external parameters that want to pass to Loader
+        :param extras: An extra parameters that want to pass to Loader
             object.
 
         :raise ValueError: If the type does not match with current object.
 
         :rtype: Self
         """
-        loader: Loader = Loader(name, externals=(externals or {}))
+        loader: Loader = Loader(name, externals=(extras or {}))
 
         # NOTE: Validate the config type match with current connection model
         if loader.type != cls.__name__:
@@ -325,16 +324,16 @@ class Schedule(BaseModel):
         start_date: datetime,
         queue: dict[str, ReleaseQueue],
         *,
-        externals: DictData | None = None,
+        extras: DictData | None = None,
     ) -> list[WorkflowTask]:
         """Return the list of WorkflowTask object from the specific input
         datetime that mapping with the on field from workflow schedule model.
 
         :param start_date: A start date that get from the workflow schedule.
-        :param queue: A mapping of name and list of datetime for queue.
-        :type queue: dict[str, ReleaseQueue]
-        :param externals: An external parameters that pass to the Loader object.
-        :type externals: DictData | None
+        :param queue: (dict[str, ReleaseQueue]) A mapping of name and list of
+            datetime for queue.
+        :param extras: (DictData) An extra parameters that pass to the Loader
+            object.
 
         :rtype: list[WorkflowTask]
         :return: Return the list of WorkflowTask object from the specific
@@ -348,7 +347,7 @@ class Schedule(BaseModel):
                 queue[workflow.alias] = ReleaseQueue()
 
             workflow_tasks.extend(
-                workflow.tasks(start_date, queue=queue, externals=externals)
+                workflow.tasks(start_date, queue=queue, extras=extras)
             )
 
         return workflow_tasks
@@ -357,14 +356,14 @@ class Schedule(BaseModel):
         self,
         *,
         stop: datetime | None = None,
-        externals: DictData | None = None,
+        extras: DictData | None = None,
         audit: type[Audit] | None = None,
         parent_run_id: str | None = None,
     ) -> Result:  # pragma: no cov
         """Pending this schedule tasks with the schedule package.
 
         :param stop: A datetime value that use to stop running schedule.
-        :param externals: An external parameters that pass to Loader.
+        :param extras: An extra parameters that pass to Loader.
         :param audit: An audit class that use on the workflow task release for
             writing its release audit context.
         :param parent_run_id: A parent workflow running ID for this release.
@@ -385,9 +384,7 @@ class Schedule(BaseModel):
         ) + timedelta(minutes=1)
 
         scheduler_pending(
-            tasks=self.tasks(
-                start_date_waiting, queue=queue, externals=externals
-            ),
+            tasks=self.tasks(start_date_waiting, queue=queue, extras=extras),
             stop=stop_date,
             queue=queue,
             threads=threads,
@@ -395,7 +392,7 @@ class Schedule(BaseModel):
             audit=audit,
         )
 
-        return result.catch(status=Status.SUCCESS)
+        return result.catch(status=SUCCESS)
 
 
 ResultOrCancel = Union[type[CancelJob], Result]
@@ -574,9 +571,7 @@ def schedule_task(
         f"[SCHEDULE]: End schedule task that run since "
         f"{current_date:%Y-%m-%d %H:%M:%S} {'=' * 30}"
     )
-    return result.catch(
-        status=Status.SUCCESS, context={"task_date": current_date}
-    )
+    return result.catch(status=SUCCESS, context={"task_date": current_date})
 
 
 def monitor(
@@ -692,7 +687,7 @@ def scheduler_pending(
         f"[SCHEDULE]: Queue: {[list(queue[wf].queue) for wf in queue]}"
     )
     return result.catch(
-        status=Status.SUCCESS,
+        status=SUCCESS,
         context={
             "threads": [
                 {
@@ -709,7 +704,7 @@ def scheduler_pending(
 def schedule_control(
     schedules: list[str],
     stop: datetime | None = None,
-    externals: DictData | None = None,
+    extras: DictData | None = None,
     *,
     audit: type[Audit] | None = None,
     parent_run_id: str | None = None,
@@ -720,7 +715,7 @@ def schedule_control(
 
     :param schedules: A list of workflow names that want to schedule running.
     :param stop: A datetime value that use to stop running schedule.
-    :param externals: An external parameters that pass to Loader.
+    :param extras: An extra parameters that pass to Loader.
     :param audit: An audit class that use on the workflow task release for
         writing its release audit context.
     :param parent_run_id: A parent workflow running ID for this release.
@@ -745,10 +740,10 @@ def schedule_control(
     tasks: list[WorkflowTask] = []
     for name in schedules:
         tasks.extend(
-            Schedule.from_loader(name, externals=externals).tasks(
+            Schedule.from_conf(name, extras=extras).tasks(
                 start_date_waiting,
                 queue=queue,
-                externals=externals,
+                extras=extras,
             ),
         )
 
@@ -761,7 +756,7 @@ def schedule_control(
         audit=audit,
     )
 
-    return result.catch(status=Status.SUCCESS, context={"schedules": schedules})
+    return result.catch(status=SUCCESS, context={"schedules": schedules})
 
 
 def schedule_runner(