ddeutil-workflow 0.0.34__py3-none-any.whl → 0.0.36__py3-none-any.whl

ddeutil/workflow/params.py

@@ -3,37 +3,42 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
-"""Param Model that use for parsing incoming parameters that pass to the
-Workflow and Schedule objects.
+"""This module include all Param Pydantic Models that use for parsing an
+incoming parameters that was passed to the Workflow and Schedule objects before
+execution or release methods.
+
+    The Param model allow you to handle validation and preparation steps before
+passing an input value to target execution method.
 """
 from __future__ import annotations
 
 import decimal
-import logging
 from abc import ABC, abstractmethod
 from datetime import date, datetime
-from typing import Any, Literal, Optional, Union
+from typing import Annotated, Any, Literal, Optional, TypeVar, Union
 
 from pydantic import BaseModel, Field
 
 from .__types import TupleStr
 from .exceptions import ParamValueException
-from .utils import get_dt_now
-
-logger = logging.getLogger("ddeutil.workflow")
+from .utils import get_d_now, get_dt_now
 
 __all__: TupleStr = (
     "ChoiceParam",
     "DatetimeParam",
+    "DateParam",
     "IntParam",
     "Param",
     "StrParam",
 )
 
+T = TypeVar("T")
+
 
 class BaseParam(BaseModel, ABC):
-    """Base Parameter that use to make any Params Model. The type will dynamic
-    with the type field that made from literal string."""
+    """Base Parameter that use to make any Params Models. The parameter type
+    will dynamic with the setup type field that made from literal string.
+    """
 
     desc: Optional[str] = Field(
         default=None, description="A description of parameter providing."
@@ -45,7 +50,7 @@ class BaseParam(BaseModel, ABC):
     type: str = Field(description="A type of parameter.")
 
     @abstractmethod
-    def receive(self, value: Optional[Any] = None) -> Any:
+    def receive(self, value: Optional[T] = None) -> T:
         raise NotImplementedError(
             "Receive value and validate typing before return valid value."
         )
@@ -72,17 +77,42 @@ class DefaultParam(BaseParam):
         )
 
 
-# TODO: Not implement this parameter yet
 class DateParam(DefaultParam):  # pragma: no cov
-    """Date parameter."""
+    """Date parameter model."""
 
     type: Literal["date"] = "date"
+    default: date = Field(default_factory=get_d_now)
+
+    def receive(self, value: Optional[str | datetime | date] = None) -> date:
+        """Receive value that match with date. If an input value pass with
+        None, it will use default value instead.
+
+        :param value: A value that want to validate with date parameter type.
+
+        :rtype: date
+        """
+        if value is None:
+            return self.default
 
-    def receive(self, value: Optional[str | date] = None) -> date: ...
+        if isinstance(value, datetime):
+            return value.date()
+        elif isinstance(value, date):
+            return value
+        elif not isinstance(value, str):
+            raise ParamValueException(
+                f"Value that want to convert to date does not support for "
+                f"type: {type(value)}"
+            )
+        try:
+            return date.fromisoformat(value)
+        except ValueError:
+            raise ParamValueException(
+                f"Invalid the ISO format string for date: {value!r}"
+            ) from None
 
 
 class DatetimeParam(DefaultParam):
-    """Datetime parameter."""
+    """Datetime parameter model."""
 
     type: Literal["datetime"] = "datetime"
     default: datetime = Field(default_factory=get_dt_now)
@@ -93,6 +123,7 @@ class DatetimeParam(DefaultParam):
 
         :param value: A value that want to validate with datetime parameter
             type.
+
         :rtype: datetime
         """
         if value is None:
@@ -111,7 +142,7 @@ class DatetimeParam(DefaultParam):
             return datetime.fromisoformat(value)
         except ValueError:
             raise ParamValueException(
-                f"Invalid the ISO format string: {value!r}"
+                f"Invalid the ISO format string for datetime: {value!r}"
             ) from None
 
 
@@ -169,9 +200,11 @@ class ChoiceParam(BaseParam):
     """Choice parameter."""
 
     type: Literal["choice"] = "choice"
-    options: list[str] = Field(description="A list of choice parameters.")
+    options: Union[list[str], list[int]] = Field(
+        description="A list of choice parameters that able be str or int.",
+    )
 
-    def receive(self, value: str | None = None) -> str:
+    def receive(self, value: Union[str, int] | None = None) -> Union[str, int]:
         """Receive value that match with options.
 
         :param value: A value that want to select from the options field.
@@ -188,9 +221,41 @@ class ChoiceParam(BaseParam):
         return value
 
 
-Param = Union[
-    ChoiceParam,
-    DatetimeParam,
-    IntParam,
-    StrParam,
+# TODO: Not implement this parameter yet
+class MapParam(DefaultParam):  # pragma: no cov
+
+    type: Literal["map"] = "map"
+    default: dict[Any, Any] = Field(default_factory=dict)
+
+    def receive(self, value: Optional[dict[Any, Any]] = None) -> dict[Any, Any]:
+        if value is None:
+            return self.default
+
+
+# TODO: Not implement this parameter yet
+class ArrayParam(DefaultParam):  # pragma: no cov
+
+    type: Literal["array"] = "array"
+    default: list[Any] = Field(default_factory=list)
+
+    def receive(self, value: Optional[list[T]] = None) -> list[T]:
+        if value is None:
+            return self.default
+        if not isinstance(value, list):
+            raise ParamValueException(
+                f"Value that want to convert to array does not support for "
+                f"type: {type(value)}"
+            )
+        return value
+
+
+Param = Annotated[
+    Union[
+        ChoiceParam,
+        DatetimeParam,
+        DateParam,
+        IntParam,
+        StrParam,
+    ],
+    Field(discriminator="type"),
 ]

ddeutil/workflow/result.py

@@ -4,36 +4,29 @@
 # license information.
 # ------------------------------------------------------------------------------
 """This is the Result module. It is the data context transfer objects that use
-by all object in this package.
+by all object in this package. This module provide Result dataclass.
 """
 from __future__ import annotations
 
-import os
-from abc import ABC, abstractmethod
 from dataclasses import field
 from datetime import datetime
 from enum import IntEnum
-from inspect import Traceback, currentframe, getframeinfo
-from pathlib import Path
-from threading import Event, get_ident
+from threading import Event
 from typing import Optional
 
 from pydantic import ConfigDict
 from pydantic.dataclasses import dataclass
+from pydantic.functional_validators import model_validator
 from typing_extensions import Self
 
 from .__types import DictData, TupleStr
-from .conf import config, get_logger
-from .utils import cut_id, gen_id, get_dt_now
-
-logger = get_logger("ddeutil.workflow")
+from .logs import TraceLog, get_dt_tznow, get_trace
+from .utils import gen_id
 
 __all__: TupleStr = (
     "Result",
     "Status",
-    "TraceLog",
     "default_gen_id",
-    "get_dt_tznow",
 )
 
 
@@ -46,14 +39,6 @@ def default_gen_id() -> str:
     return gen_id("manual", unique=True)
 
 
-def get_dt_tznow() -> datetime:
-    """Return the current datetime object that passing the config timezone.
-
-    :rtype: datetime
-    """
-    return get_dt_now(tz=config.tz)
-
-
 class Status(IntEnum):
     """Status Int Enum object."""
 
@@ -62,111 +47,6 @@ class Status(IntEnum):
     WAIT: int = 2
 
 
-@dataclass(frozen=True)
-class BaseTraceLog(ABC):  # pragma: no cov
-    """Base Trace Log dataclass object."""
-
-    run_id: str
-    parent_run_id: Optional[str] = None
-
-    @abstractmethod
-    def writer(self, message: str, is_err: bool = False) -> None: ...
-
-    @abstractmethod
-    def make_message(self, message: str) -> str: ...
-
-    def debug(self, message: str):
-        msg: str = self.make_message(message)
-
-        # NOTE: Write file if debug mode.
-        if config.debug:
-            self.writer(msg)
-
-        logger.debug(msg, stacklevel=2)
-
-    def info(self, message: str):
-        msg: str = self.make_message(message)
-        self.writer(msg)
-        logger.info(msg, stacklevel=2)
-
-    def warning(self, message: str):
-        msg: str = self.make_message(message)
-        self.writer(msg)
-        logger.warning(msg, stacklevel=2)
-
-    def error(self, message: str):
-        msg: str = self.make_message(message)
-        self.writer(msg, is_err=True)
-        logger.error(msg, stacklevel=2)
-
-
-class TraceLog(BaseTraceLog):  # pragma: no cov
-    """Trace Log object that write file to the local storage."""
-
-    @property
-    def log_file(self) -> Path:
-        log_file: Path = (
-            config.log_path / f"run_id={self.parent_run_id or self.run_id}"
-        )
-        if not log_file.exists():
-            log_file.mkdir(parents=True)
-        return log_file
-
-    @property
-    def cut_id(self) -> str:
-        """Combine cutting ID of parent running ID if it set."""
-        cut_run_id: str = cut_id(self.run_id)
-        if not self.parent_run_id:
-            return f"{cut_run_id} -> {' ' * 6}"
-
-        cut_parent_run_id: str = cut_id(self.parent_run_id)
-        return f"{cut_parent_run_id} -> {cut_run_id}"
-
-    def make_message(self, message: str) -> str:
-        return f"({self.cut_id}) {message}"
-
-    def writer(self, message: str, is_err: bool = False) -> None:
-        """The path of logging data will store by format:
-
-            ... ./logs/run_id=<run-id>/stdout.txt
-            ... ./logs/run_id=<run-id>/stderr.txt
-
-        :param message:
-        :param is_err:
-        """
-        if not config.enable_write_log:
-            return
-
-        frame_info: Traceback = getframeinfo(currentframe().f_back.f_back)
-        filename: str = frame_info.filename.split(os.path.sep)[-1]
-        lineno: int = frame_info.lineno
-
-        # NOTE: set process and thread IDs.
-        process: int = os.getpid()
-        thread: int = get_ident()
-
-        write_file: str = "stderr.txt" if is_err else "stdout.txt"
-        with (self.log_file / write_file).open(
-            mode="at", encoding="utf-8"
-        ) as f:
-            msg_fmt: str = f"{config.log_format_file}\n"
-            print(msg_fmt)
-            f.write(
-                msg_fmt.format(
-                    **{
-                        "datetime": get_dt_tznow().strftime(
-                            config.log_datetime_format
-                        ),
-                        "process": process,
-                        "thread": thread,
-                        "message": message,
-                        "filename": filename,
-                        "lineno": lineno,
-                    }
-                )
-            )
-
-
 @dataclass(
     config=ConfigDict(arbitrary_types_allowed=True, use_enum_values=True)
 )
@@ -182,12 +62,12 @@ class Result:
     status: Status = field(default=Status.WAIT)
     context: DictData = field(default_factory=dict)
     run_id: Optional[str] = field(default_factory=default_gen_id)
-
-    # NOTE: Ignore this field to compare another result model with __eq__.
     parent_run_id: Optional[str] = field(default=None, compare=False)
-    event: Event = field(default_factory=Event, compare=False)
     ts: datetime = field(default_factory=get_dt_tznow, compare=False)
 
+    event: Event = field(default_factory=Event, compare=False, repr=False)
+    trace: Optional[TraceLog] = field(default=None, compare=False, repr=False)
+
     @classmethod
     def construct_with_rs_or_id(
         cls,
@@ -208,13 +88,12 @@ class Result:
             result.set_parent_run_id(parent_run_id)
         return result
 
-    def set_run_id(self, running_id: str) -> Self:
-        """Set a running ID.
+    @model_validator(mode="after")
+    def __prepare_trace(self) -> Self:
+        """Prepare trace field that want to pass after its initialize step."""
+        if self.trace is None:  # pragma: no cove
+            self.trace: TraceLog = get_trace(self.run_id, self.parent_run_id)
 
-        :param running_id: A running ID that want to update on this model.
-        :rtype: Self
-        """
-        self.run_id: str = running_id
         return self
 
     def set_parent_run_id(self, running_id: str) -> Self:
@@ -224,6 +103,7 @@ class Result:
         :rtype: Self
         """
         self.parent_run_id: str = running_id
+        self.trace: TraceLog = get_trace(self.run_id, running_id)
         return self
 
     def catch(
@@ -244,13 +124,9 @@ class Result:
         self.__dict__["context"].update(context or {})
         return self
 
-    @property
-    def trace(self) -> TraceLog:
-        """Return TraceLog object that passing its running ID.
+    def alive_time(self) -> float:  # pragma: no cov
+        """Return total seconds that this object use since it was created.
 
-        :rtype: TraceLog
+        :rtype: float
         """
-        return TraceLog(self.run_id, self.parent_run_id)
-
-    def alive_time(self) -> float:  # pragma: no cov
         return (get_dt_tznow() - self.ts).total_seconds()

ddeutil/workflow/scheduler.py

@@ -4,18 +4,18 @@
 # license information.
 # ------------------------------------------------------------------------------
 """
-The main schedule running is ``schedule_runner`` function that trigger the
-multiprocess of ``schedule_control`` function for listing schedules on the
-config by ``Loader.finds(Schedule)``.
+The main schedule running is `schedule_runner` function that trigger the
+multiprocess of `schedule_control` function for listing schedules on the
+config by `Loader.finds(Schedule)`.
 
-    The ``schedule_control`` is the scheduler function that release 2 schedule
-functions; ``workflow_task``, and ``workflow_monitor``.
+    The `schedule_control` is the scheduler function that release 2 schedule
+functions; `workflow_task`, and `workflow_monitor`.
 
-    ``schedule_control`` --- Every minute at :02 --> ``schedule_task``
-                         --- Every 5 minutes     --> ``monitor``
+    `schedule_control` ---( Every minute at :02 )--> `schedule_task`
+                       ---( Every 5 minutes     )--> `monitor`
 
-    The ``schedule_task`` will run ``task.release`` method in threading object
-for multithreading strategy. This ``release`` method will run only one crontab
+    The `schedule_task` will run `task.release` method in threading object
+for multithreading strategy. This `release` method will run only one crontab
 value with the on field.
 """
 from __future__ import annotations
@@ -134,7 +134,7 @@ class ScheduleWorkflow(BaseModel):
                 on: list[str] = [on]
 
             if any(not isinstance(n, (dict, str)) for n in on):
-                raise TypeError("The ``on`` key should be list of str or dict")
+                raise TypeError("The `on` key should be list of str or dict")
 
             # NOTE: Pass on value to Loader and keep on model object to on
             #   field.
@@ -344,7 +344,7 @@ class Schedule(BaseModel):
             tasks=self.tasks(
                 start_date_waiting, queue=queue, externals=externals
             ),
-            stop_date=stop_date,
+            stop=stop_date,
             queue=queue,
             threads=threads,
             result=result,
@@ -359,12 +359,16 @@ ReturnResultOrCancel = Callable[P, ResultOrCancel]
 DecoratorCancelJob = Callable[[ReturnResultOrCancel], ReturnResultOrCancel]
 
 
-def catch_exceptions(cancel_on_failure: bool = False) -> DecoratorCancelJob:
+def catch_exceptions(
+    cancel_on_failure: bool = False,
+    parent_run_id: str | None = None,
+) -> DecoratorCancelJob:
     """Catch exception error from scheduler job that running with schedule
     package and return CancelJob if this function raise an error.
 
     :param cancel_on_failure: A flag that allow to return the CancelJob or not
         it will raise.
+    :param parent_run_id:
 
     :rtype: DecoratorCancelJob
     """
@@ -375,10 +379,17 @@ def catch_exceptions(cancel_on_failure: bool = False) -> DecoratorCancelJob:
 
         @wraps(func)
         def wrapper(*args: P.args, **kwargs: P.kwargs) -> ResultOrCancel:
+
             try:
                 return func(*args, **kwargs)
+
             except Exception as err:
-                logger.exception(err)
+                if parent_run_id:
+                    (
+                        Result(parent_run_id=parent_run_id).trace.exception(
+                            str(err)
+                        )
+                    )
                 if cancel_on_failure:
                     return CancelJob
                 raise err
@@ -399,13 +410,13 @@ class ReleaseThread(TypedDict):
 ReleaseThreads = dict[str, ReleaseThread]
 
 
-@catch_exceptions(cancel_on_failure=True)
 def schedule_task(
     tasks: list[WorkflowTask],
     stop: datetime,
     queue: dict[str, ReleaseQueue],
     threads: ReleaseThreads,
     audit: type[Audit],
+    *,
     parent_run_id: str | None = None,
 ) -> ResultOrCancel:
     """Schedule task function that generate thread of workflow task release
@@ -491,8 +502,14 @@ def schedule_task(
         #   job.
         thread_name: str = f"{task.alias}|{release.date:%Y%m%d%H%M}"
         thread: Thread = Thread(
-            target=catch_exceptions(cancel_on_failure=True)(task.release),
-            kwargs={"release": release, "queue": q, "audit": audit},
+            target=catch_exceptions(
+                cancel_on_failure=True,
+            )(task.release),
+            kwargs={
+                "release": release,
+                "queue": q,
+                "audit": audit,
+            },
             name=thread_name,
             daemon=True,
         )
@@ -508,22 +525,28 @@ def schedule_task(
         delay()
 
     result.trace.debug(
-        f"[SCHEDULE]: End schedule task at {current_date:%Y-%m-%d %H:%M:%S} "
-        f"{'=' * 80}"
+        f"[SCHEDULE]: End schedule task that run since "
+        f"{current_date:%Y-%m-%d %H:%M:%S} {'=' * 30}"
     )
     return result.catch(
         status=Status.SUCCESS, context={"task_date": current_date}
     )
 
 
-def monitor(threads: ReleaseThreads) -> None:  # pragma: no cov
+def monitor(
+    threads: ReleaseThreads,
+    parent_run_id: str | None = None,
+) -> None:  # pragma: no cov
     """Monitoring function that running every five minute for track long-running
     thread instance from the schedule_control function that run every minute.
 
     :param threads: A mapping of Thread object and its name.
+    :param parent_run_id: A parent workflow running ID for this release.
+
     :type threads: ReleaseThreads
     """
-    logger.debug("[MONITOR]: Start checking long running schedule task.")
+    result: Result = Result().set_parent_run_id(parent_run_id)
+    result.trace.debug("[MONITOR]: Start checking long running schedule task.")
 
     snapshot_threads: list[str] = list(threads.keys())
     for thread_name in snapshot_threads:
@@ -538,20 +561,20 @@ def monitor(threads: ReleaseThreads) -> None:  # pragma: no cov
 
 def scheduler_pending(
     tasks: list[WorkflowTask],
-    stop_date,
-    queue,
-    threads,
+    stop: datetime,
+    queue: dict[str, ReleaseQueue],
+    threads: ReleaseThreads,
     result: Result,
     audit: type[Audit],
 ) -> Result:  # pragma: no cov
-    """
+    """Scheduler pending function.
 
-    :param tasks:
-    :param stop_date:
-    :param queue:
-    :param threads:
-    :param result:
-    :param audit:
+    :param tasks: A list of WorkflowTask object.
+    :param stop: A stop datetime object that force stop running scheduler.
+    :param queue: A mapping of alias name and ReleaseQueue object.
+    :param threads: A mapping of alias name and Thread object.
+    :param result: A result object.
+    :param audit: An audit class that want to make audit object.
 
     :rtype: Result
     """
@@ -569,9 +592,12 @@ def scheduler_pending(
         scheduler.every(1)
         .minutes.at(":02")
         .do(
-            schedule_task,
+            catch_exceptions(
+                cancel_on_failure=True,
+                parent_run_id=result.parent_run_id,
+            )(schedule_task),
             tasks=tasks,
-            stop=stop_date,
+            stop=stop,
             queue=queue,
             threads=threads,
             audit=audit,
@@ -588,13 +614,14 @@ def scheduler_pending(
         .do(
             monitor,
             threads=threads,
+            parent_run_id=result.parent_run_id,
         )
         .tag("monitor")
     )
 
     # NOTE: Start running schedule
     result.trace.info(
-        f"[SCHEDULE]: Schedule with stopper: {stop_date:%Y-%m-%d %H:%M:%S}"
+        f"[SCHEDULE]: Schedule with stopper: {stop:%Y-%m-%d %H:%M:%S}"
     )
 
     while True:
@@ -611,7 +638,7 @@ def scheduler_pending(
                     "running in background."
                 )
                 delay(10)
-                monitor(threads)
+                monitor(threads, parent_run_id=result.parent_run_id)
 
             break
 
@@ -681,7 +708,7 @@ def schedule_control(
 
     scheduler_pending(
         tasks=tasks,
-        stop_date=stop_date,
+        stop=stop_date,
         queue=queue,
         threads=threads,
         result=result,
@@ -707,15 +734,16 @@ def schedule_runner(
 
         This function will get all workflows that include on value that was
     created in config path and chuck it with application config variable
-    ``WORKFLOW_APP_MAX_SCHEDULE_PER_PROCESS`` env var to multiprocess executor
+    `WORKFLOW_APP_MAX_SCHEDULE_PER_PROCESS` env var to multiprocess executor
     pool.
 
         The current workflow logic that split to process will be below diagram:
 
-        MAIN ==> process 01 ==> schedule --> thread of release task 01 01
-                                        --> thread of release task 01 02
-                            ==> schedule --> thread of release task 02 01
-                                        --> thread of release task 02 02
+        MAIN ==> process 01 ==> schedule ==> thread 01 --> 01
+                                         ==> thread 01 --> 02
+                            ==> schedule ==> thread 02 --> 01
+                                         ==> thread 02 --> 02
+                                         ==> ...
             ==> process 02  ==> ...
 
     :rtype: Result
@@ -745,7 +773,7 @@ def schedule_runner(
 
             # NOTE: Raise error when it has any error from schedule_control.
             if err := future.exception():
-                logger.error(str(err))
+                result.trace.error(str(err))
                 raise WorkflowException(str(err)) from err
 
             rs: Result = future.result(timeout=1)