ddeutil-workflow 0.0.35__py3-none-any.whl → 0.0.37__py3-none-any.whl

ddeutil/workflow/logs.py

@@ -3,10 +3,11 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
-"""This is the Logs module. This module provide TraceLog dataclasses.
+"""A Logs module contain a TraceLog dataclass.
 """
 from __future__ import annotations
 
+import json
 import os
 from abc import ABC, abstractmethod
 from collections.abc import Iterator
@@ -14,11 +15,13 @@ from datetime import datetime
 from inspect import Traceback, currentframe, getframeinfo
 from pathlib import Path
 from threading import get_ident
-from typing import Optional, Union
+from typing import ClassVar, Literal, Optional, Union
 
+from pydantic import BaseModel, Field
 from pydantic.dataclasses import dataclass
+from typing_extensions import Self
 
-from .__types import TupleStr
+from .__types import DictStr, TupleStr
 from .conf import config, get_logger
 from .utils import cut_id, get_dt_now
 
@@ -26,13 +29,17 @@ logger = get_logger("ddeutil.workflow")
 
 __all__: TupleStr = (
     "FileTraceLog",
+    "SQLiteTraceLog",
+    "TraceData",
+    "TraceMeda",
     "TraceLog",
     "get_dt_tznow",
     "get_trace",
+    "get_trace_obj",
 )
 
 
-def get_dt_tznow() -> datetime:
+def get_dt_tznow() -> datetime:  # pragma: no cov
     """Return the current datetime object that passing the config timezone.
 
     :rtype: datetime
@@ -49,76 +56,154 @@ class BaseTraceLog(ABC):  # pragma: no cov
 
     @abstractmethod
     def writer(self, message: str, is_err: bool = False) -> None:
+        """Write a trace message after making to target pointer object. The
+        target can be anything be inherited this class and overwrite this method
+        such as file, console, or database.
+
+        :param message: A message after making.
+        :param is_err: A flag for writing with an error trace or not.
+        """
         raise NotImplementedError(
             "Create writer logic for this trace object before using."
         )
 
     @abstractmethod
     def make_message(self, message: str) -> str:
+        """Prepare and Make a message before write and log processes.
+
+        :param message: A message that want to prepare and make before.
+
+        :rtype: str
+        """
         raise NotImplementedError(
             "Adjust make message method for this trace object before using."
         )
 
     def debug(self, message: str):
+        """Write trace log with append mode and logging this message with the
+        DEBUG level.
+
+        :param message: (str) A message that want to log.
+        """
         msg: str = self.make_message(message)
 
-        # NOTE: Write file if debug mode.
+        # NOTE: Write file if debug mode was enabled.
         if config.debug:
             self.writer(msg)
 
         logger.debug(msg, stacklevel=2)
 
-    def info(self, message: str):
+    def info(self, message: str) -> None:
+        """Write trace log with append mode and logging this message with the
+        INFO level.
+
+        :param message: (str) A message that want to log.
+        """
         msg: str = self.make_message(message)
         self.writer(msg)
         logger.info(msg, stacklevel=2)
 
-    def warning(self, message: str):
+    def warning(self, message: str) -> None:
+        """Write trace log with append mode and logging this message with the
+        WARNING level.
+
+        :param message: (str) A message that want to log.
+        """
         msg: str = self.make_message(message)
         self.writer(msg)
         logger.warning(msg, stacklevel=2)
 
-    def error(self, message: str):
+    def error(self, message: str) -> None:
+        """Write trace log with append mode and logging this message with the
+        ERROR level.
+
+        :param message: (str) A message that want to log.
+        """
         msg: str = self.make_message(message)
         self.writer(msg, is_err=True)
         logger.error(msg, stacklevel=2)
 
+    def exception(self, message: str) -> None:
+        """Write trace log with append mode and logging this message with the
+        EXCEPTION level.
 
-class FileTraceLog(BaseTraceLog):  # pragma: no cov
-    """Trace Log object that write file to the local storage."""
+        :param message: (str) A message that want to log.
+        """
+        msg: str = self.make_message(message)
+        self.writer(msg, is_err=True)
+        logger.exception(msg, stacklevel=2)
+
+
+class TraceMeda(BaseModel):  # pragma: no cov
+    mode: Literal["stdout", "stderr"]
+    datetime: str
+    process: int
+    thread: int
+    message: str
+    filename: str
+    lineno: int
+
+
+class TraceData(BaseModel):  # pragma: no cov
+    stdout: str = Field(description="A standard output trace data.")
+    stderr: str = Field(description="A standard error trace data.")
+    meta: list[TraceMeda] = Field(
+        default_factory=list,
+        description=(
+            "A metadata mapping of this output and error before making it to "
+            "standard value."
+        ),
+    )
 
     @classmethod
-    def find_logs(cls) -> Iterator[dict[str, str]]:  # pragma: no cov
-        for file in config.log_path.glob("./run_id=*"):
-            data: dict[str, str] = {}
+    def from_path(cls, file: Path) -> Self:
+        data: DictStr = {"stdout": "", "stderr": "", "meta": []}
 
-            if (file / "stdout.txt").exists():
-                data["stdout"] = (file / "stdout.txt").read_text(
-                    encoding="utf-8"
-                )
+        if (file / "stdout.txt").exists():
+            data["stdout"] = (file / "stdout.txt").read_text(encoding="utf-8")
 
-            if (file / "stderr.txt").exists():
-                data["stdout"] = (file / "stdout.txt").read_text(
-                    encoding="utf-8"
+        if (file / "stderr.txt").exists():
+            data["stderr"] = (file / "stderr.txt").read_text(encoding="utf-8")
+
+        if (file / "metadata.json").exists():
+            data["meta"] = [
+                json.loads(line)
+                for line in (
+                    (file / "metadata.json")
+                    .read_text(encoding="utf-8")
+                    .splitlines()
                 )
+            ]
 
-            yield data
+        return cls.model_validate(data)
 
-    @classmethod
-    def find_log_with_id(cls, run_id: str) -> dict[str, str]:
-        file: Path = config.log_path / f"run_id={run_id}"
-        data: dict[str, str] = {}
 
-        if (file / "stdout.txt").exists():
-            data["stdout"] = (file / "stdout.txt").read_text(encoding="utf-8")
+class FileTraceLog(BaseTraceLog):  # pragma: no cov
+    """Trace Log object that write file to the local storage."""
 
-        if (file / "stderr.txt").exists():
-            data["stdout"] = (file / "stdout.txt").read_text(encoding="utf-8")
+    @classmethod
+    def find_logs(cls) -> Iterator[TraceData]:  # pragma: no cov
+        for file in sorted(
+            config.log_path.glob("./run_id=*"),
+            key=lambda f: f.lstat().st_mtime,
+        ):
+            yield TraceData.from_path(file)
 
-        return data
+    @classmethod
+    def find_log_with_id(
+        cls, run_id: str, force_raise: bool = True
+    ) -> TraceData:
+        file: Path = config.log_path / f"run_id={run_id}"
+        if file.exists():
+            return TraceData.from_path(file)
+        elif force_raise:
+            raise FileNotFoundError(
+                f"Trace log on path 'run_id={run_id}' does not found."
+            )
+        return {}
 
     @property
-    def log_file(self) -> Path:
+    def pointer(self) -> Path:
         log_file: Path = (
             config.log_path / f"run_id={self.parent_run_id or self.run_id}"
         )
@@ -128,7 +213,10 @@ class FileTraceLog(BaseTraceLog):  # pragma: no cov
 
     @property
     def cut_id(self) -> str:
-        """Combine cutting ID of parent running ID if it set."""
+        """Combine cutting ID of parent running ID if it set.
+
+        :rtype: str
+        """
         cut_run_id: str = cut_id(self.run_id)
         if not self.parent_run_id:
             return f"{cut_run_id} -> {' ' * 6}"
@@ -137,16 +225,26 @@ class FileTraceLog(BaseTraceLog):  # pragma: no cov
         return f"{cut_parent_run_id} -> {cut_run_id}"
 
     def make_message(self, message: str) -> str:
+        """Prepare and Make a message before write and log processes.
+
+        :param message: A message that want to prepare and make before.
+
+        :rtype: str
+        """
         return f"({self.cut_id}) {message}"
 
     def writer(self, message: str, is_err: bool = False) -> None:
-        """The path of logging data will store by format:
+        """ "Write a trace message after making to target file and write metadata
+        in the same path of standard files.
+
+            The path of logging data will store by format:
 
+            ... ./logs/run_id=<run-id>/metadata.json
             ... ./logs/run_id=<run-id>/stdout.txt
             ... ./logs/run_id=<run-id>/stderr.txt
 
-        :param message:
-        :param is_err:
+        :param message: A message after making.
+        :param is_err: A flag for writing with an error trace or not.
         """
         if not config.enable_write_log:
             return
@@ -160,34 +258,47 @@ class FileTraceLog(BaseTraceLog):  # pragma: no cov
         thread: int = get_ident()
 
         write_file: str = "stderr.txt" if is_err else "stdout.txt"
-        with (self.log_file / write_file).open(
+        write_data: dict[str, Union[str, int]] = {
+            "datetime": get_dt_tznow().strftime(config.log_datetime_format),
+            "process": process,
+            "thread": thread,
+            "message": message,
+            "filename": filename,
+            "lineno": lineno,
+        }
+
+        with (self.pointer / write_file).open(mode="at", encoding="utf-8") as f:
+            msg_fmt: str = f"{config.log_format_file}\n"
+            f.write(msg_fmt.format(**write_data))
+
+        with (self.pointer / "metadata.json").open(
             mode="at", encoding="utf-8"
         ) as f:
-            msg_fmt: str = f"{config.log_format_file}\n"
-            print(msg_fmt)
             f.write(
-                msg_fmt.format(
-                    **{
-                        "datetime": get_dt_tznow().strftime(
-                            config.log_datetime_format
-                        ),
-                        "process": process,
-                        "thread": thread,
-                        "message": message,
-                        "filename": filename,
-                        "lineno": lineno,
-                    }
-                )
+                json.dumps({"mode": write_file.split(".")[0]} | write_data)
+                + "\n"
             )
 
 
 class SQLiteTraceLog(BaseTraceLog):  # pragma: no cov
+    """Trace Log object that write trace log to the SQLite database file."""
+
+    table_name: ClassVar[str] = "audits"
+    schemas: ClassVar[
+        str
+    ] = """
+        run_id          int,
+        stdout          str,
+        stderr          str,
+        update          datetime
+        primary key ( run_id )
+        """
 
     @classmethod
-    def find_logs(cls) -> Iterator[dict[str, str]]: ...
+    def find_logs(cls) -> Iterator[DictStr]: ...
 
     @classmethod
-    def find_log_with_id(cls, run_id: str) -> dict[str, str]: ...
+    def find_log_with_id(cls, run_id: str) -> DictStr: ...
 
     def make_message(self, message: str) -> str: ...
 
@@ -203,6 +314,7 @@ TraceLog = Union[
 def get_trace(
     run_id: str, parent_run_id: str | None = None
 ) -> TraceLog:  # pragma: no cov
+    """Get dynamic TraceLog object from the setting config."""
     if config.log_path.is_file():
         return SQLiteTraceLog(run_id, parent_run_id=parent_run_id)
     return FileTraceLog(run_id, parent_run_id=parent_run_id)

ddeutil/workflow/params.py

@@ -3,33 +3,37 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
-"""This module include all Param Models that use for parsing incoming parameters
-that pass to the Workflow and Schedule objects.
+"""This module include all Param Pydantic Models that use for parsing an
+incoming parameters that was passed to the Workflow and Schedule objects before
+execution or release methods.
+
+    The Param model allow you to handle validation and preparation steps before
+passing an input value to target execution method.
 """
 from __future__ import annotations
 
 import decimal
-import logging
 from abc import ABC, abstractmethod
 from datetime import date, datetime
-from typing import Annotated, Any, Literal, Optional, Union
+from typing import Annotated, Any, Literal, Optional, TypeVar, Union
 
 from pydantic import BaseModel, Field
 
 from .__types import TupleStr
 from .exceptions import ParamValueException
-from .utils import get_dt_now
-
-logger = logging.getLogger("ddeutil.workflow")
+from .utils import get_d_now, get_dt_now
 
 __all__: TupleStr = (
     "ChoiceParam",
     "DatetimeParam",
+    "DateParam",
     "IntParam",
     "Param",
     "StrParam",
 )
 
+T = TypeVar("T")
+
 
 class BaseParam(BaseModel, ABC):
     """Base Parameter that use to make any Params Models. The parameter type
@@ -46,7 +50,7 @@ class BaseParam(BaseModel, ABC):
     type: str = Field(description="A type of parameter.")
 
     @abstractmethod
-    def receive(self, value: Optional[Any] = None) -> Any:
+    def receive(self, value: Optional[T] = None) -> T:
         raise NotImplementedError(
             "Receive value and validate typing before return valid value."
         )
@@ -73,17 +77,42 @@ class DefaultParam(BaseParam):
         )
 
 
-# TODO: Not implement this parameter yet
 class DateParam(DefaultParam):  # pragma: no cov
-    """Date parameter."""
+    """Date parameter model."""
 
     type: Literal["date"] = "date"
+    default: date = Field(default_factory=get_d_now)
+
+    def receive(self, value: Optional[str | datetime | date] = None) -> date:
+        """Receive value that match with date. If an input value pass with
+        None, it will use default value instead.
 
-    def receive(self, value: Optional[str | date] = None) -> date: ...
+        :param value: A value that want to validate with date parameter type.
+
+        :rtype: date
+        """
+        if value is None:
+            return self.default
+
+        if isinstance(value, datetime):
+            return value.date()
+        elif isinstance(value, date):
+            return value
+        elif not isinstance(value, str):
+            raise ParamValueException(
+                f"Value that want to convert to date does not support for "
+                f"type: {type(value)}"
+            )
+        try:
+            return date.fromisoformat(value)
+        except ValueError:
+            raise ParamValueException(
+                f"Invalid the ISO format string for date: {value!r}"
+            ) from None
 
 
 class DatetimeParam(DefaultParam):
-    """Datetime parameter."""
+    """Datetime parameter model."""
 
     type: Literal["datetime"] = "datetime"
     default: datetime = Field(default_factory=get_dt_now)
@@ -94,6 +123,7 @@ class DatetimeParam(DefaultParam):
 
         :param value: A value that want to validate with datetime parameter
             type.
+
         :rtype: datetime
         """
         if value is None:
@@ -112,7 +142,7 @@ class DatetimeParam(DefaultParam):
             return datetime.fromisoformat(value)
         except ValueError:
             raise ParamValueException(
-                f"Invalid the ISO format string: {value!r}"
+                f"Invalid the ISO format string for datetime: {value!r}"
             ) from None
 
 
@@ -192,7 +222,7 @@ class ChoiceParam(BaseParam):
 
 
 # TODO: Not implement this parameter yet
-class MappingParam(DefaultParam):  # pragma: no cov
+class MapParam(DefaultParam):  # pragma: no cov
 
     type: Literal["map"] = "map"
     default: dict[Any, Any] = Field(default_factory=dict)
@@ -208,15 +238,22 @@ class ArrayParam(DefaultParam):  # pragma: no cov
     type: Literal["array"] = "array"
     default: list[Any] = Field(default_factory=list)
 
-    def receive(self, value: Optional[list[Any]] = None) -> list[Any]:
+    def receive(self, value: Optional[list[T]] = None) -> list[T]:
         if value is None:
             return self.default
+        if not isinstance(value, list):
+            raise ParamValueException(
+                f"Value that want to convert to array does not support for "
+                f"type: {type(value)}"
+            )
+        return value
 
 
 Param = Annotated[
     Union[
         ChoiceParam,
         DatetimeParam,
+        DateParam,
         IntParam,
         StrParam,
     ],

ddeutil/workflow/result.py

@@ -20,12 +20,9 @@ from pydantic.functional_validators import model_validator
 from typing_extensions import Self
 
 from .__types import DictData, TupleStr
-from .conf import get_logger
 from .logs import TraceLog, get_dt_tznow, get_trace
 from .utils import gen_id
 
-logger = get_logger("ddeutil.workflow")
-
 __all__: TupleStr = (
     "Result",
     "Status",
@@ -66,9 +63,10 @@ class Result:
     context: DictData = field(default_factory=dict)
     run_id: Optional[str] = field(default_factory=default_gen_id)
     parent_run_id: Optional[str] = field(default=None, compare=False)
-    event: Event = field(default_factory=Event, compare=False)
     ts: datetime = field(default_factory=get_dt_tznow, compare=False)
-    trace: Optional[TraceLog] = field(default=None)
+
+    event: Event = field(default_factory=Event, compare=False, repr=False)
+    trace: Optional[TraceLog] = field(default=None, compare=False, repr=False)
 
     @classmethod
     def construct_with_rs_or_id(

ddeutil/workflow/scheduler.py

@@ -83,6 +83,11 @@ class ScheduleWorkflow(BaseModel):
     the Schedule model. it should not use Workflow model directly because on the
     schedule config it can adjust crontab value that different from the Workflow
     model.
+
+        This on field does not equal to the on field of Workflow model, but it
+    uses same logic to generate running release date with crontab object. It use
+    for override the on field if the schedule time was change but you do not
+    want to change on the workflow model.
     """
 
     alias: Optional[str] = Field(
@@ -97,7 +102,7 @@ class ScheduleWorkflow(BaseModel):
     values: DictData = Field(
         default_factory=dict,
         description=(
-            "A value that want to pass to the workflow parameters when "
+            "A value that want to pass to the workflow params field when auto "
             "calling release method."
         ),
         alias="params",
@@ -222,8 +227,8 @@ class ScheduleWorkflow(BaseModel):
 class Schedule(BaseModel):
     """Schedule Pydantic model that use to run with any scheduler package.
 
-        It does not equal the on value in Workflow model, but it uses same logic
-    to running release date with crontab interval.
+        The workflows field of this model include ScheduleWorkflow objects that
+    enhance the workflow object by adding the alias and values fields.
     """
 
     desc: Optional[str] = Field(
@@ -359,12 +364,16 @@ ReturnResultOrCancel = Callable[P, ResultOrCancel]
 DecoratorCancelJob = Callable[[ReturnResultOrCancel], ReturnResultOrCancel]
 
 
-def catch_exceptions(cancel_on_failure: bool = False) -> DecoratorCancelJob:
+def catch_exceptions(
+    cancel_on_failure: bool = False,
+    parent_run_id: str | None = None,
+) -> DecoratorCancelJob:
     """Catch exception error from scheduler job that running with schedule
     package and return CancelJob if this function raise an error.
 
     :param cancel_on_failure: A flag that allow to return the CancelJob or not
         it will raise.
+    :param parent_run_id:
 
     :rtype: DecoratorCancelJob
     """
@@ -375,10 +384,17 @@ def catch_exceptions(cancel_on_failure: bool = False) -> DecoratorCancelJob:
 
         @wraps(func)
         def wrapper(*args: P.args, **kwargs: P.kwargs) -> ResultOrCancel:
+
             try:
                 return func(*args, **kwargs)
+
             except Exception as err:
-                logger.exception(err)
+                if parent_run_id:
+                    (
+                        Result(parent_run_id=parent_run_id).trace.exception(
+                            str(err)
+                        )
+                    )
                 if cancel_on_failure:
                     return CancelJob
                 raise err
@@ -399,13 +415,13 @@ class ReleaseThread(TypedDict):
 ReleaseThreads = dict[str, ReleaseThread]
 
 
-@catch_exceptions(cancel_on_failure=True)
 def schedule_task(
     tasks: list[WorkflowTask],
     stop: datetime,
     queue: dict[str, ReleaseQueue],
     threads: ReleaseThreads,
     audit: type[Audit],
+    *,
     parent_run_id: str | None = None,
 ) -> ResultOrCancel:
     """Schedule task function that generate thread of workflow task release
@@ -466,7 +482,9 @@ def schedule_task(
         current_release: datetime = current_date.replace(
             second=0, microsecond=0
         )
-        if (first_date := q.first_queue.date) > current_release:
+        if (
+            first_date := q.first_queue.date
+        ) > current_release:  # pragma: no cov
             result.trace.debug(
                 f"[WORKFLOW]: Skip schedule "
                 f"{first_date:%Y-%m-%d %H:%M:%S} for : {task.alias!r}"
@@ -491,8 +509,14 @@ def schedule_task(
         #   job.
         thread_name: str = f"{task.alias}|{release.date:%Y%m%d%H%M}"
         thread: Thread = Thread(
-            target=catch_exceptions(cancel_on_failure=True)(task.release),
-            kwargs={"release": release, "queue": q, "audit": audit},
+            target=catch_exceptions(
+                cancel_on_failure=True,
+            )(task.release),
+            kwargs={
+                "release": release,
+                "queue": q,
+                "audit": audit,
+            },
             name=thread_name,
             daemon=True,
         )
@@ -508,8 +532,8 @@ def schedule_task(
         delay()
 
     result.trace.debug(
-        f"[SCHEDULE]: End schedule task at {current_date:%Y-%m-%d %H:%M:%S} "
-        f"{'=' * 60}"
+        f"[SCHEDULE]: End schedule task that run since "
+        f"{current_date:%Y-%m-%d %H:%M:%S} {'=' * 30}"
     )
     return result.catch(
         status=Status.SUCCESS, context={"task_date": current_date}
@@ -575,7 +599,10 @@ def scheduler_pending(
         scheduler.every(1)
         .minutes.at(":02")
         .do(
-            schedule_task,
+            catch_exceptions(
+                cancel_on_failure=True,
+                parent_run_id=result.parent_run_id,
+            )(schedule_task),
             tasks=tasks,
             stop=stop,
             queue=queue,
@@ -594,6 +621,7 @@ def scheduler_pending(
         .do(
             monitor,
             threads=threads,
+            parent_run_id=result.parent_run_id,
         )
         .tag("monitor")
     )
@@ -617,7 +645,7 @@ def scheduler_pending(
                     "running in background."
                 )
                 delay(10)
-                monitor(threads)
+                monitor(threads, parent_run_id=result.parent_run_id)
 
             break
 
@@ -752,7 +780,7 @@ def schedule_runner(
 
             # NOTE: Raise error when it has any error from schedule_control.
             if err := future.exception():
-                logger.error(str(err))
+                result.trace.error(str(err))
                 raise WorkflowException(str(err)) from err
 
             rs: Result = future.result(timeout=1)