ddeutil-workflow 0.0.34__py3-none-any.whl → 0.0.35__py3-none-any.whl

ddeutil/workflow/logs.py

@@ -0,0 +1,214 @@
+# ------------------------------------------------------------------------------
+# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
+# Licensed under the MIT License. See LICENSE in the project root for
+# license information.
+# ------------------------------------------------------------------------------
+"""This is the Logs module. This module provide TraceLog dataclasses.
+"""
+from __future__ import annotations
+
+import os
+from abc import ABC, abstractmethod
+from collections.abc import Iterator
+from datetime import datetime
+from inspect import Traceback, currentframe, getframeinfo
+from pathlib import Path
+from threading import get_ident
+from typing import Optional, Union
+
+from pydantic.dataclasses import dataclass
+
+from .__types import TupleStr
+from .conf import config, get_logger
+from .utils import cut_id, get_dt_now
+
+logger = get_logger("ddeutil.workflow")
+
+__all__: TupleStr = (
+    "FileTraceLog",
+    "TraceLog",
+    "get_dt_tznow",
+    "get_trace",
+)
+
+
+def get_dt_tznow() -> datetime:
+    """Return the current datetime object that passing the config timezone.
+
+    :rtype: datetime
+    """
+    return get_dt_now(tz=config.tz)
+
+
+@dataclass(frozen=True)
+class BaseTraceLog(ABC):  # pragma: no cov
+    """Base Trace Log dataclass object."""
+
+    run_id: str
+    parent_run_id: Optional[str] = None
+
+    @abstractmethod
+    def writer(self, message: str, is_err: bool = False) -> None:
+        raise NotImplementedError(
+            "Create writer logic for this trace object before using."
+        )
+
+    @abstractmethod
+    def make_message(self, message: str) -> str:
+        raise NotImplementedError(
+            "Adjust make message method for this trace object before using."
+        )
+
+    def debug(self, message: str):
+        msg: str = self.make_message(message)
+
+        # NOTE: Write file if debug mode.
+        if config.debug:
+            self.writer(msg)
+
+        logger.debug(msg, stacklevel=2)
+
+    def info(self, message: str):
+        msg: str = self.make_message(message)
+        self.writer(msg)
+        logger.info(msg, stacklevel=2)
+
+    def warning(self, message: str):
+        msg: str = self.make_message(message)
+        self.writer(msg)
+        logger.warning(msg, stacklevel=2)
+
+    def error(self, message: str):
+        msg: str = self.make_message(message)
+        self.writer(msg, is_err=True)
+        logger.error(msg, stacklevel=2)
+
+
+class FileTraceLog(BaseTraceLog):  # pragma: no cov
+    """Trace Log object that write file to the local storage."""
+
+    @classmethod
+    def find_logs(cls) -> Iterator[dict[str, str]]:  # pragma: no cov
+        for file in config.log_path.glob("./run_id=*"):
+            data: dict[str, str] = {}
+
+            if (file / "stdout.txt").exists():
+                data["stdout"] = (file / "stdout.txt").read_text(
+                    encoding="utf-8"
+                )
+
+            if (file / "stderr.txt").exists():
+                data["stdout"] = (file / "stdout.txt").read_text(
+                    encoding="utf-8"
+                )
+
+            yield data
+
+    @classmethod
+    def find_log_with_id(cls, run_id: str) -> dict[str, str]:
+        file: Path = config.log_path / f"run_id={run_id}"
+        data: dict[str, str] = {}
+
+        if (file / "stdout.txt").exists():
+            data["stdout"] = (file / "stdout.txt").read_text(encoding="utf-8")
+
+        if (file / "stderr.txt").exists():
+            data["stdout"] = (file / "stdout.txt").read_text(encoding="utf-8")
+
+        return data
+
+    @property
+    def log_file(self) -> Path:
+        log_file: Path = (
+            config.log_path / f"run_id={self.parent_run_id or self.run_id}"
+        )
+        if not log_file.exists():
+            log_file.mkdir(parents=True)
+        return log_file
+
+    @property
+    def cut_id(self) -> str:
+        """Combine cutting ID of parent running ID if it set."""
+        cut_run_id: str = cut_id(self.run_id)
+        if not self.parent_run_id:
+            return f"{cut_run_id} -> {' ' * 6}"
+
+        cut_parent_run_id: str = cut_id(self.parent_run_id)
+        return f"{cut_parent_run_id} -> {cut_run_id}"
+
+    def make_message(self, message: str) -> str:
+        return f"({self.cut_id}) {message}"
+
+    def writer(self, message: str, is_err: bool = False) -> None:
+        """The path of logging data will store by format:
+
+            ... ./logs/run_id=<run-id>/stdout.txt
+            ... ./logs/run_id=<run-id>/stderr.txt
+
+        :param message:
+        :param is_err:
+        """
+        if not config.enable_write_log:
+            return
+
+        frame_info: Traceback = getframeinfo(currentframe().f_back.f_back)
+        filename: str = frame_info.filename.split(os.path.sep)[-1]
+        lineno: int = frame_info.lineno
+
+        # NOTE: set process and thread IDs.
+        process: int = os.getpid()
+        thread: int = get_ident()
+
+        write_file: str = "stderr.txt" if is_err else "stdout.txt"
+        with (self.log_file / write_file).open(
+            mode="at", encoding="utf-8"
+        ) as f:
+            msg_fmt: str = f"{config.log_format_file}\n"
+            print(msg_fmt)
+            f.write(
+                msg_fmt.format(
+                    **{
+                        "datetime": get_dt_tznow().strftime(
+                            config.log_datetime_format
+                        ),
+                        "process": process,
+                        "thread": thread,
+                        "message": message,
+                        "filename": filename,
+                        "lineno": lineno,
+                    }
+                )
+            )
+
+
+class SQLiteTraceLog(BaseTraceLog):  # pragma: no cov
+
+    @classmethod
+    def find_logs(cls) -> Iterator[dict[str, str]]: ...
+
+    @classmethod
+    def find_log_with_id(cls, run_id: str) -> dict[str, str]: ...
+
+    def make_message(self, message: str) -> str: ...
+
+    def writer(self, message: str, is_err: bool = False) -> None: ...
+
+
+TraceLog = Union[
+    FileTraceLog,
+    SQLiteTraceLog,
+]
+
+
+def get_trace(
+    run_id: str, parent_run_id: str | None = None
+) -> TraceLog:  # pragma: no cov
+    if config.log_path.is_file():
+        return SQLiteTraceLog(run_id, parent_run_id=parent_run_id)
+    return FileTraceLog(run_id, parent_run_id=parent_run_id)
+
+
+def get_trace_obj() -> type[TraceLog]:  # pragma: no cov
+    if config.log_path.is_file():
+        return SQLiteTraceLog
+    return FileTraceLog

ddeutil/workflow/params.py

@@ -3,8 +3,8 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
-"""Param Model that use for parsing incoming parameters that pass to the
-Workflow and Schedule objects.
+"""This module include all Param Models that use for parsing incoming parameters
+that pass to the Workflow and Schedule objects.
 """
 from __future__ import annotations
 
@@ -12,7 +12,7 @@ import decimal
 import logging
 from abc import ABC, abstractmethod
 from datetime import date, datetime
-from typing import Any, Literal, Optional, Union
+from typing import Annotated, Any, Literal, Optional, Union
 
 from pydantic import BaseModel, Field
 
@@ -32,8 +32,9 @@ __all__: TupleStr = (
 
 
 class BaseParam(BaseModel, ABC):
-    """Base Parameter that use to make any Params Model. The type will dynamic
-    with the type field that made from literal string."""
+    """Base Parameter that use to make any Params Models. The parameter type
+    will dynamic with the setup type field that made from literal string.
+    """
 
     desc: Optional[str] = Field(
         default=None, description="A description of parameter providing."
@@ -169,9 +170,11 @@ class ChoiceParam(BaseParam):
     """Choice parameter."""
 
     type: Literal["choice"] = "choice"
-    options: list[str] = Field(description="A list of choice parameters.")
+    options: Union[list[str], list[int]] = Field(
+        description="A list of choice parameters that able be str or int.",
+    )
 
-    def receive(self, value: str | None = None) -> str:
+    def receive(self, value: Union[str, int] | None = None) -> Union[str, int]:
         """Receive value that match with options.
 
         :param value: A value that want to select from the options field.
@@ -188,9 +191,34 @@ class ChoiceParam(BaseParam):
         return value
 
 
-Param = Union[
-    ChoiceParam,
-    DatetimeParam,
-    IntParam,
-    StrParam,
+# TODO: Not implement this parameter yet
+class MappingParam(DefaultParam):  # pragma: no cov
+
+    type: Literal["map"] = "map"
+    default: dict[Any, Any] = Field(default_factory=dict)
+
+    def receive(self, value: Optional[dict[Any, Any]] = None) -> dict[Any, Any]:
+        if value is None:
+            return self.default
+
+
+# TODO: Not implement this parameter yet
+class ArrayParam(DefaultParam):  # pragma: no cov
+
+    type: Literal["array"] = "array"
+    default: list[Any] = Field(default_factory=list)
+
+    def receive(self, value: Optional[list[Any]] = None) -> list[Any]:
+        if value is None:
+            return self.default
+
+
+Param = Annotated[
+    Union[
+        ChoiceParam,
+        DatetimeParam,
+        IntParam,
+        StrParam,
+    ],
+    Field(discriminator="type"),
 ]

ddeutil/workflow/result.py

@@ -4,36 +4,32 @@
 # license information.
 # ------------------------------------------------------------------------------
 """This is the Result module. It is the data context transfer objects that use
-by all object in this package.
+by all object in this package. This module provide Result dataclass.
 """
 from __future__ import annotations
 
-import os
-from abc import ABC, abstractmethod
 from dataclasses import field
 from datetime import datetime
 from enum import IntEnum
-from inspect import Traceback, currentframe, getframeinfo
-from pathlib import Path
-from threading import Event, get_ident
+from threading import Event
 from typing import Optional
 
 from pydantic import ConfigDict
 from pydantic.dataclasses import dataclass
+from pydantic.functional_validators import model_validator
 from typing_extensions import Self
 
 from .__types import DictData, TupleStr
-from .conf import config, get_logger
-from .utils import cut_id, gen_id, get_dt_now
+from .conf import get_logger
+from .logs import TraceLog, get_dt_tznow, get_trace
+from .utils import gen_id
 
 logger = get_logger("ddeutil.workflow")
 
 __all__: TupleStr = (
     "Result",
     "Status",
-    "TraceLog",
     "default_gen_id",
-    "get_dt_tznow",
 )
 
 
@@ -46,14 +42,6 @@ def default_gen_id() -> str:
     return gen_id("manual", unique=True)
 
 
-def get_dt_tznow() -> datetime:
-    """Return the current datetime object that passing the config timezone.
-
-    :rtype: datetime
-    """
-    return get_dt_now(tz=config.tz)
-
-
 class Status(IntEnum):
     """Status Int Enum object."""
 
@@ -62,111 +50,6 @@ class Status(IntEnum):
     WAIT: int = 2
 
 
-@dataclass(frozen=True)
-class BaseTraceLog(ABC):  # pragma: no cov
-    """Base Trace Log dataclass object."""
-
-    run_id: str
-    parent_run_id: Optional[str] = None
-
-    @abstractmethod
-    def writer(self, message: str, is_err: bool = False) -> None: ...
-
-    @abstractmethod
-    def make_message(self, message: str) -> str: ...
-
-    def debug(self, message: str):
-        msg: str = self.make_message(message)
-
-        # NOTE: Write file if debug mode.
-        if config.debug:
-            self.writer(msg)
-
-        logger.debug(msg, stacklevel=2)
-
-    def info(self, message: str):
-        msg: str = self.make_message(message)
-        self.writer(msg)
-        logger.info(msg, stacklevel=2)
-
-    def warning(self, message: str):
-        msg: str = self.make_message(message)
-        self.writer(msg)
-        logger.warning(msg, stacklevel=2)
-
-    def error(self, message: str):
-        msg: str = self.make_message(message)
-        self.writer(msg, is_err=True)
-        logger.error(msg, stacklevel=2)
-
-
-class TraceLog(BaseTraceLog):  # pragma: no cov
-    """Trace Log object that write file to the local storage."""
-
-    @property
-    def log_file(self) -> Path:
-        log_file: Path = (
-            config.log_path / f"run_id={self.parent_run_id or self.run_id}"
-        )
-        if not log_file.exists():
-            log_file.mkdir(parents=True)
-        return log_file
-
-    @property
-    def cut_id(self) -> str:
-        """Combine cutting ID of parent running ID if it set."""
-        cut_run_id: str = cut_id(self.run_id)
-        if not self.parent_run_id:
-            return f"{cut_run_id} -> {' ' * 6}"
-
-        cut_parent_run_id: str = cut_id(self.parent_run_id)
-        return f"{cut_parent_run_id} -> {cut_run_id}"
-
-    def make_message(self, message: str) -> str:
-        return f"({self.cut_id}) {message}"
-
-    def writer(self, message: str, is_err: bool = False) -> None:
-        """The path of logging data will store by format:
-
-            ... ./logs/run_id=<run-id>/stdout.txt
-            ... ./logs/run_id=<run-id>/stderr.txt
-
-        :param message:
-        :param is_err:
-        """
-        if not config.enable_write_log:
-            return
-
-        frame_info: Traceback = getframeinfo(currentframe().f_back.f_back)
-        filename: str = frame_info.filename.split(os.path.sep)[-1]
-        lineno: int = frame_info.lineno
-
-        # NOTE: set process and thread IDs.
-        process: int = os.getpid()
-        thread: int = get_ident()
-
-        write_file: str = "stderr.txt" if is_err else "stdout.txt"
-        with (self.log_file / write_file).open(
-            mode="at", encoding="utf-8"
-        ) as f:
-            msg_fmt: str = f"{config.log_format_file}\n"
-            print(msg_fmt)
-            f.write(
-                msg_fmt.format(
-                    **{
-                        "datetime": get_dt_tznow().strftime(
-                            config.log_datetime_format
-                        ),
-                        "process": process,
-                        "thread": thread,
-                        "message": message,
-                        "filename": filename,
-                        "lineno": lineno,
-                    }
-                )
-            )
-
-
 @dataclass(
     config=ConfigDict(arbitrary_types_allowed=True, use_enum_values=True)
 )
@@ -182,11 +65,10 @@ class Result:
     status: Status = field(default=Status.WAIT)
     context: DictData = field(default_factory=dict)
     run_id: Optional[str] = field(default_factory=default_gen_id)
-
-    # NOTE: Ignore this field to compare another result model with __eq__.
     parent_run_id: Optional[str] = field(default=None, compare=False)
     event: Event = field(default_factory=Event, compare=False)
     ts: datetime = field(default_factory=get_dt_tznow, compare=False)
+    trace: Optional[TraceLog] = field(default=None)
 
     @classmethod
     def construct_with_rs_or_id(
@@ -208,13 +90,12 @@ class Result:
             result.set_parent_run_id(parent_run_id)
         return result
 
-    def set_run_id(self, running_id: str) -> Self:
-        """Set a running ID.
+    @model_validator(mode="after")
+    def __prepare_trace(self) -> Self:
+        """Prepare trace field that want to pass after its initialize step."""
+        if self.trace is None:  # pragma: no cove
+            self.trace: TraceLog = get_trace(self.run_id, self.parent_run_id)
 
-        :param running_id: A running ID that want to update on this model.
-        :rtype: Self
-        """
-        self.run_id: str = running_id
         return self
 
     def set_parent_run_id(self, running_id: str) -> Self:
@@ -224,6 +105,7 @@ class Result:
         :rtype: Self
         """
         self.parent_run_id: str = running_id
+        self.trace: TraceLog = get_trace(self.run_id, running_id)
         return self
 
     def catch(
@@ -244,13 +126,9 @@ class Result:
         self.__dict__["context"].update(context or {})
         return self
 
-    @property
-    def trace(self) -> TraceLog:
-        """Return TraceLog object that passing its running ID.
+    def alive_time(self) -> float:  # pragma: no cov
+        """Return total seconds that this object use since it was created.
 
-        :rtype: TraceLog
+        :rtype: float
         """
-        return TraceLog(self.run_id, self.parent_run_id)
-
-    def alive_time(self) -> float:  # pragma: no cov
         return (get_dt_tznow() - self.ts).total_seconds()

ddeutil/workflow/scheduler.py

@@ -4,18 +4,18 @@
 # license information.
 # ------------------------------------------------------------------------------
 """
-The main schedule running is ``schedule_runner`` function that trigger the
-multiprocess of ``schedule_control`` function for listing schedules on the
-config by ``Loader.finds(Schedule)``.
+The main schedule running is `schedule_runner` function that trigger the
+multiprocess of `schedule_control` function for listing schedules on the
+config by `Loader.finds(Schedule)`.
 
-    The ``schedule_control`` is the scheduler function that release 2 schedule
-functions; ``workflow_task``, and ``workflow_monitor``.
+    The `schedule_control` is the scheduler function that release 2 schedule
+functions; `workflow_task`, and `workflow_monitor`.
 
-    ``schedule_control`` --- Every minute at :02 --> ``schedule_task``
-                         --- Every 5 minutes     --> ``monitor``
+    `schedule_control` ---( Every minute at :02 )--> `schedule_task`
+                       ---( Every 5 minutes     )--> `monitor`
 
-    The ``schedule_task`` will run ``task.release`` method in threading object
-for multithreading strategy. This ``release`` method will run only one crontab
+    The `schedule_task` will run `task.release` method in threading object
+for multithreading strategy. This `release` method will run only one crontab
 value with the on field.
 """
 from __future__ import annotations
@@ -134,7 +134,7 @@ class ScheduleWorkflow(BaseModel):
                 on: list[str] = [on]
 
             if any(not isinstance(n, (dict, str)) for n in on):
-                raise TypeError("The ``on`` key should be list of str or dict")
+                raise TypeError("The `on` key should be list of str or dict")
 
             # NOTE: Pass on value to Loader and keep on model object to on
             #   field.
@@ -344,7 +344,7 @@ class Schedule(BaseModel):
             tasks=self.tasks(
                 start_date_waiting, queue=queue, externals=externals
             ),
-            stop_date=stop_date,
+            stop=stop_date,
             queue=queue,
             threads=threads,
             result=result,
@@ -509,21 +509,27 @@ def schedule_task(
 
     result.trace.debug(
         f"[SCHEDULE]: End schedule task at {current_date:%Y-%m-%d %H:%M:%S} "
-        f"{'=' * 80}"
+        f"{'=' * 60}"
     )
     return result.catch(
         status=Status.SUCCESS, context={"task_date": current_date}
     )
 
 
-def monitor(threads: ReleaseThreads) -> None:  # pragma: no cov
+def monitor(
+    threads: ReleaseThreads,
+    parent_run_id: str | None = None,
+) -> None:  # pragma: no cov
     """Monitoring function that running every five minute for track long-running
     thread instance from the schedule_control function that run every minute.
 
     :param threads: A mapping of Thread object and its name.
+    :param parent_run_id: A parent workflow running ID for this release.
+
     :type threads: ReleaseThreads
     """
-    logger.debug("[MONITOR]: Start checking long running schedule task.")
+    result: Result = Result().set_parent_run_id(parent_run_id)
+    result.trace.debug("[MONITOR]: Start checking long running schedule task.")
 
     snapshot_threads: list[str] = list(threads.keys())
     for thread_name in snapshot_threads:
@@ -538,20 +544,20 @@ def monitor(threads: ReleaseThreads) -> None:  # pragma: no cov
 
 def scheduler_pending(
     tasks: list[WorkflowTask],
-    stop_date,
-    queue,
-    threads,
+    stop: datetime,
+    queue: dict[str, ReleaseQueue],
+    threads: ReleaseThreads,
     result: Result,
     audit: type[Audit],
 ) -> Result:  # pragma: no cov
-    """
+    """Scheduler pending function.
 
-    :param tasks:
-    :param stop_date:
-    :param queue:
-    :param threads:
-    :param result:
-    :param audit:
+    :param tasks: A list of WorkflowTask object.
+    :param stop: A stop datetime object that force stop running scheduler.
+    :param queue: A mapping of alias name and ReleaseQueue object.
+    :param threads: A mapping of alias name and Thread object.
+    :param result: A result object.
+    :param audit: An audit class that want to make audit object.
 
     :rtype: Result
     """
@@ -571,7 +577,7 @@ def scheduler_pending(
         .do(
             schedule_task,
             tasks=tasks,
-            stop=stop_date,
+            stop=stop,
             queue=queue,
             threads=threads,
             audit=audit,
@@ -594,7 +600,7 @@ def scheduler_pending(
 
     # NOTE: Start running schedule
     result.trace.info(
-        f"[SCHEDULE]: Schedule with stopper: {stop_date:%Y-%m-%d %H:%M:%S}"
+        f"[SCHEDULE]: Schedule with stopper: {stop:%Y-%m-%d %H:%M:%S}"
     )
 
     while True:
@@ -681,7 +687,7 @@ def schedule_control(
 
     scheduler_pending(
         tasks=tasks,
-        stop_date=stop_date,
+        stop=stop_date,
         queue=queue,
         threads=threads,
         result=result,
@@ -707,15 +713,16 @@ def schedule_runner(
 
         This function will get all workflows that include on value that was
     created in config path and chuck it with application config variable
-    ``WORKFLOW_APP_MAX_SCHEDULE_PER_PROCESS`` env var to multiprocess executor
+    `WORKFLOW_APP_MAX_SCHEDULE_PER_PROCESS` env var to multiprocess executor
     pool.
 
         The current workflow logic that split to process will be below diagram:
 
-        MAIN ==> process 01 ==> schedule --> thread of release task 01 01
-                                        --> thread of release task 01 02
-                            ==> schedule --> thread of release task 02 01
-                                        --> thread of release task 02 02
+        MAIN ==> process 01 ==> schedule ==> thread 01 --> 01
+                                         ==> thread 01 --> 02
+                            ==> schedule ==> thread 02 --> 01
+                                         ==> thread 02 --> 02
+                                         ==> ...
             ==> process 02  ==> ...
 
     :rtype: Result