ddeutil-workflow 0.0.33__py3-none-any.whl → 0.0.35__py3-none-any.whl

ddeutil/workflow/logs.py

@@ -0,0 +1,214 @@
+# ------------------------------------------------------------------------------
+# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
+# Licensed under the MIT License. See LICENSE in the project root for
+# license information.
+# ------------------------------------------------------------------------------
+"""This is the Logs module. This module provide TraceLog dataclasses.
+"""
+from __future__ import annotations
+
+import os
+from abc import ABC, abstractmethod
+from collections.abc import Iterator
+from datetime import datetime
+from inspect import Traceback, currentframe, getframeinfo
+from pathlib import Path
+from threading import get_ident
+from typing import Optional, Union
+
+from pydantic.dataclasses import dataclass
+
+from .__types import TupleStr
+from .conf import config, get_logger
+from .utils import cut_id, get_dt_now
+
+logger = get_logger("ddeutil.workflow")
+
+__all__: TupleStr = (
+    "FileTraceLog",
+    "TraceLog",
+    "get_dt_tznow",
+    "get_trace",
+)
+
+
+def get_dt_tznow() -> datetime:
+    """Return the current datetime object that passing the config timezone.
+
+    :rtype: datetime
+    """
+    return get_dt_now(tz=config.tz)
+
+
+@dataclass(frozen=True)
+class BaseTraceLog(ABC):  # pragma: no cov
+    """Base Trace Log dataclass object."""
+
+    run_id: str
+    parent_run_id: Optional[str] = None
+
+    @abstractmethod
+    def writer(self, message: str, is_err: bool = False) -> None:
+        raise NotImplementedError(
+            "Create writer logic for this trace object before using."
+        )
+
+    @abstractmethod
+    def make_message(self, message: str) -> str:
+        raise NotImplementedError(
+            "Adjust make message method for this trace object before using."
+        )
+
+    def debug(self, message: str):
+        msg: str = self.make_message(message)
+
+        # NOTE: Write file if debug mode.
+        if config.debug:
+            self.writer(msg)
+
+        logger.debug(msg, stacklevel=2)
+
+    def info(self, message: str):
+        msg: str = self.make_message(message)
+        self.writer(msg)
+        logger.info(msg, stacklevel=2)
+
+    def warning(self, message: str):
+        msg: str = self.make_message(message)
+        self.writer(msg)
+        logger.warning(msg, stacklevel=2)
+
+    def error(self, message: str):
+        msg: str = self.make_message(message)
+        self.writer(msg, is_err=True)
+        logger.error(msg, stacklevel=2)
+
+
+class FileTraceLog(BaseTraceLog):  # pragma: no cov
+    """Trace Log object that write file to the local storage."""
+
+    @classmethod
+    def find_logs(cls) -> Iterator[dict[str, str]]:  # pragma: no cov
+        for file in config.log_path.glob("./run_id=*"):
+            data: dict[str, str] = {}
+
+            if (file / "stdout.txt").exists():
+                data["stdout"] = (file / "stdout.txt").read_text(
+                    encoding="utf-8"
+                )
+
+            if (file / "stderr.txt").exists():
+                data["stdout"] = (file / "stdout.txt").read_text(
+                    encoding="utf-8"
+                )
+
+            yield data
+
+    @classmethod
+    def find_log_with_id(cls, run_id: str) -> dict[str, str]:
+        file: Path = config.log_path / f"run_id={run_id}"
+        data: dict[str, str] = {}
+
+        if (file / "stdout.txt").exists():
+            data["stdout"] = (file / "stdout.txt").read_text(encoding="utf-8")
+
+        if (file / "stderr.txt").exists():
+            data["stdout"] = (file / "stdout.txt").read_text(encoding="utf-8")
+
+        return data
+
+    @property
+    def log_file(self) -> Path:
+        log_file: Path = (
+            config.log_path / f"run_id={self.parent_run_id or self.run_id}"
+        )
+        if not log_file.exists():
+            log_file.mkdir(parents=True)
+        return log_file
+
+    @property
+    def cut_id(self) -> str:
+        """Combine cutting ID of parent running ID if it set."""
+        cut_run_id: str = cut_id(self.run_id)
+        if not self.parent_run_id:
+            return f"{cut_run_id} -> {' ' * 6}"
+
+        cut_parent_run_id: str = cut_id(self.parent_run_id)
+        return f"{cut_parent_run_id} -> {cut_run_id}"
+
+    def make_message(self, message: str) -> str:
+        return f"({self.cut_id}) {message}"
+
+    def writer(self, message: str, is_err: bool = False) -> None:
+        """The path of logging data will store by format:
+
+            ... ./logs/run_id=<run-id>/stdout.txt
+            ... ./logs/run_id=<run-id>/stderr.txt
+
+        :param message:
+        :param is_err:
+        """
+        if not config.enable_write_log:
+            return
+
+        frame_info: Traceback = getframeinfo(currentframe().f_back.f_back)
+        filename: str = frame_info.filename.split(os.path.sep)[-1]
+        lineno: int = frame_info.lineno
+
+        # NOTE: set process and thread IDs.
+        process: int = os.getpid()
+        thread: int = get_ident()
+
+        write_file: str = "stderr.txt" if is_err else "stdout.txt"
+        with (self.log_file / write_file).open(
+            mode="at", encoding="utf-8"
+        ) as f:
+            msg_fmt: str = f"{config.log_format_file}\n"
+            print(msg_fmt)
+            f.write(
+                msg_fmt.format(
+                    **{
+                        "datetime": get_dt_tznow().strftime(
+                            config.log_datetime_format
+                        ),
+                        "process": process,
+                        "thread": thread,
+                        "message": message,
+                        "filename": filename,
+                        "lineno": lineno,
+                    }
+                )
+            )
+
+
+class SQLiteTraceLog(BaseTraceLog):  # pragma: no cov
+
+    @classmethod
+    def find_logs(cls) -> Iterator[dict[str, str]]: ...
+
+    @classmethod
+    def find_log_with_id(cls, run_id: str) -> dict[str, str]: ...
+
+    def make_message(self, message: str) -> str: ...
+
+    def writer(self, message: str, is_err: bool = False) -> None: ...
+
+
+TraceLog = Union[
+    FileTraceLog,
+    SQLiteTraceLog,
+]
+
+
+def get_trace(
+    run_id: str, parent_run_id: str | None = None
+) -> TraceLog:  # pragma: no cov
+    if config.log_path.is_file():
+        return SQLiteTraceLog(run_id, parent_run_id=parent_run_id)
+    return FileTraceLog(run_id, parent_run_id=parent_run_id)
+
+
+def get_trace_obj() -> type[TraceLog]:  # pragma: no cov
+    if config.log_path.is_file():
+        return SQLiteTraceLog
+    return FileTraceLog

ddeutil/workflow/params.py

@@ -3,8 +3,8 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
-"""Param Model that use for parsing incoming parameters that pass to the
-Workflow and Schedule objects.
+"""This module include all Param Models that use for parsing incoming parameters
+that pass to the Workflow and Schedule objects.
 """
 from __future__ import annotations
 
@@ -12,7 +12,7 @@ import decimal
 import logging
 from abc import ABC, abstractmethod
 from datetime import date, datetime
-from typing import Any, Literal, Optional, Union
+from typing import Annotated, Any, Literal, Optional, Union
 
 from pydantic import BaseModel, Field
 
@@ -32,8 +32,9 @@ __all__: TupleStr = (
 
 
 class BaseParam(BaseModel, ABC):
-    """Base Parameter that use to make any Params Model. The type will dynamic
-    with the type field that made from literal string."""
+    """Base Parameter that use to make any Params Models. The parameter type
+    will dynamic with the setup type field that made from literal string.
+    """
 
     desc: Optional[str] = Field(
         default=None, description="A description of parameter providing."
@@ -169,9 +170,11 @@ class ChoiceParam(BaseParam):
     """Choice parameter."""
 
     type: Literal["choice"] = "choice"
-    options: list[str] = Field(description="A list of choice parameters.")
+    options: Union[list[str], list[int]] = Field(
+        description="A list of choice parameters that able be str or int.",
+    )
 
-    def receive(self, value: str | None = None) -> str:
+    def receive(self, value: Union[str, int] | None = None) -> Union[str, int]:
         """Receive value that match with options.
 
         :param value: A value that want to select from the options field.
@@ -188,9 +191,34 @@ class ChoiceParam(BaseParam):
         return value
 
 
-Param = Union[
-    ChoiceParam,
-    DatetimeParam,
-    IntParam,
-    StrParam,
+# TODO: Not implement this parameter yet
+class MappingParam(DefaultParam):  # pragma: no cov
+
+    type: Literal["map"] = "map"
+    default: dict[Any, Any] = Field(default_factory=dict)
+
+    def receive(self, value: Optional[dict[Any, Any]] = None) -> dict[Any, Any]:
+        if value is None:
+            return self.default
+
+
+# TODO: Not implement this parameter yet
+class ArrayParam(DefaultParam):  # pragma: no cov
+
+    type: Literal["array"] = "array"
+    default: list[Any] = Field(default_factory=list)
+
+    def receive(self, value: Optional[list[Any]] = None) -> list[Any]:
+        if value is None:
+            return self.default
+
+
+Param = Annotated[
+    Union[
+        ChoiceParam,
+        DatetimeParam,
+        IntParam,
+        StrParam,
+    ],
+    Field(discriminator="type"),
 ]

ddeutil/workflow/result.py

@@ -3,6 +3,9 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
+"""This is the Result module. It is the data context transfer objects that use
+by all object in this package. This module provide Result dataclass.
+"""
 from __future__ import annotations
 
 from dataclasses import field
@@ -13,17 +16,20 @@ from typing import Optional
 
 from pydantic import ConfigDict
 from pydantic.dataclasses import dataclass
+from pydantic.functional_validators import model_validator
 from typing_extensions import Self
 
 from .__types import DictData, TupleStr
-from .conf import config, get_logger
-from .utils import cut_id, gen_id, get_dt_now
+from .conf import get_logger
+from .logs import TraceLog, get_dt_tznow, get_trace
+from .utils import gen_id
 
-logger = get_logger("ddeutil.workflow.audit")
+logger = get_logger("ddeutil.workflow")
 
 __all__: TupleStr = (
     "Result",
     "Status",
+    "default_gen_id",
 )
 
 
@@ -36,14 +42,6 @@ def default_gen_id() -> str:
     return gen_id("manual", unique=True)
 
 
-def get_dt_tznow() -> datetime:
-    """Return the current datetime object that passing the config timezone.
-
-    :rtype: datetime
-    """
-    return get_dt_now(tz=config.tz)
-
-
 class Status(IntEnum):
     """Status Int Enum object."""
 
@@ -52,27 +50,6 @@ class Status(IntEnum):
     WAIT: int = 2
 
 
-class TraceLog:  # pragma: no cov
-    """Trace Log object."""
-
-    __slots__: TupleStr = ("run_id",)
-
-    def __init__(self, run_id: str):
-        self.run_id: str = run_id
-
-    def debug(self, message: str):
-        logger.debug(f"({cut_id(self.run_id)}) {message}")
-
-    def info(self, message: str):
-        logger.info(f"({cut_id(self.run_id)}) {message}")
-
-    def warning(self, message: str):
-        logger.warning(f"({cut_id(self.run_id)}) {message}")
-
-    def error(self, message: str):
-        logger.error(f"({cut_id(self.run_id)}) {message}")
-
-
 @dataclass(
     config=ConfigDict(arbitrary_types_allowed=True, use_enum_values=True)
 )
@@ -88,19 +65,37 @@ class Result:
     status: Status = field(default=Status.WAIT)
     context: DictData = field(default_factory=dict)
     run_id: Optional[str] = field(default_factory=default_gen_id)
-
-    # NOTE: Ignore this field to compare another result model with __eq__.
     parent_run_id: Optional[str] = field(default=None, compare=False)
     event: Event = field(default_factory=Event, compare=False)
     ts: datetime = field(default_factory=get_dt_tznow, compare=False)
-
-    def set_run_id(self, running_id: str) -> Self:
-        """Set a running ID.
-
-        :param running_id: A running ID that want to update on this model.
-        :rtype: Self
+    trace: Optional[TraceLog] = field(default=None)
+
+    @classmethod
+    def construct_with_rs_or_id(
+        cls,
+        result: Result | None = None,
+        run_id: str | None = None,
+        parent_run_id: str | None = None,
+        id_logic: str | None = None,
+    ) -> Self:  # pragma: no cov
+        """Create the Result object or set parent running id if passing Result
+        object.
         """
-        self.run_id: str = running_id
+        if result is None:
+            result: Result = cls(
+                run_id=(run_id or gen_id(id_logic or "", unique=True)),
+                parent_run_id=parent_run_id,
+            )
+        elif parent_run_id:
+            result.set_parent_run_id(parent_run_id)
+        return result
+
+    @model_validator(mode="after")
+    def __prepare_trace(self) -> Self:
+        """Prepare trace field that want to pass after its initialize step."""
+        if self.trace is None:  # pragma: no cove
+            self.trace: TraceLog = get_trace(self.run_id, self.parent_run_id)
+
         return self
 
     def set_parent_run_id(self, running_id: str) -> Self:
@@ -110,6 +105,7 @@ class Result:
         :rtype: Self
         """
         self.parent_run_id: str = running_id
+        self.trace: TraceLog = get_trace(self.run_id, running_id)
         return self
 
     def catch(
@@ -130,26 +126,9 @@ class Result:
         self.__dict__["context"].update(context or {})
         return self
 
-    def receive(self, result: Result) -> Self:
-        """Receive context from another result object.
-
-        :rtype: Self
-        """
-        self.__dict__["status"] = result.status
-        self.__dict__["context"].update(result.context)
-
-        # NOTE: Update running ID from an incoming result.
-        self.parent_run_id = result.parent_run_id
-        self.run_id = result.run_id
-        return self
-
-    @property
-    def trace(self) -> TraceLog:
-        """Return TraceLog object that passing its running ID.
+    def alive_time(self) -> float:  # pragma: no cov
+        """Return total seconds that this object use since it was created.
 
-        :rtype: TraceLog
+        :rtype: float
         """
-        return TraceLog(self.run_id)
-
-    def alive_time(self) -> float:  # pragma: no cov
         return (get_dt_tznow() - self.ts).total_seconds()