ddeutil-workflow 0.0.57__py3-none-any.whl → 0.0.59__py3-none-any.whl

ddeutil/workflow/logs.py

@@ -3,7 +3,6 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
-# [x] Use dynamic config
 # [x] Use fix config for `get_logger`, and Model initialize step.
 """A Logs module contain Trace dataclass and Audit Pydantic model.
 """
@@ -14,7 +13,6 @@ import logging
 import os
 from abc import ABC, abstractmethod
 from collections.abc import Iterator
-from dataclasses import field
 from datetime import datetime
 from functools import lru_cache
 from inspect import Traceback, currentframe, getframeinfo
@@ -22,12 +20,11 @@ from pathlib import Path
 from threading import get_ident
 from typing import ClassVar, Literal, Optional, TypeVar, Union
 
-from pydantic import BaseModel, Field
-from pydantic.dataclasses import dataclass
+from pydantic import BaseModel, ConfigDict, Field
 from pydantic.functional_validators import model_validator
 from typing_extensions import Self
 
-from .__types import DictData, DictStr
+from .__types import DictData
 from .conf import config, dynamic
 from .utils import cut_id, get_dt_now, prepare_newline
 
@@ -70,31 +67,44 @@ def get_dt_tznow() -> datetime:  # pragma: no cov
     return get_dt_now(tz=config.tz)
 
 
+PREFIX_LOGS: dict[str, dict] = {
+    "CALLER": {"emoji": ""},
+    "STAGE": {"emoji": ""},
+    "JOB": {"emoji": ""},
+    "WORKFLOW": {"emoji": "🏃"},
+    "RELEASE": {"emoji": ""},
+    "POKE": {"emoji": ""},
+}  # pragma: no cov
+
+
 class TraceMeta(BaseModel):  # pragma: no cov
     """Trace Metadata model for making the current metadata of this CPU, Memory
     process, and thread data.
     """
 
-    mode: Literal["stdout", "stderr"]
-    datetime: str
-    process: int
-    thread: int
-    message: str
-    filename: str
-    lineno: int
+    mode: Literal["stdout", "stderr"] = Field(description="A meta mode.")
+    level: str = Field(description="A log level.")
+    datetime: str = Field(description="A datetime in string format.")
+    process: int = Field(description="A process ID.")
+    thread: int = Field(description="A thread ID.")
+    message: str = Field(description="A message log.")
+    filename: str = Field(description="A filename of this log.")
+    lineno: int = Field(description="A line number of this log.")
 
     @classmethod
     def make(
         cls,
         mode: Literal["stdout", "stderr"],
         message: str,
+        level: str,
         *,
         extras: Optional[DictData] = None,
     ) -> Self:
         """Make the current TraceMeta instance that catching local state.
 
-        :param mode: A metadata mode.
-        :param message: A message.
+        :param mode: (Literal["stdout", "stderr"]) A metadata mode.
+        :param message: (str) A message.
+        :param level: (str) A log level.
         :param extras: (DictData) An extra parameter that want to override core
             config values.
 
@@ -106,6 +116,7 @@ class TraceMeta(BaseModel):  # pragma: no cov
         extras: DictData = extras or {}
         return cls(
             mode=mode,
+            level=level,
             datetime=(
                 get_dt_now(tz=dynamic("tz", extras=extras)).strftime(
                     dynamic("log_datetime_format", extras=extras)
@@ -140,13 +151,11 @@ class TraceData(BaseModel):  # pragma: no cov
 
         :rtype: Self
         """
-        data: DictStr = {"stdout": "", "stderr": "", "meta": []}
-
-        if (file / "stdout.txt").exists():
-            data["stdout"] = (file / "stdout.txt").read_text(encoding="utf-8")
+        data: DictData = {"stdout": "", "stderr": "", "meta": []}
 
-        if (file / "stderr.txt").exists():
-            data["stderr"] = (file / "stderr.txt").read_text(encoding="utf-8")
+        for mode in ("stdout", "stderr"):
+            if (file / f"{mode}.txt").exists():
+                data[mode] = (file / f"{mode}.txt").read_text(encoding="utf-8")
 
         if (file / "metadata.json").exists():
             data["meta"] = [
@@ -161,19 +170,28 @@ class TraceData(BaseModel):  # pragma: no cov
         return cls.model_validate(data)
 
 
-@dataclass(frozen=True)
-class BaseTrace(ABC):  # pragma: no cov
-    """Base Trace dataclass with abstraction class property."""
+class BaseTrace(BaseModel, ABC):  # pragma: no cov
+    """Base Trace model with abstraction class property."""
 
-    run_id: str
-    parent_run_id: Optional[str] = field(default=None)
-    extras: DictData = field(default_factory=dict, compare=False, repr=False)
+    model_config = ConfigDict(frozen=True)
+
+    run_id: str = Field(default="A running ID")
+    parent_run_id: Optional[str] = Field(
+        default=None, description="A parent running ID"
+    )
+    extras: DictData = Field(
+        default_factory=dict,
+        description=(
+            "An extra parameter that want to override on the core config "
+            "values."
+        ),
+    )
 
     @classmethod
     @abstractmethod
     def find_traces(
         cls,
-        path: Path | None = None,
+        path: Optional[Path] = None,
         extras: Optional[DictData] = None,
     ) -> Iterator[TraceData]:  # pragma: no cov
         raise NotImplementedError(
@@ -187,7 +205,7 @@ class BaseTrace(ABC):  # pragma: no cov
         run_id: str,
         force_raise: bool = True,
         *,
-        path: Path | None = None,
+        path: Optional[Path] = None,
         extras: Optional[DictData] = None,
     ) -> TraceData:
         raise NotImplementedError(
@@ -196,24 +214,30 @@ class BaseTrace(ABC):  # pragma: no cov
         )
 
     @abstractmethod
-    def writer(self, message: str, is_err: bool = False) -> None:
+    def writer(self, message: str, level: str, is_err: bool = False) -> None:
         """Write a trace message after making to target pointer object. The
         target can be anything be inherited this class and overwrite this method
         such as file, console, or database.
 
-        :param message: A message after making.
-        :param is_err: A flag for writing with an error trace or not.
+        :param message: (str) A message after making.
+        :param level: (str) A log level.
+        :param is_err: (bool) A flag for writing with an error trace or not.
+            (Default be False)
         """
         raise NotImplementedError(
             "Create writer logic for this trace object before using."
         )
 
     @abstractmethod
-    async def awriter(self, message: str, is_err: bool = False) -> None:
+    async def awriter(
+        self, message: str, level: str, is_err: bool = False
+    ) -> None:
         """Async Write a trace message after making to target pointer object.
 
-        :param message:
-        :param is_err:
+        :param message: (str) A message after making.
+        :param level: (str) A log level.
+        :param is_err: (bool) A flag for writing with an error trace or not.
+            (Default be False)
         """
         raise NotImplementedError(
             "Create async writer logic for this trace object before using."
@@ -244,7 +268,7 @@ class BaseTrace(ABC):  # pragma: no cov
         if mode != "debug" or (
             mode == "debug" and dynamic("debug", extras=self.extras)
         ):
-            self.writer(msg, is_err=is_err)
+            self.writer(msg, level=mode, is_err=is_err)
 
         getattr(logger, mode)(msg, stacklevel=3)
 
@@ -288,16 +312,30 @@ class BaseTrace(ABC):  # pragma: no cov
         """
         self.__logging(message, mode="exception", is_err=True)
 
+    async def __alogging(
+        self, message: str, mode: str, *, is_err: bool = False
+    ) -> None:
+        """Write trace log with append mode and logging this message with any
+        logging level.
+
+        :param message: (str) A message that want to log.
+        """
+        msg: str = prepare_newline(self.make_message(message))
+
+        if mode != "debug" or (
+            mode == "debug" and dynamic("debug", extras=self.extras)
+        ):
+            await self.awriter(msg, level=mode, is_err=is_err)
+
+        getattr(logger, mode)(msg, stacklevel=3)
+
     async def adebug(self, message: str) -> None:  # pragma: no cov
         """Async write trace log with append mode and logging this message with
         the DEBUG level.
 
         :param message: (str) A message that want to log.
         """
-        msg: str = self.make_message(message)
-        if dynamic("debug", extras=self.extras):
-            await self.awriter(msg)
-        logger.info(msg, stacklevel=2)
+        await self.__alogging(message, mode="debug")
 
     async def ainfo(self, message: str) -> None:  # pragma: no cov
         """Async write trace log with append mode and logging this message with
@@ -305,9 +343,7 @@ class BaseTrace(ABC):  # pragma: no cov
 
         :param message: (str) A message that want to log.
         """
-        msg: str = self.make_message(message)
-        await self.awriter(msg)
-        logger.info(msg, stacklevel=2)
+        await self.__alogging(message, mode="info")
 
     async def awarning(self, message: str) -> None:  # pragma: no cov
         """Async write trace log with append mode and logging this message with
@@ -315,9 +351,7 @@ class BaseTrace(ABC):  # pragma: no cov
 
         :param message: (str) A message that want to log.
         """
-        msg: str = self.make_message(message)
-        await self.awriter(msg)
-        logger.warning(msg, stacklevel=2)
+        await self.__alogging(message, mode="warning")
 
     async def aerror(self, message: str) -> None:  # pragma: no cov
         """Async write trace log with append mode and logging this message with
@@ -325,9 +359,7 @@ class BaseTrace(ABC):  # pragma: no cov
 
         :param message: (str) A message that want to log.
         """
-        msg: str = self.make_message(message)
-        await self.awriter(msg, is_err=True)
-        logger.error(msg, stacklevel=2)
+        await self.__alogging(message, mode="error", is_err=True)
 
     async def aexception(self, message: str) -> None:  # pragma: no cov
         """Async write trace log with append mode and logging this message with
@@ -335,9 +367,7 @@ class BaseTrace(ABC):  # pragma: no cov
 
         :param message: (str) A message that want to log.
         """
-        msg: str = self.make_message(message)
-        await self.awriter(msg, is_err=True)
-        logger.exception(msg, stacklevel=2)
+        await self.__alogging(message, mode="exception", is_err=True)
 
 
 class FileTrace(BaseTrace):  # pragma: no cov
@@ -346,12 +376,12 @@ class FileTrace(BaseTrace):  # pragma: no cov
     @classmethod
     def find_traces(
         cls,
-        path: Path | None = None,
+        path: Optional[Path] = None,
         extras: Optional[DictData] = None,
     ) -> Iterator[TraceData]:  # pragma: no cov
         """Find trace logs.
 
-        :param path: (Path)
+        :param path: (Path) A trace path that want to find.
         :param extras: An extra parameter that want to override core config.
         """
         for file in sorted(
@@ -364,16 +394,16 @@ class FileTrace(BaseTrace):  # pragma: no cov
     def find_trace_with_id(
         cls,
         run_id: str,
-        force_raise: bool = True,
         *,
-        path: Path | None = None,
+        force_raise: bool = True,
+        path: Optional[Path] = None,
         extras: Optional[DictData] = None,
     ) -> TraceData:
         """Find trace log with an input specific run ID.
 
         :param run_id: A running ID of trace log.
-        :param force_raise:
-        :param path:
+        :param force_raise: (bool)
+        :param path: (Path)
         :param extras: An extra parameter that want to override core config.
         """
         base_path: Path = path or dynamic("trace_path", extras=extras)
@@ -385,7 +415,7 @@ class FileTrace(BaseTrace):  # pragma: no cov
                 f"Trace log on path {base_path}, does not found trace "
                 f"'run_id={run_id}'."
             )
-        return {}
+        return TraceData(stdout="", stderr="")
 
     @property
     def pointer(self) -> Path:
@@ -419,7 +449,7 @@ class FileTrace(BaseTrace):  # pragma: no cov
         """
         return f"({self.cut_id}) {message}"
 
-    def writer(self, message: str, is_err: bool = False) -> None:
+    def writer(self, message: str, level: str, is_err: bool = False) -> None:
         """Write a trace message after making to target file and write metadata
         in the same path of standard files.
 
@@ -429,16 +459,19 @@ class FileTrace(BaseTrace):  # pragma: no cov
             ... ./logs/run_id=<run-id>/stdout.txt
             ... ./logs/run_id=<run-id>/stderr.txt
 
-        :param message: A message after making.
+        :param message: (str) A message after making.
+        :param level: (str) A log level.
         :param is_err: A flag for writing with an error trace or not.
         """
         if not dynamic("enable_write_log", extras=self.extras):
             return
 
-        write_file: str = "stderr" if is_err else "stdout"
-        trace_meta: TraceMeta = TraceMeta.make(mode=write_file, message=message)
+        mode: Literal["stdout", "stderr"] = "stderr" if is_err else "stdout"
+        trace_meta: TraceMeta = TraceMeta.make(
+            mode=mode, level=level, message=message
+        )
 
-        with (self.pointer / f"{write_file}.txt").open(
+        with (self.pointer / f"{mode}.txt").open(
             mode="at", encoding="utf-8"
         ) as f:
             fmt: str = dynamic("log_format_file", extras=self.extras)
@@ -450,7 +483,7 @@ class FileTrace(BaseTrace):  # pragma: no cov
             f.write(trace_meta.model_dump_json() + "\n")
 
     async def awriter(
-        self, message: str, is_err: bool = False
+        self, message: str, level: str, is_err: bool = False
     ) -> None:  # pragma: no cov
         """Write with async mode."""
         if not dynamic("enable_write_log", extras=self.extras):
@@ -461,11 +494,13 @@ class FileTrace(BaseTrace):  # pragma: no cov
         except ImportError as e:
             raise ImportError("Async mode need aiofiles package") from e
 
-        write_file: str = "stderr" if is_err else "stdout"
-        trace_meta: TraceMeta = TraceMeta.make(mode=write_file, message=message)
+        mode: Literal["stdout", "stderr"] = "stderr" if is_err else "stdout"
+        trace_meta: TraceMeta = TraceMeta.make(
+            mode=mode, level=level, message=message
+        )
 
         async with aiofiles.open(
-            self.pointer / f"{write_file}.txt", mode="at", encoding="utf-8"
+            self.pointer / f"{mode}.txt", mode="at", encoding="utf-8"
         ) as f:
             fmt: str = dynamic("log_format_file", extras=self.extras)
             await f.write(f"{fmt}\n".format(**trace_meta.model_dump()))
@@ -493,7 +528,7 @@ class SQLiteTrace(BaseTrace):  # pragma: no cov
     @classmethod
     def find_traces(
         cls,
-        path: Path | None = None,
+        path: Optional[Path] = None,
         extras: Optional[DictData] = None,
     ) -> Iterator[TraceData]: ...
 
@@ -503,15 +538,19 @@ class SQLiteTrace(BaseTrace):  # pragma: no cov
         run_id: str,
         force_raise: bool = True,
         *,
-        path: Path | None = None,
+        path: Optional[Path] = None,
         extras: Optional[DictData] = None,
     ) -> TraceData: ...
 
     def make_message(self, message: str) -> str: ...
 
-    def writer(self, message: str, is_err: bool = False) -> None: ...
+    def writer(
+        self, message: str, level: str, is_err: bool = False
+    ) -> None: ...
 
-    def awriter(self, message: str, is_err: bool = False) -> None: ...
+    def awriter(
+        self, message: str, level: str, is_err: bool = False
+    ) -> None: ...
 
 
 Trace = TypeVar("Trace", bound=BaseTrace)
@@ -524,7 +563,7 @@ TraceModel = Union[
 def get_trace(
     run_id: str,
     *,
-    parent_run_id: str | None = None,
+    parent_run_id: Optional[str] = None,
     extras: Optional[DictData] = None,
 ) -> TraceModel:  # pragma: no cov
     """Get dynamic Trace instance from the core config (it can override by an
@@ -539,9 +578,11 @@ def get_trace(
     """
     if dynamic("trace_path", extras=extras).is_file():
         return SQLiteTrace(
-            run_id, parent_run_id=parent_run_id, extras=(extras or {})
+            run_id=run_id, parent_run_id=parent_run_id, extras=(extras or {})
         )
-    return FileTrace(run_id, parent_run_id=parent_run_id, extras=(extras or {}))
+    return FileTrace(
+        run_id=run_id, parent_run_id=parent_run_id, extras=(extras or {})
+    )
 
 
 class BaseAudit(BaseModel, ABC):
@@ -605,7 +646,7 @@ class BaseAudit(BaseModel, ABC):
     def find_audit_with_release(
         cls,
         name: str,
-        release: datetime | None = None,
+        release: Optional[datetime] = None,
         *,
         extras: Optional[DictData] = None,
     ) -> Self:
@@ -617,7 +658,7 @@ class BaseAudit(BaseModel, ABC):
         """To something before end up of initial log model."""
 
     @abstractmethod
-    def save(self, excluded: list[str] | None) -> None:  # pragma: no cov
+    def save(self, excluded: Optional[list[str]]) -> None:  # pragma: no cov
         """Save this model logging to target logging store."""
         raise NotImplementedError("Audit should implement ``save`` method.")
 
@@ -662,7 +703,7 @@ class FileAudit(BaseAudit):
     def find_audit_with_release(
         cls,
         name: str,
-        release: datetime | None = None,
+        release: Optional[datetime] = None,
         *,
         extras: Optional[DictData] = None,
     ) -> Self:
@@ -735,7 +776,7 @@ class FileAudit(BaseAudit):
             "audit_path", extras=self.extras
         ) / self.filename_fmt.format(name=self.name, release=self.release)
 
-    def save(self, excluded: list[str] | None) -> Self:
+    def save(self, excluded: Optional[list[str]]) -> Self:
         """Save logging data that receive a context data from a workflow
         execution result.
 
@@ -744,7 +785,7 @@ class FileAudit(BaseAudit):
 
         :rtype: Self
         """
-        trace: Trace = get_trace(
+        trace: TraceModel = get_trace(
             self.run_id,
             parent_run_id=self.parent_run_id,
             extras=self.extras,
@@ -804,16 +845,16 @@ class SQLiteAudit(BaseAudit):  # pragma: no cov
     def find_audit_with_release(
         cls,
         name: str,
-        release: datetime | None = None,
+        release: Optional[datetime] = None,
         *,
         extras: Optional[DictData] = None,
     ) -> Self: ...
 
-    def save(self, excluded: list[str] | None) -> SQLiteAudit:
+    def save(self, excluded: Optional[list[str]]) -> SQLiteAudit:
         """Save logging data that receive a context data from a workflow
         execution result.
         """
-        trace: Trace = get_trace(
+        trace: TraceModel = get_trace(
             self.run_id,
             parent_run_id=self.parent_run_id,
             extras=self.extras,

ddeutil/workflow/params.py

@@ -82,7 +82,9 @@ class DateParam(DefaultParam):  # pragma: no cov
         description="A default date that make from the current date func.",
     )
 
-    def receive(self, value: Optional[str | datetime | date] = None) -> date:
+    def receive(
+        self, value: Optional[Union[str, datetime, date]] = None
+    ) -> date:
         """Receive value that match with date. If an input value pass with
         None, it will use default value instead.
 
@@ -121,7 +123,9 @@ class DatetimeParam(DefaultParam):
         ),
     )
 
-    def receive(self, value: str | datetime | date | None = None) -> datetime:
+    def receive(
+        self, value: Optional[Union[str, datetime, date]] = None
+    ) -> datetime:
         """Receive value that match with datetime. If an input value pass with
         None, it will use default value instead.
 
@@ -155,11 +159,11 @@ class StrParam(DefaultParam):
 
     type: Literal["str"] = "str"
 
-    def receive(self, value: str | None = None) -> str | None:
+    def receive(self, value: Optional[str] = None) -> Optional[str]:
         """Receive value that match with str.
 
         :param value: A value that want to validate with string parameter type.
-        :rtype: str | None
+        :rtype: Optional[str]
         """
         if value is None:
             return self.default
@@ -171,7 +175,7 @@ class IntParam(DefaultParam):
 
     type: Literal["int"] = "int"
 
-    def receive(self, value: int | None = None) -> int | None:
+    def receive(self, value: Optional[int] = None) -> Optional[int]:
         """Receive value that match with int.
 
         :param value: A value that want to validate with integer parameter type.
@@ -190,10 +194,13 @@ class IntParam(DefaultParam):
 
 
 class FloatParam(DefaultParam):  # pragma: no cov
+    """Float parameter."""
+
     type: Literal["float"] = "float"
     precision: int = 6
 
     def rounding(self, value: float) -> float:
+        """Rounding float value with the specific precision field."""
         round_str: str = f"{{0:.{self.precision}f}}"
         return float(round_str.format(round(value, self.precision)))
 
@@ -224,6 +231,7 @@ class DecimalParam(DefaultParam):  # pragma: no cov
     precision: int = 6
 
     def rounding(self, value: Decimal) -> Decimal:
+        """Rounding float value with the specific precision field."""
         return value.quantize(Decimal(10) ** -self.precision)
 
     def receive(self, value: float | Decimal | None = None) -> Decimal:

ddeutil/workflow/result.py

@@ -3,16 +3,16 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
-# [x] Use dynamic config
-"""Result module. It is the data context transfer objects that use by all object
-in this package. This module provide Status enum object and Result dataclass.
+"""A Result module. It is the data context transfer objects that use by all
+object in this package. This module provide Status enum object and Result
+dataclass.
 """
 from __future__ import annotations
 
 from dataclasses import field
 from datetime import datetime
 from enum import IntEnum
-from typing import Optional
+from typing import Optional, Union
 
 from pydantic import ConfigDict
 from pydantic.dataclasses import dataclass
@@ -22,7 +22,7 @@ from typing_extensions import Self
 from .__types import DictData
 from .conf import dynamic
 from .exceptions import ResultException
-from .logs import Trace, get_dt_tznow, get_trace
+from .logs import TraceModel, get_dt_tznow, get_trace
 from .utils import default_gen_id, gen_id, get_dt_now
 
 
@@ -31,11 +31,11 @@ class Status(IntEnum):
     Result dataclass object.
     """
 
-    SUCCESS: int = 0
-    FAILED: int = 1
-    WAIT: int = 2
-    SKIP: int = 3
-    CANCEL: int = 4
+    SUCCESS = 0
+    FAILED = 1
+    WAIT = 2
+    SKIP = 3
+    CANCEL = 4
 
     @property
     def emoji(self) -> str:
@@ -75,16 +75,16 @@ class Result:
     parent_run_id: Optional[str] = field(default=None, compare=False)
     ts: datetime = field(default_factory=get_dt_tznow, compare=False)
 
-    trace: Optional[Trace] = field(default=None, compare=False, repr=False)
+    trace: Optional[TraceModel] = field(default=None, compare=False, repr=False)
     extras: DictData = field(default_factory=dict, compare=False, repr=False)
 
     @classmethod
     def construct_with_rs_or_id(
         cls,
-        result: Result | None = None,
-        run_id: str | None = None,
-        parent_run_id: str | None = None,
-        id_logic: str | None = None,
+        result: Optional[Result] = None,
+        run_id: Optional[str] = None,
+        parent_run_id: Optional[str] = None,
+        id_logic: Optional[str] = None,
         *,
         extras: DictData | None = None,
     ) -> Self:
@@ -121,7 +121,7 @@ class Result:
         :rtype: Self
         """
         if self.trace is None:  # pragma: no cov
-            self.trace: Trace = get_trace(
+            self.trace: TraceModel = get_trace(
                 self.run_id,
                 parent_run_id=self.parent_run_id,
                 extras=self.extras,
@@ -136,14 +136,14 @@ class Result:
         :rtype: Self
         """
         self.parent_run_id: str = running_id
-        self.trace: Trace = get_trace(
+        self.trace: TraceModel = get_trace(
             self.run_id, parent_run_id=running_id, extras=self.extras
         )
         return self
 
     def catch(
         self,
-        status: int | Status,
+        status: Union[int, Status],
         context: DictData | None = None,
         **kwargs,
     ) -> Self: