ddeutil-workflow 0.0.58__py3-none-any.whl → 0.0.60__py3-none-any.whl

ddeutil/workflow/logs.py

@@ -3,40 +3,44 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
-# [x] Use dynamic config
 # [x] Use fix config for `get_logger`, and Model initialize step.
-"""A Logs module contain Trace dataclass and Audit Pydantic model.
+"""A Logs module contain Trace and Audit Pydantic models for process log from
+the core workflow engine. I separate part of log to 2 types:
+  - Trace: A stdout and stderr log
+  - Audit: An audit release log for tracking incremental running workflow.
 """
 from __future__ import annotations
 
 import json
 import logging
 import os
+import re
 from abc import ABC, abstractmethod
 from collections.abc import Iterator
-from dataclasses import field
 from datetime import datetime
 from functools import lru_cache
 from inspect import Traceback, currentframe, getframeinfo
 from pathlib import Path
 from threading import get_ident
+from types import FrameType
 from typing import ClassVar, Literal, Optional, TypeVar, Union
 
-from pydantic import BaseModel, Field
-from pydantic.dataclasses import dataclass
+from pydantic import BaseModel, ConfigDict, Field
 from pydantic.functional_validators import model_validator
 from typing_extensions import Self
 
-from .__types import DictData, DictStr
+from .__types import DictData
 from .conf import config, dynamic
 from .utils import cut_id, get_dt_now, prepare_newline
 
+METADATA: str = "metadata.json"
+
 
 @lru_cache
 def get_logger(name: str):
     """Return logger object with an input module name.
 
-    :param name: A module name that want to log.
+    :param name: (str) A module name that want to log.
     """
     lg = logging.getLogger(name)
 
@@ -71,13 +75,55 @@ def get_dt_tznow() -> datetime:  # pragma: no cov
 
 
 PREFIX_LOGS: dict[str, dict] = {
-    "CALLER": {"emoji": ""},
-    "STAGE": {"emoji": ""},
-    "JOB": {"emoji": ""},
-    "WORKFLOW": {"emoji": "🏃"},
-    "RELEASE": {"emoji": ""},
-    "POKE": {"emoji": ""},
+    "CALLER": {
+        "emoji": "📍",
+        "desc": "logs from any usage from custom caller function.",
+    },
+    "STAGE": {"emoji": "⚙️", "desc": "logs from stages module."},
+    "JOB": {"emoji": "⛓️", "desc": "logs from job module."},
+    "WORKFLOW": {"emoji": "🏃", "desc": "logs from workflow module."},
+    "RELEASE": {"emoji": "📅", "desc": "logs from release workflow method."},
+    "POKING": {"emoji": "⏰", "desc": "logs from poke workflow method."},
 }  # pragma: no cov
+PREFIX_DEFAULT: str = "CALLER"
+PREFIX_LOGS_REGEX: re.Pattern[str] = re.compile(
+    rf"(^\[(?P<name>{'|'.join(PREFIX_LOGS)})]:\s?)?(?P<message>.*)",
+    re.MULTILINE | re.DOTALL | re.ASCII | re.VERBOSE,
+)  # pragma: no cov
+
+
+class PrefixMsg(BaseModel):
+    """Prefix Message model for receive grouping dict from searching prefix data
+    from logging message.
+    """
+
+    name: Optional[str] = Field(default=None)
+    message: Optional[str] = Field(default=None)
+
+    def prepare(self, extras: Optional[DictData] = None) -> str:
+        """Prepare message with force add prefix before writing trace log.
+
+        :rtype: str
+        """
+        name: str = self.name or PREFIX_DEFAULT
+        emoji: str = (
+            f"{PREFIX_LOGS[name]['emoji']} "
+            if (extras or {}).get("log_add_emoji", True)
+            else ""
+        )
+        return f"{emoji}[{name}]: {self.message}"
+
+
+def extract_msg_prefix(msg: str) -> PrefixMsg:
+    """Extract message prefix from an input message.
+
+    :param msg: A message that want to extract.
+
+    :rtype: PrefixMsg
+    """
+    return PrefixMsg.model_validate(
+        obj=PREFIX_LOGS_REGEX.search(msg).groupdict()
+    )
 
 
 class TraceMeta(BaseModel):  # pragma: no cov
@@ -86,6 +132,7 @@ class TraceMeta(BaseModel):  # pragma: no cov
     """
 
     mode: Literal["stdout", "stderr"] = Field(description="A meta mode.")
+    level: str = Field(description="A log level.")
     datetime: str = Field(description="A datetime in string format.")
     process: int = Field(description="A process ID.")
     thread: int = Field(description="A thread ID.")
@@ -93,29 +140,54 @@ class TraceMeta(BaseModel):  # pragma: no cov
     filename: str = Field(description="A filename of this log.")
     lineno: int = Field(description="A line number of this log.")
 
+    @classmethod
+    def dynamic_frame(
+        cls, frame: FrameType, *, extras: Optional[DictData] = None
+    ) -> Traceback:
+        """Dynamic Frame information base on the `logs_trace_frame_layer` config
+        value that was set from the extra parameter.
+
+        :param frame: (FrameType) The current frame that want to dynamic.
+        :param extras: (DictData) An extra parameter that want to get the
+            `logs_trace_frame_layer` config value.
+        """
+        extras: DictData = extras or {}
+        layer: int = extras.get("logs_trace_frame_layer", 4)
+        for _ in range(layer):
+            _frame: Optional[FrameType] = frame.f_back
+            if _frame is None:
+                raise ValueError(
+                    f"Layer value does not valid, the maximum frame is: {_ + 1}"
+                )
+            frame: FrameType = _frame
+        return getframeinfo(frame)
+
     @classmethod
     def make(
         cls,
         mode: Literal["stdout", "stderr"],
         message: str,
+        level: str,
         *,
         extras: Optional[DictData] = None,
     ) -> Self:
-        """Make the current TraceMeta instance that catching local state.
+        """Make the current metric for contract this TraceMeta model instance
+        that will catch local states like PID, thread identity.
 
-        :param mode: A metadata mode.
-        :param message: A message.
+        :param mode: (Literal["stdout", "stderr"]) A metadata mode.
+        :param message: (str) A message.
+        :param level: (str) A log level.
         :param extras: (DictData) An extra parameter that want to override core
             config values.
 
         :rtype: Self
         """
-        frame_info: Traceback = getframeinfo(
-            currentframe().f_back.f_back.f_back
-        )
+        frame: FrameType = currentframe()
+        frame_info: Traceback = cls.dynamic_frame(frame, extras=extras)
         extras: DictData = extras or {}
         return cls(
             mode=mode,
+            level=level,
             datetime=(
                 get_dt_now(tz=dynamic("tz", extras=extras)).strftime(
                     dynamic("log_datetime_format", extras=extras)
@@ -150,38 +222,45 @@ class TraceData(BaseModel):  # pragma: no cov
 
         :rtype: Self
         """
-        data: DictStr = {"stdout": "", "stderr": "", "meta": []}
+        data: DictData = {"stdout": "", "stderr": "", "meta": []}
 
         for mode in ("stdout", "stderr"):
             if (file / f"{mode}.txt").exists():
                 data[mode] = (file / f"{mode}.txt").read_text(encoding="utf-8")
 
-        if (file / "metadata.json").exists():
+        if (file / METADATA).exists():
             data["meta"] = [
                 json.loads(line)
                 for line in (
-                    (file / "metadata.json")
-                    .read_text(encoding="utf-8")
-                    .splitlines()
+                    (file / METADATA).read_text(encoding="utf-8").splitlines()
                 )
             ]
 
         return cls.model_validate(data)
 
 
-@dataclass(frozen=True)
-class BaseTrace(ABC):  # pragma: no cov
-    """Base Trace dataclass with abstraction class property."""
+class BaseTrace(BaseModel, ABC):  # pragma: no cov
+    """Base Trace model with abstraction class property."""
+
+    model_config = ConfigDict(frozen=True)
 
-    run_id: str
-    parent_run_id: Optional[str] = field(default=None)
-    extras: DictData = field(default_factory=dict, compare=False, repr=False)
+    run_id: str = Field(default="A running ID")
+    parent_run_id: Optional[str] = Field(
+        default=None, description="A parent running ID"
+    )
+    extras: DictData = Field(
+        default_factory=dict,
+        description=(
+            "An extra parameter that want to override on the core config "
+            "values."
+        ),
+    )
 
     @classmethod
     @abstractmethod
     def find_traces(
         cls,
-        path: Path | None = None,
+        path: Optional[Path] = None,
         extras: Optional[DictData] = None,
     ) -> Iterator[TraceData]:  # pragma: no cov
         raise NotImplementedError(
@@ -195,7 +274,7 @@ class BaseTrace(ABC):  # pragma: no cov
         run_id: str,
         force_raise: bool = True,
         *,
-        path: Path | None = None,
+        path: Optional[Path] = None,
         extras: Optional[DictData] = None,
     ) -> TraceData:
         raise NotImplementedError(
@@ -204,24 +283,30 @@ class BaseTrace(ABC):  # pragma: no cov
         )
 
     @abstractmethod
-    def writer(self, message: str, is_err: bool = False) -> None:
+    def writer(self, message: str, level: str, is_err: bool = False) -> None:
         """Write a trace message after making to target pointer object. The
         target can be anything be inherited this class and overwrite this method
         such as file, console, or database.
 
-        :param message: A message after making.
-        :param is_err: A flag for writing with an error trace or not.
+        :param message: (str) A message after making.
+        :param level: (str) A log level.
+        :param is_err: (bool) A flag for writing with an error trace or not.
+            (Default be False)
         """
         raise NotImplementedError(
             "Create writer logic for this trace object before using."
         )
 
     @abstractmethod
-    async def awriter(self, message: str, is_err: bool = False) -> None:
+    async def awriter(
+        self, message: str, level: str, is_err: bool = False
+    ) -> None:
         """Async Write a trace message after making to target pointer object.
 
-        :param message:
-        :param is_err:
+        :param message: (str) A message after making.
+        :param level: (str) A log level.
+        :param is_err: (bool) A flag for writing with an error trace or not.
+            (Default be False)
         """
         raise NotImplementedError(
             "Create async writer logic for this trace object before using."
@@ -247,12 +332,14 @@ class BaseTrace(ABC):  # pragma: no cov
 
         :param message: (str) A message that want to log.
         """
-        msg: str = prepare_newline(self.make_message(message))
+        msg: str = prepare_newline(
+            self.make_message(extract_msg_prefix(message).prepare(self.extras))
+        )
 
         if mode != "debug" or (
             mode == "debug" and dynamic("debug", extras=self.extras)
         ):
-            self.writer(msg, is_err=is_err)
+            self.writer(msg, level=mode, is_err=is_err)
 
         getattr(logger, mode)(msg, stacklevel=3)
 
@@ -304,12 +391,14 @@ class BaseTrace(ABC):  # pragma: no cov
 
         :param message: (str) A message that want to log.
         """
-        msg: str = prepare_newline(self.make_message(message))
+        msg: str = prepare_newline(
+            self.make_message(extract_msg_prefix(message).prepare(self.extras))
+        )
 
         if mode != "debug" or (
             mode == "debug" and dynamic("debug", extras=self.extras)
         ):
-            await self.awriter(msg, is_err=is_err)
+            await self.awriter(msg, level=mode, is_err=is_err)
 
         getattr(logger, mode)(msg, stacklevel=3)
 
@@ -360,7 +449,7 @@ class FileTrace(BaseTrace):  # pragma: no cov
     @classmethod
     def find_traces(
         cls,
-        path: Path | None = None,
+        path: Optional[Path] = None,
         extras: Optional[DictData] = None,
     ) -> Iterator[TraceData]:  # pragma: no cov
         """Find trace logs.
@@ -380,7 +469,7 @@ class FileTrace(BaseTrace):  # pragma: no cov
         run_id: str,
         *,
         force_raise: bool = True,
-        path: Path | None = None,
+        path: Optional[Path] = None,
         extras: Optional[DictData] = None,
     ) -> TraceData:
         """Find trace log with an input specific run ID.
@@ -399,7 +488,7 @@ class FileTrace(BaseTrace):  # pragma: no cov
                 f"Trace log on path {base_path}, does not found trace "
                 f"'run_id={run_id}'."
             )
-        return {}
+        return TraceData(stdout="", stderr="")
 
     @property
     def pointer(self) -> Path:
@@ -433,7 +522,7 @@ class FileTrace(BaseTrace):  # pragma: no cov
         """
         return f"({self.cut_id}) {message}"
 
-    def writer(self, message: str, is_err: bool = False) -> None:
+    def writer(self, message: str, level: str, is_err: bool = False) -> None:
         """Write a trace message after making to target file and write metadata
         in the same path of standard files.
 
@@ -443,28 +532,29 @@ class FileTrace(BaseTrace):  # pragma: no cov
             ... ./logs/run_id=<run-id>/stdout.txt
             ... ./logs/run_id=<run-id>/stderr.txt
 
-        :param message: A message after making.
+        :param message: (str) A message after making.
+        :param level: (str) A log level.
         :param is_err: A flag for writing with an error trace or not.
         """
         if not dynamic("enable_write_log", extras=self.extras):
             return
 
-        write_file: str = "stderr" if is_err else "stdout"
-        trace_meta: TraceMeta = TraceMeta.make(mode=write_file, message=message)
+        mode: Literal["stdout", "stderr"] = "stderr" if is_err else "stdout"
+        trace_meta: TraceMeta = TraceMeta.make(
+            mode=mode, level=level, message=message, extras=self.extras
+        )
 
-        with (self.pointer / f"{write_file}.txt").open(
+        with (self.pointer / f"{mode}.txt").open(
             mode="at", encoding="utf-8"
         ) as f:
             fmt: str = dynamic("log_format_file", extras=self.extras)
             f.write(f"{fmt}\n".format(**trace_meta.model_dump()))
 
-        with (self.pointer / "metadata.json").open(
-            mode="at", encoding="utf-8"
-        ) as f:
+        with (self.pointer / METADATA).open(mode="at", encoding="utf-8") as f:
             f.write(trace_meta.model_dump_json() + "\n")
 
     async def awriter(
-        self, message: str, is_err: bool = False
+        self, message: str, level: str, is_err: bool = False
     ) -> None:  # pragma: no cov
         """Write with async mode."""
         if not dynamic("enable_write_log", extras=self.extras):
@@ -475,17 +565,19 @@ class FileTrace(BaseTrace):  # pragma: no cov
         except ImportError as e:
             raise ImportError("Async mode need aiofiles package") from e
 
-        write_file: str = "stderr" if is_err else "stdout"
-        trace_meta: TraceMeta = TraceMeta.make(mode=write_file, message=message)
+        mode: Literal["stdout", "stderr"] = "stderr" if is_err else "stdout"
+        trace_meta: TraceMeta = TraceMeta.make(
+            mode=mode, level=level, message=message, extras=self.extras
+        )
 
         async with aiofiles.open(
-            self.pointer / f"{write_file}.txt", mode="at", encoding="utf-8"
+            self.pointer / f"{mode}.txt", mode="at", encoding="utf-8"
         ) as f:
             fmt: str = dynamic("log_format_file", extras=self.extras)
             await f.write(f"{fmt}\n".format(**trace_meta.model_dump()))
 
         async with aiofiles.open(
-            self.pointer / "metadata.json", mode="at", encoding="utf-8"
+            self.pointer / METADATA, mode="at", encoding="utf-8"
         ) as f:
             await f.write(trace_meta.model_dump_json() + "\n")
 
@@ -507,7 +599,7 @@ class SQLiteTrace(BaseTrace):  # pragma: no cov
     @classmethod
     def find_traces(
         cls,
-        path: Path | None = None,
+        path: Optional[Path] = None,
         extras: Optional[DictData] = None,
     ) -> Iterator[TraceData]: ...
 
@@ -517,15 +609,19 @@ class SQLiteTrace(BaseTrace):  # pragma: no cov
         run_id: str,
         force_raise: bool = True,
         *,
-        path: Path | None = None,
+        path: Optional[Path] = None,
         extras: Optional[DictData] = None,
     ) -> TraceData: ...
 
     def make_message(self, message: str) -> str: ...
 
-    def writer(self, message: str, is_err: bool = False) -> None: ...
+    def writer(
+        self, message: str, level: str, is_err: bool = False
+    ) -> None: ...
 
-    def awriter(self, message: str, is_err: bool = False) -> None: ...
+    def awriter(
+        self, message: str, level: str, is_err: bool = False
+    ) -> None: ...
 
 
 Trace = TypeVar("Trace", bound=BaseTrace)
@@ -538,7 +634,7 @@ TraceModel = Union[
 def get_trace(
     run_id: str,
     *,
-    parent_run_id: str | None = None,
+    parent_run_id: Optional[str] = None,
     extras: Optional[DictData] = None,
 ) -> TraceModel:  # pragma: no cov
     """Get dynamic Trace instance from the core config (it can override by an
@@ -553,9 +649,11 @@ def get_trace(
     """
     if dynamic("trace_path", extras=extras).is_file():
         return SQLiteTrace(
-            run_id, parent_run_id=parent_run_id, extras=(extras or {})
+            run_id=run_id, parent_run_id=parent_run_id, extras=(extras or {})
         )
-    return FileTrace(run_id, parent_run_id=parent_run_id, extras=(extras or {}))
+    return FileTrace(
+        run_id=run_id, parent_run_id=parent_run_id, extras=(extras or {})
+    )
 
 
 class BaseAudit(BaseModel, ABC):
@@ -619,7 +717,7 @@ class BaseAudit(BaseModel, ABC):
     def find_audit_with_release(
         cls,
         name: str,
-        release: datetime | None = None,
+        release: Optional[datetime] = None,
         *,
         extras: Optional[DictData] = None,
     ) -> Self:
@@ -631,7 +729,7 @@ class BaseAudit(BaseModel, ABC):
         """To something before end up of initial log model."""
 
     @abstractmethod
-    def save(self, excluded: list[str] | None) -> None:  # pragma: no cov
+    def save(self, excluded: Optional[list[str]]) -> None:  # pragma: no cov
         """Save this model logging to target logging store."""
         raise NotImplementedError("Audit should implement ``save`` method.")
 
@@ -676,7 +774,7 @@ class FileAudit(BaseAudit):
     def find_audit_with_release(
         cls,
         name: str,
-        release: datetime | None = None,
+        release: Optional[datetime] = None,
         *,
         extras: Optional[DictData] = None,
     ) -> Self:
@@ -749,7 +847,7 @@ class FileAudit(BaseAudit):
             "audit_path", extras=self.extras
         ) / self.filename_fmt.format(name=self.name, release=self.release)
 
-    def save(self, excluded: list[str] | None) -> Self:
+    def save(self, excluded: Optional[list[str]]) -> Self:
         """Save logging data that receive a context data from a workflow
         execution result.
 
@@ -758,7 +856,7 @@ class FileAudit(BaseAudit):
 
         :rtype: Self
         """
-        trace: Trace = get_trace(
+        trace: TraceModel = get_trace(
             self.run_id,
             parent_run_id=self.parent_run_id,
             extras=self.extras,
@@ -818,16 +916,16 @@ class SQLiteAudit(BaseAudit):  # pragma: no cov
     def find_audit_with_release(
         cls,
         name: str,
-        release: datetime | None = None,
+        release: Optional[datetime] = None,
         *,
         extras: Optional[DictData] = None,
     ) -> Self: ...
 
-    def save(self, excluded: list[str] | None) -> SQLiteAudit:
+    def save(self, excluded: Optional[list[str]]) -> SQLiteAudit:
         """Save logging data that receive a context data from a workflow
         execution result.
         """
-        trace: Trace = get_trace(
+        trace: TraceModel = get_trace(
             self.run_id,
             parent_run_id=self.parent_run_id,
             extras=self.extras,

ddeutil/workflow/params.py

@@ -82,7 +82,9 @@ class DateParam(DefaultParam):  # pragma: no cov
         description="A default date that make from the current date func.",
     )
 
-    def receive(self, value: Optional[str | datetime | date] = None) -> date:
+    def receive(
+        self, value: Optional[Union[str, datetime, date]] = None
+    ) -> date:
         """Receive value that match with date. If an input value pass with
         None, it will use default value instead.
 
@@ -121,7 +123,9 @@ class DatetimeParam(DefaultParam):
         ),
     )
 
-    def receive(self, value: str | datetime | date | None = None) -> datetime:
+    def receive(
+        self, value: Optional[Union[str, datetime, date]] = None
+    ) -> datetime:
         """Receive value that match with datetime. If an input value pass with
         None, it will use default value instead.
 
@@ -155,11 +159,11 @@ class StrParam(DefaultParam):
 
     type: Literal["str"] = "str"
 
-    def receive(self, value: str | None = None) -> str | None:
+    def receive(self, value: Optional[str] = None) -> Optional[str]:
         """Receive value that match with str.
 
         :param value: A value that want to validate with string parameter type.
-        :rtype: str | None
+        :rtype: Optional[str]
         """
         if value is None:
             return self.default
@@ -171,7 +175,7 @@ class IntParam(DefaultParam):
 
     type: Literal["int"] = "int"
 
-    def receive(self, value: int | None = None) -> int | None:
+    def receive(self, value: Optional[int] = None) -> Optional[int]:
         """Receive value that match with int.
 
         :param value: A value that want to validate with integer parameter type.

ddeutil/workflow/result.py

@@ -3,16 +3,16 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
-# [x] Use dynamic config
-"""Result module. It is the data context transfer objects that use by all object
-in this package. This module provide Status enum object and Result dataclass.
+"""A Result module. It is the data context transfer objects that use by all
+object in this package. This module provide Status enum object and Result
+dataclass.
 """
 from __future__ import annotations
 
 from dataclasses import field
 from datetime import datetime
 from enum import IntEnum
-from typing import Optional
+from typing import Optional, Union
 
 from pydantic import ConfigDict
 from pydantic.dataclasses import dataclass
@@ -22,7 +22,7 @@ from typing_extensions import Self
 from .__types import DictData
 from .conf import dynamic
 from .exceptions import ResultException
-from .logs import Trace, get_dt_tznow, get_trace
+from .logs import TraceModel, get_dt_tznow, get_trace
 from .utils import default_gen_id, gen_id, get_dt_now
 
 
@@ -31,14 +31,14 @@ class Status(IntEnum):
     Result dataclass object.
     """
 
-    SUCCESS: int = 0
-    FAILED: int = 1
-    WAIT: int = 2
-    SKIP: int = 3
-    CANCEL: int = 4
+    SUCCESS = 0
+    FAILED = 1
+    WAIT = 2
+    SKIP = 3
+    CANCEL = 4
 
     @property
-    def emoji(self) -> str:
+    def emoji(self) -> str:  # pragma: no cov
         """Return the emoji value of this status.
 
         :rtype: str
@@ -75,16 +75,16 @@ class Result:
     parent_run_id: Optional[str] = field(default=None, compare=False)
     ts: datetime = field(default_factory=get_dt_tznow, compare=False)
 
-    trace: Optional[Trace] = field(default=None, compare=False, repr=False)
+    trace: Optional[TraceModel] = field(default=None, compare=False, repr=False)
     extras: DictData = field(default_factory=dict, compare=False, repr=False)
 
     @classmethod
     def construct_with_rs_or_id(
         cls,
-        result: Result | None = None,
-        run_id: str | None = None,
-        parent_run_id: str | None = None,
-        id_logic: str | None = None,
+        result: Optional[Result] = None,
+        run_id: Optional[str] = None,
+        parent_run_id: Optional[str] = None,
+        id_logic: Optional[str] = None,
         *,
         extras: DictData | None = None,
     ) -> Self:
@@ -121,7 +121,7 @@ class Result:
         :rtype: Self
         """
         if self.trace is None:  # pragma: no cov
-            self.trace: Trace = get_trace(
+            self.trace: TraceModel = get_trace(
                 self.run_id,
                 parent_run_id=self.parent_run_id,
                 extras=self.extras,
@@ -136,14 +136,14 @@ class Result:
         :rtype: Self
         """
         self.parent_run_id: str = running_id
-        self.trace: Trace = get_trace(
+        self.trace: TraceModel = get_trace(
             self.run_id, parent_run_id=running_id, extras=self.extras
         )
         return self
 
     def catch(
         self,
-        status: int | Status,
+        status: Union[int, Status],
         context: DictData | None = None,
         **kwargs,
     ) -> Self: