ddeutil-workflow 0.0.67__py3-none-any.whl → 0.0.69__py3-none-any.whl

ddeutil/workflow/{logs.py → traces.py}

@@ -3,12 +3,7 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
-# [x] Use fix config for `get_logger`, and Model initialize step.
-"""A Logs module contain Trace and Audit Pydantic models for process log from
-the core workflow engine. I separate part of log to 2 types:
-  - Trace: A stdout and stderr log
-  - Audit: An audit release log for tracking incremental running workflow.
-"""
+# [x] Use fix config for `set_logging`, and Model initialize step.
 from __future__ import annotations
 
 import json
@@ -17,7 +12,6 @@ import os
 import re
 from abc import ABC, abstractmethod
 from collections.abc import Iterator
-from datetime import datetime
 from functools import lru_cache
 from inspect import Traceback, currentframe, getframeinfo
 from pathlib import Path
@@ -26,7 +20,6 @@ from types import FrameType
 from typing import ClassVar, Final, Literal, Optional, TypeVar, Union
 
 from pydantic import BaseModel, ConfigDict, Field
-from pydantic.functional_validators import model_validator
 from typing_extensions import Self
 
 from .__types import DictData
@@ -34,44 +27,35 @@ from .conf import config, dynamic
 from .utils import cut_id, get_dt_now, prepare_newline
 
 METADATA: str = "metadata.json"
+logger = logging.getLogger("ddeutil.workflow")
 
 
 @lru_cache
-def get_logger(name: str):
-    """Return logger object with an input module name.
+def set_logging(name: str) -> logging.Logger:
+    """Return logger object with an input module name that already implement the
+    custom handler and formatter from this package config.
 
     :param name: (str) A module name that want to log.
+
+    :rtype: logging.Logger
     """
-    lg = logging.getLogger(name)
+    _logger = logging.getLogger(name)
 
     # NOTE: Developers using this package can then disable all logging just for
     #   this package by;
     #
     #   `logging.getLogger('ddeutil.workflow').propagate = False`
     #
-    lg.addHandler(logging.NullHandler())
+    _logger.addHandler(logging.NullHandler())
 
     formatter = logging.Formatter(
-        fmt=config.log_format,
-        datefmt=config.log_datetime_format,
+        fmt=config.log_format, datefmt=config.log_datetime_format
     )
-    stream = logging.StreamHandler()
-    stream.setFormatter(formatter)
-    lg.addHandler(stream)
-
-    lg.setLevel(logging.DEBUG if config.debug else logging.INFO)
-    return lg
-
-
-logger = get_logger("ddeutil.workflow")
-
-
-def get_dt_tznow() -> datetime:  # pragma: no cov
-    """Return the current datetime object that passing the config timezone.
-
-    :rtype: datetime
-    """
-    return get_dt_now(tz=config.tz)
+    stream_handler = logging.StreamHandler()
+    stream_handler.setFormatter(formatter)
+    _logger.addHandler(stream_handler)
+    _logger.setLevel(logging.DEBUG if config.debug else logging.INFO)
+    return _logger
 
 
 PREFIX_LOGS: Final[dict[str, dict]] = {
@@ -97,8 +81,22 @@ class PrefixMsg(BaseModel):
     from logging message.
     """
 
-    name: Optional[str] = Field(default=None)
-    message: Optional[str] = Field(default=None)
+    name: Optional[str] = Field(default=None, description="A prefix name.")
+    message: Optional[str] = Field(default=None, description="A message.")
+
+    @classmethod
+    def from_str(cls, msg: str) -> Self:
+        """Extract message prefix from an input message.
+
+        Args:
+            msg (str): A message that want to extract.
+
+        Returns:
+            PrefixMsg: the validated model from a string message.
+        """
+        return PrefixMsg.model_validate(
+            obj=PREFIX_LOGS_REGEX.search(msg).groupdict()
+        )
 
     def prepare(self, extras: Optional[DictData] = None) -> str:
         """Prepare message with force add prefix before writing trace log.
@@ -117,18 +115,6 @@ class PrefixMsg(BaseModel):
         return f"{emoji}[{name}]: {self.message}"
 
 
-def extract_msg_prefix(msg: str) -> PrefixMsg:
-    """Extract message prefix from an input message.
-
-    :param msg: A message that want to extract.
-
-    :rtype: PrefixMsg
-    """
-    return PrefixMsg.model_validate(
-        obj=PREFIX_LOGS_REGEX.search(msg).groupdict()
-    )
-
-
 class TraceMeta(BaseModel):  # pragma: no cov
     """Trace Metadata model for making the current metadata of this CPU, Memory
     process, and thread data.
@@ -247,10 +233,6 @@ class BaseTrace(BaseModel, ABC):  # pragma: no cov
 
     model_config = ConfigDict(frozen=True)
 
-    run_id: str = Field(default="A running ID")
-    parent_run_id: Optional[str] = Field(
-        default=None, description="A parent running ID"
-    )
     extras: DictData = Field(
         default_factory=dict,
         description=(
@@ -258,6 +240,11 @@ class BaseTrace(BaseModel, ABC):  # pragma: no cov
             "values."
         ),
     )
+    run_id: str = Field(description="A running ID")
+    parent_run_id: Optional[str] = Field(
+        default=None,
+        description="A parent running ID",
+    )
 
     @classmethod
     @abstractmethod
@@ -266,6 +253,17 @@ class BaseTrace(BaseModel, ABC):  # pragma: no cov
         path: Optional[Path] = None,
         extras: Optional[DictData] = None,
     ) -> Iterator[TraceData]:  # pragma: no cov
+        """Return iterator of TraceData models from the target pointer.
+
+        Args:
+            path (:obj:`Path`, optional): A pointer path that want to override.
+            extras (:obj:`DictData`, optional): An extras parameter that want to
+                override default engine config.
+
+        Returns:
+            Iterator[TracData]: An iterator object that generate a TracData
+                model.
+        """
         raise NotImplementedError(
             "Trace dataclass should implement `find_traces` class-method."
         )
@@ -286,7 +284,12 @@ class BaseTrace(BaseModel, ABC):  # pragma: no cov
         )
 
     @abstractmethod
-    def writer(self, message: str, level: str, is_err: bool = False) -> None:
+    def writer(
+        self,
+        message: str,
+        level: str,
+        is_err: bool = False,
+    ) -> None:
         """Write a trace message after making to target pointer object. The
         target can be anything be inherited this class and overwrite this method
         such as file, console, or database.
@@ -302,7 +305,10 @@ class BaseTrace(BaseModel, ABC):  # pragma: no cov
 
     @abstractmethod
     async def awriter(
-        self, message: str, level: str, is_err: bool = False
+        self,
+        message: str,
+        level: str,
+        is_err: bool = False,
     ) -> None:
         """Async Write a trace message after making to target pointer object.
 
@@ -327,6 +333,89 @@ class BaseTrace(BaseModel, ABC):  # pragma: no cov
             "Adjust make message method for this trace object before using."
         )
 
+
+class ConsoleTrace(BaseTrace):  # pragma: no cov
+    """Console Trace log model."""
+
+    @classmethod
+    def find_traces(
+        cls,
+        path: Optional[Path] = None,
+        extras: Optional[DictData] = None,
+    ) -> Iterator[TraceData]:  # pragma: no cov
+        raise NotImplementedError(
+            "Console Trace does not support to find history traces data."
+        )
+
+    @classmethod
+    def find_trace_with_id(
+        cls,
+        run_id: str,
+        force_raise: bool = True,
+        *,
+        path: Optional[Path] = None,
+        extras: Optional[DictData] = None,
+    ) -> TraceData:
+        raise NotImplementedError(
+            "Console Trace does not support to find history traces data with "
+            "the specific running ID."
+        )
+
+    def writer(
+        self,
+        message: str,
+        level: str,
+        is_err: bool = False,
+    ) -> None:
+        """Write a trace message after making to target pointer object. The
+        target can be anything be inherited this class and overwrite this method
+        such as file, console, or database.
+
+        :param message: (str) A message after making.
+        :param level: (str) A log level.
+        :param is_err: (bool) A flag for writing with an error trace or not.
+            (Default be False)
+        """
+
+    async def awriter(
+        self,
+        message: str,
+        level: str,
+        is_err: bool = False,
+    ) -> None:
+        """Async Write a trace message after making to target pointer object.
+
+        :param message: (str) A message after making.
+        :param level: (str) A log level.
+        :param is_err: (bool) A flag for writing with an error trace or not.
+            (Default be False)
+        """
+
+    @property
+    def cut_id(self) -> str:
+        """Combine cutting ID of parent running ID if it set.
+
+        :rtype: str
+        """
+        cut_run_id: str = cut_id(self.run_id)
+        if not self.parent_run_id:
+            return f"{cut_run_id}"
+
+        cut_parent_run_id: str = cut_id(self.parent_run_id)
+        return f"{cut_parent_run_id} -> {cut_run_id}"
+
+    def make_message(self, message: str) -> str:
+        """Prepare and Make a message before write and log steps.
+
+        :param message: (str) A message that want to prepare and make before.
+
+        :rtype: str
+        """
+        return prepare_newline(
+            f"({self.cut_id}) "
+            f"{PrefixMsg.from_str(message).prepare(self.extras)}"
+        )
+
     def __logging(
         self, message: str, mode: str, *, is_err: bool = False
     ) -> None:
@@ -342,7 +431,7 @@ class BaseTrace(BaseModel, ABC):  # pragma: no cov
         ):
             self.writer(msg, level=mode, is_err=is_err)
 
-        getattr(logger, mode)(msg, stacklevel=3)
+        getattr(logger, mode)(msg, stacklevel=3, extra={"cut_id": self.cut_id})
 
     def debug(self, message: str):
         """Write trace log with append mode and logging this message with the
@@ -399,7 +488,7 @@ class BaseTrace(BaseModel, ABC):  # pragma: no cov
         ):
             await self.awriter(msg, level=mode, is_err=is_err)
 
-        getattr(logger, mode)(msg, stacklevel=3)
+        getattr(logger, mode)(msg, stacklevel=3, extra={"cut_id": self.cut_id})
 
     async def adebug(self, message: str) -> None:  # pragma: no cov
         """Async write trace log with append mode and logging this message with
@@ -442,7 +531,7 @@ class BaseTrace(BaseModel, ABC):  # pragma: no cov
         await self.__alogging(message, mode="exception", is_err=True)
 
 
-class FileTrace(BaseTrace):  # pragma: no cov
+class FileTrace(ConsoleTrace):  # pragma: no cov
     """File Trace dataclass that write file to the local storage."""
 
     @classmethod
@@ -491,6 +580,11 @@ class FileTrace(BaseTrace):  # pragma: no cov
 
     @property
     def pointer(self) -> Path:
+        """Pointer of the target path that use to writing trace log or searching
+        trace log.
+
+        :rtype: Path
+        """
         log_file: Path = (
             dynamic("trace_path", extras=self.extras)
             / f"run_id={self.parent_run_id or self.run_id}"
@@ -499,31 +593,12 @@ class FileTrace(BaseTrace):  # pragma: no cov
             log_file.mkdir(parents=True)
         return log_file
 
-    @property
-    def cut_id(self) -> str:
-        """Combine cutting ID of parent running ID if it set.
-
-        :rtype: str
-        """
-        cut_run_id: str = cut_id(self.run_id)
-        if not self.parent_run_id:
-            return f"{cut_run_id}"
-
-        cut_parent_run_id: str = cut_id(self.parent_run_id)
-        return f"{cut_parent_run_id} -> {cut_run_id}"
-
-    def make_message(self, message: str) -> str:
-        """Prepare and Make a message before write and log steps.
-
-        :param message: (str) A message that want to prepare and make before.
-
-        :rtype: str
-        """
-        return prepare_newline(
-            f"({self.cut_id}) {extract_msg_prefix(message).prepare(self.extras)}"
-        )
-
-    def writer(self, message: str, level: str, is_err: bool = False) -> None:
+    def writer(
+        self,
+        message: str,
+        level: str,
+        is_err: bool = False,
+    ) -> None:
         """Write a trace message after making to target file and write metadata
         in the same path of standard files.
 
@@ -555,7 +630,10 @@ class FileTrace(BaseTrace):  # pragma: no cov
             f.write(trace_meta.model_dump_json() + "\n")
 
     async def awriter(
-        self, message: str, level: str, is_err: bool = False
+        self,
+        message: str,
+        level: str,
+        is_err: bool = False,
     ) -> None:  # pragma: no cov
         """Write with async mode."""
         if not dynamic("enable_write_log", extras=self.extras):
@@ -583,7 +661,7 @@ class FileTrace(BaseTrace):  # pragma: no cov
             await f.write(trace_meta.model_dump_json() + "\n")
 
 
-class SQLiteTrace(BaseTrace):  # pragma: no cov
+class SQLiteTrace(ConsoleTrace):  # pragma: no cov
     """SQLite Trace dataclass that write trace log to the SQLite database file."""
 
     table_name: ClassVar[str] = "audits"
@@ -617,16 +695,23 @@ class SQLiteTrace(BaseTrace):  # pragma: no cov
     def make_message(self, message: str) -> str: ...
 
     def writer(
-        self, message: str, level: str, is_err: bool = False
+        self,
+        message: str,
+        level: str,
+        is_err: bool = False,
     ) -> None: ...
 
     def awriter(
-        self, message: str, level: str, is_err: bool = False
+        self,
+        message: str,
+        level: str,
+        is_err: bool = False,
     ) -> None: ...
 
 
 Trace = TypeVar("Trace", bound=BaseTrace)
 TraceModel = Union[
+    ConsoleTrace,
     FileTrace,
     SQLiteTrace,
 ]
@@ -655,307 +740,3 @@ def get_trace(
     return FileTrace(
         run_id=run_id, parent_run_id=parent_run_id, extras=(extras or {})
     )
-
-
-class BaseAudit(BaseModel, ABC):
-    """Base Audit Pydantic Model with abstraction class property that implement
-    only model fields. This model should to use with inherit to logging
-    subclass like file, sqlite, etc.
-    """
-
-    extras: DictData = Field(
-        default_factory=dict,
-        description="An extras parameter that want to override core config",
-    )
-    name: str = Field(description="A workflow name.")
-    release: datetime = Field(description="A release datetime.")
-    type: str = Field(description="A running type before logging.")
-    context: DictData = Field(
-        default_factory=dict,
-        description="A context that receive from a workflow execution result.",
-    )
-    parent_run_id: Optional[str] = Field(
-        default=None, description="A parent running ID."
-    )
-    run_id: str = Field(description="A running ID")
-    update: datetime = Field(default_factory=get_dt_tznow)
-    execution_time: float = Field(default=0, description="An execution time.")
-
-    @model_validator(mode="after")
-    def __model_action(self) -> Self:
-        """Do before the Audit action with WORKFLOW_AUDIT_ENABLE_WRITE env variable.
-
-        :rtype: Self
-        """
-        if dynamic("enable_write_audit", extras=self.extras):
-            self.do_before()
-        return self
-
-    @classmethod
-    @abstractmethod
-    def is_pointed(
-        cls,
-        name: str,
-        release: datetime,
-        *,
-        extras: Optional[DictData] = None,
-    ) -> bool:
-        raise NotImplementedError(
-            "Audit should implement `is_pointed` class-method"
-        )
-
-    @classmethod
-    @abstractmethod
-    def find_audits(
-        cls, name: str, *, extras: Optional[DictData] = None
-    ) -> Iterator[Self]:
-        raise NotImplementedError(
-            "Audit should implement `find_audits` class-method"
-        )
-
-    @classmethod
-    @abstractmethod
-    def find_audit_with_release(
-        cls,
-        name: str,
-        release: Optional[datetime] = None,
-        *,
-        extras: Optional[DictData] = None,
-    ) -> Self:
-        raise NotImplementedError(
-            "Audit should implement `find_audit_with_release` class-method"
-        )
-
-    def do_before(self) -> None:  # pragma: no cov
-        """To something before end up of initial log model."""
-
-    @abstractmethod
-    def save(self, excluded: Optional[list[str]]) -> None:  # pragma: no cov
-        """Save this model logging to target logging store."""
-        raise NotImplementedError("Audit should implement ``save`` method.")
-
-
-class FileAudit(BaseAudit):
-    """File Audit Pydantic Model that use to saving log data from result of
-    workflow execution. It inherits from BaseAudit model that implement the
-    ``self.save`` method for file.
-    """
-
-    filename_fmt: ClassVar[str] = (
-        "workflow={name}/release={release:%Y%m%d%H%M%S}"
-    )
-
-    def do_before(self) -> None:
-        """Create directory of release before saving log file."""
-        self.pointer().mkdir(parents=True, exist_ok=True)
-
-    @classmethod
-    def find_audits(
-        cls, name: str, *, extras: Optional[DictData] = None
-    ) -> Iterator[Self]:
-        """Generate the audit data that found from logs path with specific a
-        workflow name.
-
-        :param name: A workflow name that want to search release logging data.
-        :param extras: An extra parameter that want to override core config.
-
-        :rtype: Iterator[Self]
-        """
-        pointer: Path = (
-            dynamic("audit_path", extras=extras) / f"workflow={name}"
-        )
-        if not pointer.exists():
-            raise FileNotFoundError(f"Pointer: {pointer.absolute()}.")
-
-        for file in pointer.glob("./release=*/*.log"):
-            with file.open(mode="r", encoding="utf-8") as f:
-                yield cls.model_validate(obj=json.load(f))
-
-    @classmethod
-    def find_audit_with_release(
-        cls,
-        name: str,
-        release: Optional[datetime] = None,
-        *,
-        extras: Optional[DictData] = None,
-    ) -> Self:
-        """Return the audit data that found from logs path with specific
-        workflow name and release values. If a release does not pass to an input
-        argument, it will return the latest release from the current log path.
-
-        :param name: (str) A workflow name that want to search log.
-        :param release: (datetime) A release datetime that want to search log.
-        :param extras: An extra parameter that want to override core config.
-
-        :raise FileNotFoundError:
-        :raise NotImplementedError: If an input release does not pass to this
-            method. Because this method does not implement latest log.
-
-        :rtype: Self
-        """
-        if release is None:
-            raise NotImplementedError("Find latest log does not implement yet.")
-
-        pointer: Path = (
-            dynamic("audit_path", extras=extras)
-            / f"workflow={name}/release={release:%Y%m%d%H%M%S}"
-        )
-        if not pointer.exists():
-            raise FileNotFoundError(
-                f"Pointer: ./logs/workflow={name}/"
-                f"release={release:%Y%m%d%H%M%S} does not found."
-            )
-
-        latest_file: Path = max(pointer.glob("./*.log"), key=os.path.getctime)
-        with latest_file.open(mode="r", encoding="utf-8") as f:
-            return cls.model_validate(obj=json.load(f))
-
-    @classmethod
-    def is_pointed(
-        cls,
-        name: str,
-        release: datetime,
-        *,
-        extras: Optional[DictData] = None,
-    ) -> bool:
-        """Check the release log already pointed or created at the destination
-        log path.
-
-        :param name: (str) A workflow name.
-        :param release: (datetime) A release datetime.
-        :param extras: An extra parameter that want to override core config.
-
-        :rtype: bool
-        :return: Return False if the release log was not pointed or created.
-        """
-        # NOTE: Return False if enable writing log flag does not set.
-        if not dynamic("enable_write_audit", extras=extras):
-            return False
-
-        # NOTE: create pointer path that use the same logic of pointer method.
-        pointer: Path = dynamic(
-            "audit_path", extras=extras
-        ) / cls.filename_fmt.format(name=name, release=release)
-
-        return pointer.exists()
-
-    def pointer(self) -> Path:
-        """Return release directory path that was generated from model data.
-
-        :rtype: Path
-        """
-        return dynamic(
-            "audit_path", extras=self.extras
-        ) / self.filename_fmt.format(name=self.name, release=self.release)
-
-    def save(self, excluded: Optional[list[str]] = None) -> Self:
-        """Save logging data that receive a context data from a workflow
-        execution result.
-
-        :param excluded: An excluded list of key name that want to pass in the
-            model_dump method.
-
-        :rtype: Self
-        """
-        trace: TraceModel = get_trace(
-            self.run_id,
-            parent_run_id=self.parent_run_id,
-            extras=self.extras,
-        )
-
-        # NOTE: Check environ variable was set for real writing.
-        if not dynamic("enable_write_audit", extras=self.extras):
-            trace.debug("[AUDIT]: Skip writing log cause config was set")
-            return self
-
-        log_file: Path = (
-            self.pointer() / f"{self.parent_run_id or self.run_id}.log"
-        )
-        log_file.write_text(
-            json.dumps(
-                self.model_dump(exclude=excluded),
-                default=str,
-                indent=2,
-            ),
-            encoding="utf-8",
-        )
-        return self
-
-
-class SQLiteAudit(BaseAudit):  # pragma: no cov
-    """SQLite Audit Pydantic Model."""
-
-    table_name: ClassVar[str] = "audits"
-    schemas: ClassVar[
-        str
-    ] = """
-        workflow        str,
-        release         int,
-        type            str,
-        context         json,
-        parent_run_id   int,
-        run_id          int,
-        update          datetime
-        primary key ( run_id )
-        """
-
-    @classmethod
-    def is_pointed(
-        cls,
-        name: str,
-        release: datetime,
-        *,
-        extras: Optional[DictData] = None,
-    ) -> bool: ...
-
-    @classmethod
-    def find_audits(
-        cls, name: str, *, extras: Optional[DictData] = None
-    ) -> Iterator[Self]: ...
-
-    @classmethod
-    def find_audit_with_release(
-        cls,
-        name: str,
-        release: Optional[datetime] = None,
-        *,
-        extras: Optional[DictData] = None,
-    ) -> Self: ...
-
-    def save(self, excluded: Optional[list[str]]) -> SQLiteAudit:
-        """Save logging data that receive a context data from a workflow
-        execution result.
-        """
-        trace: TraceModel = get_trace(
-            self.run_id,
-            parent_run_id=self.parent_run_id,
-            extras=self.extras,
-        )
-
-        # NOTE: Check environ variable was set for real writing.
-        if not dynamic("enable_write_audit", extras=self.extras):
-            trace.debug("[AUDIT]: Skip writing log cause config was set")
-            return self
-
-        raise NotImplementedError("SQLiteAudit does not implement yet.")
-
-
-Audit = TypeVar("Audit", bound=BaseAudit)
-AuditModel = Union[
-    FileAudit,
-    SQLiteAudit,
-]
-
-
-def get_audit(
-    extras: Optional[DictData] = None,
-) -> type[AuditModel]:  # pragma: no cov
-    """Get an audit class that dynamic base on the config audit path value.
-
-    :param extras: An extra parameter that want to override the core config.
-
-    :rtype: type[Audit]
-    """
-    if dynamic("audit_path", extras=extras).is_file():
-        return SQLiteAudit
-    return FileAudit