ddeutil-workflow 0.0.72__py3-none-any.whl → 0.0.74__py3-none-any.whl

ddeutil/workflow/traces.py

@@ -3,7 +3,36 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
-# [x] Use fix config for `set_logging`, and Model initialize step.
+"""Tracing and Logging Module for Workflow Execution.
+
+This module provides comprehensive tracing and logging capabilities for workflow
+execution monitoring. It supports multiple trace backends including console output,
+file-based logging, and SQLite database storage.
+
+The tracing system captures detailed execution metadata including process IDs,
+thread identifiers, timestamps, and contextual information for debugging and
+monitoring workflow executions.
+
+Classes:
+    Message: Log message model with prefix parsing
+    TraceMeta: Metadata model for execution context
+    TraceData: Container for trace information
+    BaseTrace: Abstract base class for trace implementations
+    ConsoleTrace: Console-based trace output
+    FileTrace: File-based trace storage
+    SQLiteTrace: Database-based trace storage
+
+Functions:
+    set_logging: Configure logger with custom formatting
+    get_trace: Factory function for trace instances
+
+Example:
+    >>> from ddeutil.workflow.traces import get_trace
+    >>> # Create file-based trace
+    >>> trace = get_trace("running-id-101", parent_run_id="workflow-001")
+    >>> trace.info("Workflow execution started")
+    >>> trace.debug("Processing stage 1")
+"""
 from __future__ import annotations
 
 import json
@@ -17,9 +46,12 @@ from inspect import Traceback, currentframe, getframeinfo
 from pathlib import Path
 from threading import get_ident
 from types import FrameType
-from typing import ClassVar, Final, Literal, Optional, TypeVar, Union
+from typing import ClassVar, Final, Literal, Optional, Union
+from urllib.parse import ParseResult, unquote_plus, urlparse
 
 from pydantic import BaseModel, ConfigDict, Field
+from pydantic.functional_serializers import field_serializer
+from pydantic.functional_validators import field_validator
 from typing_extensions import Self
 
 from .__types import DictData
@@ -32,12 +64,23 @@ logger = logging.getLogger("ddeutil.workflow")
 
 @lru_cache
 def set_logging(name: str) -> logging.Logger:
-    """Return logger object with an input module name that already implement the
-    custom handler and formatter from this package config.
+    """Configure logger with custom formatting and handlers.
 
-    :param name: (str) A module name that want to log.
+    Creates and configures a logger instance with the custom formatter and
+    handlers defined in the package configuration. The logger includes both
+    console output and proper formatting for workflow execution tracking.
 
-    :rtype: logging.Logger
+    Args:
+        name: Module name to create logger for
+
+    Returns:
+        logging.Logger: Configured logger instance with custom formatting
+
+    Example:
+        ```python
+        logger = set_logging("ddeutil.workflow.stages")
+        logger.info("Stage execution started")
+        ```
     """
     _logger = logging.getLogger(name)
 
@@ -76,7 +119,7 @@ PREFIX_LOGS_REGEX: re.Pattern[str] = re.compile(
 )  # pragma: no cov
 
 
-class PrefixMsg(BaseModel):
+class Message(BaseModel):
     """Prefix Message model for receive grouping dict from searching prefix data
     from logging message.
     """
@@ -92,19 +135,21 @@ class PrefixMsg(BaseModel):
             msg (str): A message that want to extract.
 
         Returns:
-            PrefixMsg: the validated model from a string message.
+            Message: the validated model from a string message.
         """
-        return PrefixMsg.model_validate(
+        return Message.model_validate(
             obj=PREFIX_LOGS_REGEX.search(msg).groupdict()
         )
 
     def prepare(self, extras: Optional[DictData] = None) -> str:
         """Prepare message with force add prefix before writing trace log.
 
-        :param extras: (DictData) An extra parameter that want to get the
-            `log_add_emoji` flag.
+        Args:
+            extras: An extra parameter that want to get the
+                `log_add_emoji` flag.
 
-        :rtype: str
+        Returns:
+            str: The prepared message with prefix and optional emoji.
         """
         name: str = self.name or PREFIX_DEFAULT
         emoji: str = (
@@ -126,6 +171,9 @@ class TraceMeta(BaseModel):  # pragma: no cov
     process: int = Field(description="A process ID.")
     thread: int = Field(description="A thread ID.")
     message: str = Field(description="A message log.")
+    cut_id: Optional[str] = Field(
+        default=None, description="A cutting of running ID."
+    )
     filename: str = Field(description="A filename of this log.")
     lineno: int = Field(description="A line number of this log.")
 
@@ -136,9 +184,13 @@ class TraceMeta(BaseModel):  # pragma: no cov
         """Dynamic Frame information base on the `logs_trace_frame_layer` config
         value that was set from the extra parameter.
 
-        :param frame: (FrameType) The current frame that want to dynamic.
-        :param extras: (DictData) An extra parameter that want to get the
-            `logs_trace_frame_layer` config value.
+        Args:
+            frame: The current frame that want to dynamic.
+            extras: An extra parameter that want to get the
+                `logs_trace_frame_layer` config value.
+
+        Returns:
+            Traceback: The frame information at the specified layer.
         """
         extras: DictData = extras or {}
         layer: int = extras.get("logs_trace_frame_layer", 4)
@@ -157,19 +209,23 @@ class TraceMeta(BaseModel):  # pragma: no cov
         mode: Literal["stdout", "stderr"],
         message: str,
         level: str,
+        cutting_id: str,
         *,
         extras: Optional[DictData] = None,
     ) -> Self:
         """Make the current metric for contract this TraceMeta model instance
         that will catch local states like PID, thread identity.
 
-        :param mode: (Literal["stdout", "stderr"]) A metadata mode.
-        :param message: (str) A message.
-        :param level: (str) A log level.
-        :param extras: (DictData) An extra parameter that want to override core
-            config values.
+        Args:
+            mode: A metadata mode.
+            message: A message.
+            level: A log level.
+            cutting_id: A cutting ID string.
+            extras: An extra parameter that want to override core
+                config values.
 
-        :rtype: Self
+        Returns:
+            Self: The constructed TraceMeta instance.
         """
         frame: FrameType = currentframe()
         frame_info: Traceback = cls.dynamic_frame(frame, extras=extras)
@@ -185,6 +241,7 @@ class TraceMeta(BaseModel):  # pragma: no cov
             process=os.getpid(),
             thread=get_ident(),
             message=message,
+            cut_id=cutting_id,
             filename=frame_info.filename.split(os.path.sep)[-1],
             lineno=frame_info.lineno,
         )
@@ -246,43 +303,6 @@ class BaseTrace(BaseModel, ABC):  # pragma: no cov
         description="A parent running ID",
     )
 
-    @classmethod
-    @abstractmethod
-    def find_traces(
-        cls,
-        path: Optional[Path] = None,
-        extras: Optional[DictData] = None,
-    ) -> Iterator[TraceData]:  # pragma: no cov
-        """Return iterator of TraceData models from the target pointer.
-
-        Args:
-            path (:obj:`Path`, optional): A pointer path that want to override.
-            extras (:obj:`DictData`, optional): An extras parameter that want to
-                override default engine config.
-
-        Returns:
-            Iterator[TracData]: An iterator object that generate a TracData
-                model.
-        """
-        raise NotImplementedError(
-            "Trace dataclass should implement `find_traces` class-method."
-        )
-
-    @classmethod
-    @abstractmethod
-    def find_trace_with_id(
-        cls,
-        run_id: str,
-        force_raise: bool = True,
-        *,
-        path: Optional[Path] = None,
-        extras: Optional[DictData] = None,
-    ) -> TraceData:
-        raise NotImplementedError(
-            "Trace dataclass should implement `find_trace_with_id` "
-            "class-method."
-        )
-
     @abstractmethod
     def writer(
         self,
@@ -334,7 +354,7 @@ class BaseTrace(BaseModel, ABC):  # pragma: no cov
         )
 
     @abstractmethod
-    def _logging(
+    def emit(
         self,
         message: str,
         mode: str,
@@ -358,7 +378,7 @@ class BaseTrace(BaseModel, ABC):  # pragma: no cov
 
         :param message: (str) A message that want to log.
         """
-        self._logging(message, mode="debug")
+        self.emit(message, mode="debug")
 
     def info(self, message: str) -> None:
         """Write trace log with append mode and logging this message with the
@@ -366,7 +386,7 @@ class BaseTrace(BaseModel, ABC):  # pragma: no cov
 
         :param message: (str) A message that want to log.
         """
-        self._logging(message, mode="info")
+        self.emit(message, mode="info")
 
     def warning(self, message: str) -> None:
         """Write trace log with append mode and logging this message with the
@@ -374,7 +394,7 @@ class BaseTrace(BaseModel, ABC):  # pragma: no cov
 
         :param message: (str) A message that want to log.
         """
-        self._logging(message, mode="warning")
+        self.emit(message, mode="warning")
 
     def error(self, message: str) -> None:
         """Write trace log with append mode and logging this message with the
@@ -382,7 +402,7 @@ class BaseTrace(BaseModel, ABC):  # pragma: no cov
 
         :param message: (str) A message that want to log.
         """
-        self._logging(message, mode="error", is_err=True)
+        self.emit(message, mode="error", is_err=True)
 
     def exception(self, message: str) -> None:
         """Write trace log with append mode and logging this message with the
@@ -390,10 +410,10 @@ class BaseTrace(BaseModel, ABC):  # pragma: no cov
 
         :param message: (str) A message that want to log.
         """
-        self._logging(message, mode="exception", is_err=True)
+        self.emit(message, mode="exception", is_err=True)
 
     @abstractmethod
-    async def _alogging(
+    async def amit(
         self,
         message: str,
         mode: str,
@@ -417,7 +437,7 @@ class BaseTrace(BaseModel, ABC):  # pragma: no cov
 
         :param message: (str) A message that want to log.
         """
-        await self._alogging(message, mode="debug")
+        await self.amit(message, mode="debug")
 
     async def ainfo(self, message: str) -> None:  # pragma: no cov
         """Async write trace log with append mode and logging this message with
@@ -425,7 +445,7 @@ class BaseTrace(BaseModel, ABC):  # pragma: no cov
 
         :param message: (str) A message that want to log.
         """
-        await self._alogging(message, mode="info")
+        await self.amit(message, mode="info")
 
     async def awarning(self, message: str) -> None:  # pragma: no cov
         """Async write trace log with append mode and logging this message with
@@ -433,7 +453,7 @@ class BaseTrace(BaseModel, ABC):  # pragma: no cov
 
         :param message: (str) A message that want to log.
         """
-        await self._alogging(message, mode="warning")
+        await self.amit(message, mode="warning")
 
     async def aerror(self, message: str) -> None:  # pragma: no cov
         """Async write trace log with append mode and logging this message with
@@ -441,7 +461,7 @@ class BaseTrace(BaseModel, ABC):  # pragma: no cov
 
         :param message: (str) A message that want to log.
         """
-        await self._alogging(message, mode="error", is_err=True)
+        await self.amit(message, mode="error", is_err=True)
 
     async def aexception(self, message: str) -> None:  # pragma: no cov
         """Async write trace log with append mode and logging this message with
@@ -449,36 +469,12 @@ class BaseTrace(BaseModel, ABC):  # pragma: no cov
 
         :param message: (str) A message that want to log.
         """
-        await self._alogging(message, mode="exception", is_err=True)
+        await self.amit(message, mode="exception", is_err=True)
 
 
 class ConsoleTrace(BaseTrace):  # pragma: no cov
     """Console Trace log model."""
 
-    @classmethod
-    def find_traces(
-        cls,
-        path: Optional[Path] = None,
-        extras: Optional[DictData] = None,
-    ) -> Iterator[TraceData]:  # pragma: no cov
-        raise NotImplementedError(
-            "Console Trace does not support to find history traces data."
-        )
-
-    @classmethod
-    def find_trace_with_id(
-        cls,
-        run_id: str,
-        force_raise: bool = True,
-        *,
-        path: Optional[Path] = None,
-        extras: Optional[DictData] = None,
-    ) -> TraceData:
-        raise NotImplementedError(
-            "Console Trace does not support to find history traces data with "
-            "the specific running ID."
-        )
-
     def writer(
         self,
         message: str,
@@ -529,18 +525,15 @@ class ConsoleTrace(BaseTrace):  # pragma: no cov
 
         :rtype: str
         """
-        return prepare_newline(
-            f"({self.cut_id}) "
-            f"{PrefixMsg.from_str(message).prepare(self.extras)}"
-        )
+        return prepare_newline(Message.from_str(message).prepare(self.extras))
 
-    def _logging(
-        self, message: str, mode: str, *, is_err: bool = False
-    ) -> None:
+    def emit(self, message: str, mode: str, *, is_err: bool = False) -> None:
         """Write trace log with append mode and logging this message with any
         logging level.
 
         :param message: (str) A message that want to log.
+        :param mode: (str)
+        :param is_err: (bool)
         """
         msg: str = self.make_message(message)
 
@@ -551,13 +544,15 @@ class ConsoleTrace(BaseTrace):  # pragma: no cov
 
         getattr(logger, mode)(msg, stacklevel=3, extra={"cut_id": self.cut_id})
 
-    async def _alogging(
+    async def amit(
         self, message: str, mode: str, *, is_err: bool = False
     ) -> None:
         """Write trace log with append mode and logging this message with any
         logging level.
 
         :param message: (str) A message that want to log.
+        :param mode: (str)
+        :param is_err: (bool)
         """
         msg: str = self.make_message(message)
 
@@ -569,7 +564,62 @@ class ConsoleTrace(BaseTrace):  # pragma: no cov
         getattr(logger, mode)(msg, stacklevel=3, extra={"cut_id": self.cut_id})
 
 
-class FileTrace(ConsoleTrace):  # pragma: no cov
+class OutsideTrace(ConsoleTrace, ABC):
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    url: ParseResult = Field(description="An URL for create pointer.")
+
+    @field_validator(
+        "url", mode="before", json_schema_input_type=Union[ParseResult, str]
+    )
+    def __parse_url(cls, value: Union[ParseResult, str]) -> ParseResult:
+        if isinstance(value, str):
+            return urlparse(value)
+        return value
+
+    @field_serializer("url")
+    def __serialize_url(self, value: ParseResult) -> str:
+        return value.geturl()
+
+    @classmethod
+    @abstractmethod
+    def find_traces(
+        cls,
+        path: Optional[Path] = None,
+        extras: Optional[DictData] = None,
+    ) -> Iterator[TraceData]:  # pragma: no cov
+        """Return iterator of TraceData models from the target pointer.
+
+        Args:
+            path (:obj:`Path`, optional): A pointer path that want to override.
+            extras (:obj:`DictData`, optional): An extras parameter that want to
+                override default engine config.
+
+        Returns:
+            Iterator[TracData]: An iterator object that generate a TracData
+                model.
+        """
+        raise NotImplementedError(
+            "Trace dataclass should implement `find_traces` class-method."
+        )
+
+    @classmethod
+    @abstractmethod
+    def find_trace_with_id(
+        cls,
+        run_id: str,
+        force_raise: bool = True,
+        *,
+        path: Optional[Path] = None,
+        extras: Optional[DictData] = None,
+    ) -> TraceData:
+        raise NotImplementedError(
+            "Trace dataclass should implement `find_trace_with_id` "
+            "class-method."
+        )
+
+
+class FileTrace(OutsideTrace):  # pragma: no cov
     """File Trace dataclass that write file to the local storage."""
 
     @classmethod
@@ -584,7 +634,9 @@ class FileTrace(ConsoleTrace):  # pragma: no cov
         :param extras: An extra parameter that want to override core config.
         """
         for file in sorted(
-            (path or dynamic("trace_path", extras=extras)).glob("./run_id=*"),
+            (path or Path(dynamic("trace_url", extras=extras).path)).glob(
+                "./run_id=*"
+            ),
             key=lambda f: f.lstat().st_mtime,
         ):
             yield TraceData.from_path(file)
@@ -605,7 +657,7 @@ class FileTrace(ConsoleTrace):  # pragma: no cov
         :param path: (Path)
         :param extras: An extra parameter that want to override core config.
         """
-        base_path: Path = path or dynamic("trace_path", extras=extras)
+        base_path: Path = path or Path(dynamic("trace_url", extras=extras).path)
         file: Path = base_path / f"run_id={run_id}"
         if file.exists():
             return TraceData.from_path(file)
@@ -621,10 +673,14 @@ class FileTrace(ConsoleTrace):  # pragma: no cov
         """Pointer of the target path that use to writing trace log or searching
         trace log.
 
+            This running ID folder that use to keeping trace log data will use
+        a parent running ID first. If it does not set, it will use running ID
+        instead.
+
         :rtype: Path
         """
         log_file: Path = (
-            dynamic("trace_path", extras=self.extras)
+            Path(unquote_plus(self.url.path))
             / f"run_id={self.parent_run_id or self.run_id}"
         )
         if not log_file.exists():
@@ -655,7 +711,11 @@ class FileTrace(ConsoleTrace):  # pragma: no cov
 
         mode: Literal["stdout", "stderr"] = "stderr" if is_err else "stdout"
         trace_meta: TraceMeta = TraceMeta.make(
-            mode=mode, level=level, message=message, extras=self.extras
+            mode=mode,
+            level=level,
+            message=message,
+            cutting_id=self.cut_id,
+            extras=self.extras,
         )
 
         with (self.pointer / f"{mode}.txt").open(
@@ -684,7 +744,11 @@ class FileTrace(ConsoleTrace):  # pragma: no cov
 
         mode: Literal["stdout", "stderr"] = "stderr" if is_err else "stdout"
         trace_meta: TraceMeta = TraceMeta.make(
-            mode=mode, level=level, message=message, extras=self.extras
+            mode=mode,
+            level=level,
+            message=message,
+            cutting_id=self.cut_id,
+            extras=self.extras,
         )
 
         async with aiofiles.open(
@@ -699,17 +763,20 @@ class FileTrace(ConsoleTrace):  # pragma: no cov
             await f.write(trace_meta.model_dump_json() + "\n")
 
 
-class SQLiteTrace(ConsoleTrace):  # pragma: no cov
+class SQLiteTrace(OutsideTrace):  # pragma: no cov
     """SQLite Trace dataclass that write trace log to the SQLite database file."""
 
     table_name: ClassVar[str] = "audits"
     schemas: ClassVar[
         str
     ] = """
-        run_id          int,
-        stdout          str,
-        stderr          str,
-        update          datetime
+        run_id              str
+        , parent_run_id     str
+        , type              str
+        , text              str
+        , metadata          JSON
+        , created_at        datetime
+        , updated_at        datetime
         primary key ( run_id )
         """
 
@@ -718,7 +785,8 @@ class SQLiteTrace(ConsoleTrace):  # pragma: no cov
         cls,
         path: Optional[Path] = None,
         extras: Optional[DictData] = None,
-    ) -> Iterator[TraceData]: ...
+    ) -> Iterator[TraceData]:
+        raise NotImplementedError("SQLiteTrace does not implement yet.")
 
     @classmethod
     def find_trace_with_id(
@@ -728,30 +796,33 @@ class SQLiteTrace(ConsoleTrace):  # pragma: no cov
         *,
         path: Optional[Path] = None,
         extras: Optional[DictData] = None,
-    ) -> TraceData: ...
+    ) -> TraceData:
+        raise NotImplementedError("SQLiteTrace does not implement yet.")
 
-    def make_message(self, message: str) -> str: ...
+    def make_message(self, message: str) -> str:
+        raise NotImplementedError("SQLiteTrace does not implement yet.")
 
     def writer(
         self,
         message: str,
         level: str,
         is_err: bool = False,
-    ) -> None: ...
+    ) -> None:
+        raise NotImplementedError("SQLiteTrace does not implement yet.")
 
     def awriter(
         self,
         message: str,
         level: str,
         is_err: bool = False,
-    ) -> None: ...
+    ) -> None:
+        raise NotImplementedError("SQLiteTrace does not implement yet.")
 
 
-Trace = TypeVar("Trace", bound=BaseTrace)
-TraceModel = Union[
-    ConsoleTrace,
+Trace = Union[
     FileTrace,
     SQLiteTrace,
+    OutsideTrace,
 ]
 
 
@@ -760,7 +831,7 @@ def get_trace(
     *,
     parent_run_id: Optional[str] = None,
     extras: Optional[DictData] = None,
-) -> TraceModel:  # pragma: no cov
+) -> Trace:  # pragma: no cov
     """Get dynamic Trace instance from the core config (it can override by an
     extras argument) that passing running ID and parent running ID.
 
@@ -769,12 +840,31 @@ def get_trace(
     :param extras: (DictData) An extra parameter that want to override the core
         config values.
 
-    :rtype: TraceLog
+    :rtype: Trace
     """
-    if dynamic("trace_path", extras=extras).is_file():
-        return SQLiteTrace(
-            run_id=run_id, parent_run_id=parent_run_id, extras=(extras or {})
+    # NOTE: Allow you to override trace model by the extra parameter.
+    map_trace_models: dict[str, type[Trace]] = extras.get(
+        "trace_model_mapping", {}
+    )
+    url: ParseResult
+    if (url := dynamic("trace_url", extras=extras)).scheme and (
+        url.scheme == "sqlite"
+        or (url.scheme == "file" and Path(url.path).is_file())
+    ):
+        return map_trace_models.get("sqlite", SQLiteTrace)(
+            url=url,
+            run_id=run_id,
+            parent_run_id=parent_run_id,
+            extras=(extras or {}),
+        )
+    elif url.scheme and url.scheme != "file":
+        raise NotImplementedError(
+            f"Does not implement the outside trace model support for URL: {url}"
         )
-    return FileTrace(
-        run_id=run_id, parent_run_id=parent_run_id, extras=(extras or {})
+
+    return map_trace_models.get("file", FileTrace)(
+        url=url,
+        run_id=run_id,
+        parent_run_id=parent_run_id,
+        extras=(extras or {}),
     )