ddeutil-workflow 0.0.73__py3-none-any.whl → 0.0.74__py3-none-any.whl

ddeutil/workflow/traces.py

@@ -3,7 +3,36 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
-# [x] Use fix config for `set_logging`, and Model initialize step.
+"""Tracing and Logging Module for Workflow Execution.
+
+This module provides comprehensive tracing and logging capabilities for workflow
+execution monitoring. It supports multiple trace backends including console output,
+file-based logging, and SQLite database storage.
+
+The tracing system captures detailed execution metadata including process IDs,
+thread identifiers, timestamps, and contextual information for debugging and
+monitoring workflow executions.
+
+Classes:
+    Message: Log message model with prefix parsing
+    TraceMeta: Metadata model for execution context
+    TraceData: Container for trace information
+    BaseTrace: Abstract base class for trace implementations
+    ConsoleTrace: Console-based trace output
+    FileTrace: File-based trace storage
+    SQLiteTrace: Database-based trace storage
+
+Functions:
+    set_logging: Configure logger with custom formatting
+    get_trace: Factory function for trace instances
+
+Example:
+    >>> from ddeutil.workflow.traces import get_trace
+    >>> # Create file-based trace
+    >>> trace = get_trace("running-id-101", parent_run_id="workflow-001")
+    >>> trace.info("Workflow execution started")
+    >>> trace.debug("Processing stage 1")
+"""
 from __future__ import annotations
 
 import json
@@ -17,9 +46,12 @@ from inspect import Traceback, currentframe, getframeinfo
 from pathlib import Path
 from threading import get_ident
 from types import FrameType
-from typing import ClassVar, Final, Literal, Optional, TypeVar, Union
+from typing import ClassVar, Final, Literal, Optional, Union
+from urllib.parse import ParseResult, unquote_plus, urlparse
 
 from pydantic import BaseModel, ConfigDict, Field
+from pydantic.functional_serializers import field_serializer
+from pydantic.functional_validators import field_validator
 from typing_extensions import Self
 
 from .__types import DictData
@@ -32,12 +64,23 @@ logger = logging.getLogger("ddeutil.workflow")
 
 @lru_cache
 def set_logging(name: str) -> logging.Logger:
-    """Return logger object with an input module name that already implement the
-    custom handler and formatter from this package config.
+    """Configure logger with custom formatting and handlers.
 
-    :param name: (str) A module name that want to log.
+    Creates and configures a logger instance with the custom formatter and
+    handlers defined in the package configuration. The logger includes both
+    console output and proper formatting for workflow execution tracking.
 
-    :rtype: logging.Logger
+    Args:
+        name: Module name to create logger for
+
+    Returns:
+        logging.Logger: Configured logger instance with custom formatting
+
+    Example:
+        ```python
+        logger = set_logging("ddeutil.workflow.stages")
+        logger.info("Stage execution started")
+        ```
     """
     _logger = logging.getLogger(name)
 
@@ -101,10 +144,12 @@ class Message(BaseModel):
     def prepare(self, extras: Optional[DictData] = None) -> str:
         """Prepare message with force add prefix before writing trace log.
 
-        :param extras: (DictData) An extra parameter that want to get the
-            `log_add_emoji` flag.
+        Args:
+            extras: An extra parameter that want to get the
+                `log_add_emoji` flag.
 
-        :rtype: str
+        Returns:
+            str: The prepared message with prefix and optional emoji.
         """
         name: str = self.name or PREFIX_DEFAULT
         emoji: str = (
@@ -139,9 +184,13 @@ class TraceMeta(BaseModel):  # pragma: no cov
         """Dynamic Frame information base on the `logs_trace_frame_layer` config
         value that was set from the extra parameter.
 
-        :param frame: (FrameType) The current frame that want to dynamic.
-        :param extras: (DictData) An extra parameter that want to get the
-            `logs_trace_frame_layer` config value.
+        Args:
+            frame: The current frame that want to dynamic.
+            extras: An extra parameter that want to get the
+                `logs_trace_frame_layer` config value.
+
+        Returns:
+            Traceback: The frame information at the specified layer.
         """
         extras: DictData = extras or {}
         layer: int = extras.get("logs_trace_frame_layer", 4)
@@ -167,14 +216,16 @@ class TraceMeta(BaseModel):  # pragma: no cov
         """Make the current metric for contract this TraceMeta model instance
         that will catch local states like PID, thread identity.
 
-        :param mode: (Literal["stdout", "stderr"]) A metadata mode.
-        :param message: (str) A message.
-        :param level: (str) A log level.
-        :param cutting_id: (str)
-        :param extras: (DictData) An extra parameter that want to override core
-            config values.
+        Args:
+            mode: A metadata mode.
+            message: A message.
+            level: A log level.
+            cutting_id: A cutting ID string.
+            extras: An extra parameter that want to override core
+                config values.
 
-        :rtype: Self
+        Returns:
+            Self: The constructed TraceMeta instance.
         """
         frame: FrameType = currentframe()
         frame_info: Traceback = cls.dynamic_frame(frame, extras=extras)
@@ -252,43 +303,6 @@ class BaseTrace(BaseModel, ABC):  # pragma: no cov
         description="A parent running ID",
     )
 
-    @classmethod
-    @abstractmethod
-    def find_traces(
-        cls,
-        path: Optional[Path] = None,
-        extras: Optional[DictData] = None,
-    ) -> Iterator[TraceData]:  # pragma: no cov
-        """Return iterator of TraceData models from the target pointer.
-
-        Args:
-            path (:obj:`Path`, optional): A pointer path that want to override.
-            extras (:obj:`DictData`, optional): An extras parameter that want to
-                override default engine config.
-
-        Returns:
-            Iterator[TracData]: An iterator object that generate a TracData
-                model.
-        """
-        raise NotImplementedError(
-            "Trace dataclass should implement `find_traces` class-method."
-        )
-
-    @classmethod
-    @abstractmethod
-    def find_trace_with_id(
-        cls,
-        run_id: str,
-        force_raise: bool = True,
-        *,
-        path: Optional[Path] = None,
-        extras: Optional[DictData] = None,
-    ) -> TraceData:
-        raise NotImplementedError(
-            "Trace dataclass should implement `find_trace_with_id` "
-            "class-method."
-        )
-
     @abstractmethod
     def writer(
         self,
@@ -340,7 +354,7 @@ class BaseTrace(BaseModel, ABC):  # pragma: no cov
         )
 
     @abstractmethod
-    def _logging(
+    def emit(
         self,
         message: str,
         mode: str,
@@ -364,7 +378,7 @@ class BaseTrace(BaseModel, ABC):  # pragma: no cov
 
         :param message: (str) A message that want to log.
         """
-        self._logging(message, mode="debug")
+        self.emit(message, mode="debug")
 
     def info(self, message: str) -> None:
         """Write trace log with append mode and logging this message with the
@@ -372,7 +386,7 @@ class BaseTrace(BaseModel, ABC):  # pragma: no cov
 
         :param message: (str) A message that want to log.
         """
-        self._logging(message, mode="info")
+        self.emit(message, mode="info")
 
     def warning(self, message: str) -> None:
         """Write trace log with append mode and logging this message with the
@@ -380,7 +394,7 @@ class BaseTrace(BaseModel, ABC):  # pragma: no cov
 
         :param message: (str) A message that want to log.
         """
-        self._logging(message, mode="warning")
+        self.emit(message, mode="warning")
 
     def error(self, message: str) -> None:
         """Write trace log with append mode and logging this message with the
@@ -388,7 +402,7 @@ class BaseTrace(BaseModel, ABC):  # pragma: no cov
 
         :param message: (str) A message that want to log.
         """
-        self._logging(message, mode="error", is_err=True)
+        self.emit(message, mode="error", is_err=True)
 
     def exception(self, message: str) -> None:
         """Write trace log with append mode and logging this message with the
@@ -396,10 +410,10 @@ class BaseTrace(BaseModel, ABC):  # pragma: no cov
 
         :param message: (str) A message that want to log.
         """
-        self._logging(message, mode="exception", is_err=True)
+        self.emit(message, mode="exception", is_err=True)
 
     @abstractmethod
-    async def _alogging(
+    async def amit(
         self,
         message: str,
         mode: str,
@@ -423,7 +437,7 @@ class BaseTrace(BaseModel, ABC):  # pragma: no cov
 
         :param message: (str) A message that want to log.
         """
-        await self._alogging(message, mode="debug")
+        await self.amit(message, mode="debug")
 
     async def ainfo(self, message: str) -> None:  # pragma: no cov
         """Async write trace log with append mode and logging this message with
@@ -431,7 +445,7 @@ class BaseTrace(BaseModel, ABC):  # pragma: no cov
 
         :param message: (str) A message that want to log.
         """
-        await self._alogging(message, mode="info")
+        await self.amit(message, mode="info")
 
     async def awarning(self, message: str) -> None:  # pragma: no cov
         """Async write trace log with append mode and logging this message with
@@ -439,7 +453,7 @@ class BaseTrace(BaseModel, ABC):  # pragma: no cov
 
         :param message: (str) A message that want to log.
         """
-        await self._alogging(message, mode="warning")
+        await self.amit(message, mode="warning")
 
     async def aerror(self, message: str) -> None:  # pragma: no cov
         """Async write trace log with append mode and logging this message with
@@ -447,7 +461,7 @@ class BaseTrace(BaseModel, ABC):  # pragma: no cov
 
         :param message: (str) A message that want to log.
         """
-        await self._alogging(message, mode="error", is_err=True)
+        await self.amit(message, mode="error", is_err=True)
 
     async def aexception(self, message: str) -> None:  # pragma: no cov
         """Async write trace log with append mode and logging this message with
@@ -455,36 +469,12 @@ class BaseTrace(BaseModel, ABC):  # pragma: no cov
 
         :param message: (str) A message that want to log.
         """
-        await self._alogging(message, mode="exception", is_err=True)
+        await self.amit(message, mode="exception", is_err=True)
 
 
 class ConsoleTrace(BaseTrace):  # pragma: no cov
     """Console Trace log model."""
 
-    @classmethod
-    def find_traces(
-        cls,
-        path: Optional[Path] = None,
-        extras: Optional[DictData] = None,
-    ) -> Iterator[TraceData]:  # pragma: no cov
-        raise NotImplementedError(
-            "Console Trace does not support to find history traces data."
-        )
-
-    @classmethod
-    def find_trace_with_id(
-        cls,
-        run_id: str,
-        force_raise: bool = True,
-        *,
-        path: Optional[Path] = None,
-        extras: Optional[DictData] = None,
-    ) -> TraceData:
-        raise NotImplementedError(
-            "Console Trace does not support to find history traces data with "
-            "the specific running ID."
-        )
-
     def writer(
         self,
         message: str,
@@ -537,13 +527,13 @@ class ConsoleTrace(BaseTrace):  # pragma: no cov
         """
         return prepare_newline(Message.from_str(message).prepare(self.extras))
 
-    def _logging(
-        self, message: str, mode: str, *, is_err: bool = False
-    ) -> None:
+    def emit(self, message: str, mode: str, *, is_err: bool = False) -> None:
         """Write trace log with append mode and logging this message with any
         logging level.
 
         :param message: (str) A message that want to log.
+        :param mode: (str)
+        :param is_err: (bool)
         """
         msg: str = self.make_message(message)
 
@@ -554,13 +544,15 @@ class ConsoleTrace(BaseTrace):  # pragma: no cov
 
         getattr(logger, mode)(msg, stacklevel=3, extra={"cut_id": self.cut_id})
 
-    async def _alogging(
+    async def amit(
         self, message: str, mode: str, *, is_err: bool = False
     ) -> None:
         """Write trace log with append mode and logging this message with any
         logging level.
 
         :param message: (str) A message that want to log.
+        :param mode: (str)
+        :param is_err: (bool)
         """
         msg: str = self.make_message(message)
 
@@ -572,7 +564,62 @@ class ConsoleTrace(BaseTrace):  # pragma: no cov
         getattr(logger, mode)(msg, stacklevel=3, extra={"cut_id": self.cut_id})
 
 
-class FileTrace(ConsoleTrace):  # pragma: no cov
+class OutsideTrace(ConsoleTrace, ABC):
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    url: ParseResult = Field(description="An URL for create pointer.")
+
+    @field_validator(
+        "url", mode="before", json_schema_input_type=Union[ParseResult, str]
+    )
+    def __parse_url(cls, value: Union[ParseResult, str]) -> ParseResult:
+        if isinstance(value, str):
+            return urlparse(value)
+        return value
+
+    @field_serializer("url")
+    def __serialize_url(self, value: ParseResult) -> str:
+        return value.geturl()
+
+    @classmethod
+    @abstractmethod
+    def find_traces(
+        cls,
+        path: Optional[Path] = None,
+        extras: Optional[DictData] = None,
+    ) -> Iterator[TraceData]:  # pragma: no cov
+        """Return iterator of TraceData models from the target pointer.
+
+        Args:
+            path (:obj:`Path`, optional): A pointer path that want to override.
+            extras (:obj:`DictData`, optional): An extras parameter that want to
+                override default engine config.
+
+        Returns:
+            Iterator[TracData]: An iterator object that generate a TracData
+                model.
+        """
+        raise NotImplementedError(
+            "Trace dataclass should implement `find_traces` class-method."
+        )
+
+    @classmethod
+    @abstractmethod
+    def find_trace_with_id(
+        cls,
+        run_id: str,
+        force_raise: bool = True,
+        *,
+        path: Optional[Path] = None,
+        extras: Optional[DictData] = None,
+    ) -> TraceData:
+        raise NotImplementedError(
+            "Trace dataclass should implement `find_trace_with_id` "
+            "class-method."
+        )
+
+
+class FileTrace(OutsideTrace):  # pragma: no cov
     """File Trace dataclass that write file to the local storage."""
 
     @classmethod
@@ -587,7 +634,9 @@ class FileTrace(ConsoleTrace):  # pragma: no cov
         :param extras: An extra parameter that want to override core config.
         """
         for file in sorted(
-            (path or dynamic("trace_path", extras=extras)).glob("./run_id=*"),
+            (path or Path(dynamic("trace_url", extras=extras).path)).glob(
+                "./run_id=*"
+            ),
             key=lambda f: f.lstat().st_mtime,
         ):
             yield TraceData.from_path(file)
@@ -608,7 +657,7 @@ class FileTrace(ConsoleTrace):  # pragma: no cov
         :param path: (Path)
         :param extras: An extra parameter that want to override core config.
         """
-        base_path: Path = path or dynamic("trace_path", extras=extras)
+        base_path: Path = path or Path(dynamic("trace_url", extras=extras).path)
         file: Path = base_path / f"run_id={run_id}"
         if file.exists():
             return TraceData.from_path(file)
@@ -624,10 +673,14 @@ class FileTrace(ConsoleTrace):  # pragma: no cov
         """Pointer of the target path that use to writing trace log or searching
         trace log.
 
+            This running ID folder that use to keeping trace log data will use
+        a parent running ID first. If it does not set, it will use running ID
+        instead.
+
         :rtype: Path
         """
         log_file: Path = (
-            dynamic("trace_path", extras=self.extras)
+            Path(unquote_plus(self.url.path))
             / f"run_id={self.parent_run_id or self.run_id}"
         )
         if not log_file.exists():
@@ -710,17 +763,20 @@ class FileTrace(ConsoleTrace):  # pragma: no cov
             await f.write(trace_meta.model_dump_json() + "\n")
 
 
-class SQLiteTrace(ConsoleTrace):  # pragma: no cov
+class SQLiteTrace(OutsideTrace):  # pragma: no cov
     """SQLite Trace dataclass that write trace log to the SQLite database file."""
 
     table_name: ClassVar[str] = "audits"
     schemas: ClassVar[
         str
     ] = """
-        run_id          int,
-        stdout          str,
-        stderr          str,
-        update          datetime
+        run_id              str
+        , parent_run_id     str
+        , type              str
+        , text              str
+        , metadata          JSON
+        , created_at        datetime
+        , updated_at        datetime
         primary key ( run_id )
         """
 
@@ -729,7 +785,8 @@ class SQLiteTrace(ConsoleTrace):  # pragma: no cov
         cls,
         path: Optional[Path] = None,
         extras: Optional[DictData] = None,
-    ) -> Iterator[TraceData]: ...
+    ) -> Iterator[TraceData]:
+        raise NotImplementedError("SQLiteTrace does not implement yet.")
 
     @classmethod
     def find_trace_with_id(
@@ -739,30 +796,33 @@ class SQLiteTrace(ConsoleTrace):  # pragma: no cov
         *,
         path: Optional[Path] = None,
         extras: Optional[DictData] = None,
-    ) -> TraceData: ...
+    ) -> TraceData:
+        raise NotImplementedError("SQLiteTrace does not implement yet.")
 
-    def make_message(self, message: str) -> str: ...
+    def make_message(self, message: str) -> str:
+        raise NotImplementedError("SQLiteTrace does not implement yet.")
 
     def writer(
         self,
         message: str,
         level: str,
         is_err: bool = False,
-    ) -> None: ...
+    ) -> None:
+        raise NotImplementedError("SQLiteTrace does not implement yet.")
 
     def awriter(
         self,
         message: str,
         level: str,
         is_err: bool = False,
-    ) -> None: ...
+    ) -> None:
+        raise NotImplementedError("SQLiteTrace does not implement yet.")
 
 
-Trace = TypeVar("Trace", bound=BaseTrace)
-TraceModel = Union[
-    ConsoleTrace,
+Trace = Union[
     FileTrace,
     SQLiteTrace,
+    OutsideTrace,
 ]
 
 
@@ -771,7 +831,7 @@ def get_trace(
     *,
     parent_run_id: Optional[str] = None,
     extras: Optional[DictData] = None,
-) -> TraceModel:  # pragma: no cov
+) -> Trace:  # pragma: no cov
     """Get dynamic Trace instance from the core config (it can override by an
     extras argument) that passing running ID and parent running ID.
 
@@ -780,12 +840,31 @@ def get_trace(
     :param extras: (DictData) An extra parameter that want to override the core
         config values.
 
-    :rtype: TraceLog
+    :rtype: Trace
     """
-    if dynamic("trace_path", extras=extras).is_file():
-        return SQLiteTrace(
-            run_id=run_id, parent_run_id=parent_run_id, extras=(extras or {})
+    # NOTE: Allow you to override trace model by the extra parameter.
+    map_trace_models: dict[str, type[Trace]] = extras.get(
+        "trace_model_mapping", {}
+    )
+    url: ParseResult
+    if (url := dynamic("trace_url", extras=extras)).scheme and (
+        url.scheme == "sqlite"
+        or (url.scheme == "file" and Path(url.path).is_file())
+    ):
+        return map_trace_models.get("sqlite", SQLiteTrace)(
+            url=url,
+            run_id=run_id,
+            parent_run_id=parent_run_id,
+            extras=(extras or {}),
+        )
+    elif url.scheme and url.scheme != "file":
+        raise NotImplementedError(
+            f"Does not implement the outside trace model support for URL: {url}"
         )
-    return FileTrace(
-        run_id=run_id, parent_run_id=parent_run_id, extras=(extras or {})
+
+    return map_trace_models.get("file", FileTrace)(
+        url=url,
+        run_id=run_id,
+        parent_run_id=parent_run_id,
+        extras=(extras or {}),
     )

ddeutil/workflow/utils.py

@@ -3,7 +3,35 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
-"""Utility function model."""
+"""Utility Functions for Workflow Operations.
+
+This module provides essential utility functions used throughout the workflow
+system for ID generation, datetime handling, string processing, template
+operations, and other common tasks.
+
+Functions:
+    gen_id: Generate unique identifiers for workflow components
+    make_exec: Create executable strings for shell commands
+    filter_func: Filter functions based on criteria
+    dump_all: Serialize data to various formats
+    delay: Create delays in execution
+    to_train: Convert strings to train-case format
+    get_dt_now: Get current datetime with timezone
+    get_d_now: Get current date
+    cross_product: Generate cross product of matrix values
+    replace_sec: Replace template variables in strings
+
+Example:
+    ```python
+    from ddeutil.workflow.utils import gen_id, get_dt_now
+
+    # Generate unique ID
+    run_id = gen_id("workflow")
+
+    # Get current datetime
+    now = get_dt_now()
+    ```
+"""
 from __future__ import annotations
 
 import stat
@@ -27,13 +55,19 @@ T = TypeVar("T")
 UTC: Final[ZoneInfo] = ZoneInfo("UTC")
 MARK_NEWLINE: Final[str] = "||"
 
+# Cache for random delay values to avoid repeated randrange calls
+_CACHED_DELAYS = [randrange(0, 99, step=10) / 100 for _ in range(100)]
+_DELAY_INDEX = 0
+
 
 def to_train(camel: str) -> str:
     """Convert camel case string to train case.
 
-    :param camel: (str) A camel case string that want to convert.
+    Args:
+        camel: A camel case string that want to convert.
 
-    :rtype: str
+    Returns:
+        str: The converted train-case string.
     """
     return "".join("-" + i if i.isupper() else i for i in camel).lstrip("-")
 
@@ -41,9 +75,11 @@ def to_train(camel: str) -> str:
 def prepare_newline(msg: str) -> str:
     """Prepare message that has multiple newline char.
 
-    :param msg: (str) A message that want to prepare.
+    Args:
+        msg: A message that want to prepare.
 
-    :rtype: str
+    Returns:
+        str: The prepared message with formatted newlines.
     """
     # NOTE: Remove ending with "\n" and replace "\n" with the "||" value.
     msg: str = msg.strip("\n").replace("\n", MARK_NEWLINE)
@@ -63,9 +99,11 @@ def prepare_newline(msg: str) -> str:
 def replace_sec(dt: datetime) -> datetime:
     """Replace second and microsecond values to 0.
 
-    :param dt: A datetime object that want to replace.
+    Args:
+        dt: A datetime object that want to replace.
 
-    :rtype: datetime
+    Returns:
+        datetime: The datetime with seconds and microseconds set to 0.
     """
     return dt.replace(second=0, microsecond=0)
 
@@ -87,6 +125,17 @@ def get_dt_now(tz: Optional[ZoneInfo] = None, offset: float = 0.0) -> datetime:
     return datetime.now(tz=tz) - timedelta(seconds=offset)
 
 
+def get_dt_ntz_now() -> datetime:  # pragma: no cov
+    """Get current datetime with no timezone.
+
+    Returns the current datetime object using the None timezone.
+
+    Returns:
+        datetime: Current datetime with no timezone
+    """
+    return get_dt_now(tz=None)
+
+
 def get_d_now(
     tz: Optional[ZoneInfo] = None, offset: float = 0.0
 ) -> date:  # pragma: no cov
@@ -151,7 +200,10 @@ def delay(second: float = 0) -> None:  # pragma: no cov
 
     :param second: (float) A second number that want to adds-on random value.
     """
-    time.sleep(second + randrange(0, 99, step=10) / 100)
+    global _DELAY_INDEX
+    cached_random = _CACHED_DELAYS[_DELAY_INDEX % len(_CACHED_DELAYS)]
+    _DELAY_INDEX = (_DELAY_INDEX + 1) % len(_CACHED_DELAYS)
+    time.sleep(second + cached_random)
 
 
 def gen_id(