ddeutil-workflow 0.0.81__py3-none-any.whl → 0.0.83__py3-none-any.whl

ddeutil/workflow/traces.py

@@ -29,7 +29,16 @@ from inspect import Traceback, currentframe, getframeinfo
 from pathlib import Path
 from threading import Lock, get_ident
 from types import FrameType
-from typing import Annotated, Any, ClassVar, Final, Literal, Optional, Union
+from typing import (
+    Annotated,
+    Any,
+    ClassVar,
+    Final,
+    Literal,
+    Optional,
+    TypeVar,
+    Union,
+)
 from zoneinfo import ZoneInfo
 
 from pydantic import BaseModel, Field, PrivateAttr
@@ -42,6 +51,8 @@ from .utils import cut_id, get_dt_now, prepare_newline
 
 logger = logging.getLogger("ddeutil.workflow")
 Level = Literal["debug", "info", "warning", "error", "exception"]
+EMJ_ALERT: str = "🚨"
+EMJ_SKIP: str = "⏭️"
 
 
 @lru_cache
@@ -86,9 +97,10 @@ PREFIX_LOGS: Final[dict[str, dict]] = {
         "emoji": "⚙️",
         "desc": "logs from any usage from custom caller function.",
     },
+    "NESTED": {"emoji": "⛓️", "desc": "logs from stages module."},
     "STAGE": {"emoji": "🔗", "desc": "logs from stages module."},
-    "JOB": {"emoji": "⛓️", "desc": "logs from job module."},
-    "WORKFLOW": {"emoji": "🏃", "desc": "logs from workflow module."},
+    "JOB": {"emoji": "🏗", "desc": "logs from job module."},
+    "WORKFLOW": {"emoji": "👟", "desc": "logs from workflow module."},
     "RELEASE": {"emoji": "📅", "desc": "logs from release workflow method."},
     "POKING": {"emoji": "⏰", "desc": "logs from poke workflow method."},
     "AUDIT": {"emoji": "📌", "desc": "logs from audit model."},
@@ -229,7 +241,7 @@ class Metadata(BaseModel):  # pragma: no cov
         default=None, description="Environment (dev, staging, prod)."
     )
 
-    # System context
+    # NOTE: System context
     hostname: Optional[str] = Field(
         default=None, description="Hostname where workflow is running."
     )
@@ -243,7 +255,7 @@ class Metadata(BaseModel):  # pragma: no cov
         default=None, description="Workflow package version."
     )
 
-    # Custom metadata
+    # NOTE: Custom metadata
     tags: Optional[list[str]] = Field(
         default_factory=list, description="Custom tags for categorization."
     )
@@ -310,6 +322,8 @@ class Metadata(BaseModel):  # pragma: no cov
         import socket
         import sys
 
+        from .__about__ import __version__
+
         frame: Optional[FrameType] = currentframe()
         if frame is None:
             raise ValueError("Cannot get current frame")
@@ -374,7 +388,7 @@ class Metadata(BaseModel):  # pragma: no cov
             hostname=hostname,
             ip_address=ip_address,
             python_version=python_version,
-            package_version=extras_data.get("package_version"),
+            package_version=__version__,
             # NOTE: Custom metadata
             tags=extras_data.get("tags", []),
             metadata=extras_data.get("metadata", {}),
@@ -427,6 +441,9 @@ class BaseHandler(BaseModel, ABC):
         self, metadata: list[Metadata], *, extra: Optional[DictData] = None
     ) -> None: ...
 
+    def pre(self) -> None:
+        """Pre-process of handler that will execute when start create trance."""
+
 
 class ConsoleHandler(BaseHandler):
     """Console Handler model."""
@@ -460,14 +477,20 @@ class FileHandler(BaseHandler):
     metadata_filename: ClassVar[str] = "metadata.txt"
 
     type: Literal["file"] = "file"
-    path: str = Field(description="A file path.")
+    path: str = Field(
+        description=(
+            "A file path that use to save all trace log files that include "
+            "stdout, stderr, and metadata."
+        )
+    )
     format: str = Field(
         default=(
             "{datetime} ({process:5d}, {thread:5d}) ({cut_id}) {message:120s} "
             "({filename}:{lineno})"
-        )
+        ),
+        description="A trace log format that write on stdout and stderr files.",
     )
-    buffer_size: int = 8192
+    buffer_size: int = Field(default=8192)
 
     # NOTE: Private attrs for the internal process.
     _lock: Lock = PrivateAttr(default_factory=Lock)
@@ -488,7 +511,9 @@ class FileHandler(BaseHandler):
             log_file.mkdir(parents=True)
         return log_file
 
-    def pre(self) -> None: ...
+    def pre(self) -> None:  # pragma: no cov
+        if not (p := Path(self.path)).exists():
+            p.mkdir(parents=True)
 
     def emit(
         self,
@@ -496,6 +521,7 @@ class FileHandler(BaseHandler):
         *,
         extra: Optional[DictData] = None,
     ) -> None:
+        """Emit trace log."""
         pointer: Path = self.pointer(metadata.pointer_id)
         std_file = "stderr" if metadata.error_flag else "stdout"
         with self._lock:
@@ -518,7 +544,9 @@ class FileHandler(BaseHandler):
         try:
             import aiofiles
         except ImportError as e:
-            raise ImportError("Async mode need aiofiles package") from e
+            raise ImportError(
+                "Async mode need to install `aiofiles` package first"
+            ) from e
 
         with self._lock:
             pointer: Path = self.pointer(metadata.pointer_id)
@@ -538,6 +566,7 @@ class FileHandler(BaseHandler):
     def flush(
         self, metadata: list[Metadata], *, extra: Optional[DictData] = None
     ) -> None:
+        """Flush logs."""
         with self._lock:
             pointer: Path = self.pointer(metadata[0].pointer_id)
             stdout_file = open(
@@ -613,7 +642,7 @@ class FileHandler(BaseHandler):
         """Find trace logs.
 
         Args:
-            path: A trace path that want to find.
+            path (Path | None, default None): A trace path that want to find.
         """
         for file in sorted(
             (path or Path(self.path)).glob("./run_id=*"),
@@ -634,6 +663,9 @@ class FileHandler(BaseHandler):
             run_id: A running ID of trace log.
             force_raise: Whether to raise an exception if not found.
             path: Optional path override.
+
+        Returns:
+            TraceData: A TranceData instance that already passed searching data.
         """
         base_path: Path = path or self.path
         file: Path = base_path / f"run_id={run_id}"
@@ -757,7 +789,8 @@ class SQLiteHandler(BaseHandler):  # pragma: no cov
         metadata: Metadata,
         *,
         extra: Optional[DictData] = None,
-    ) -> None: ...
+    ) -> None:
+        raise NotImplementedError("Does not implement async emit yet.")
 
     def flush(
         self, metadata: list[Metadata], *, extra: Optional[DictData] = None
@@ -1506,7 +1539,6 @@ class ElasticHandler(BaseHandler):  # pragma: no cov
         try:
             from elasticsearch import Elasticsearch
 
-            # Create client
             client = Elasticsearch(
                 hosts=es_hosts if isinstance(es_hosts, list) else [es_hosts],
                 basic_auth=(
@@ -1653,8 +1685,6 @@ class ElasticHandler(BaseHandler):  # pragma: no cov
 
             for hit in response["hits"]["hits"]:
                 source = hit["_source"]
-
-                # Convert to TraceMeta
                 trace_meta = Metadata(
                     run_id=source["run_id"],
                     parent_run_id=source["parent_run_id"],
@@ -1724,6 +1754,7 @@ class ElasticHandler(BaseHandler):  # pragma: no cov
             return TraceData(stdout="", stderr="")
 
 
+Handler = TypeVar("Handler", bound=BaseHandler)
 TraceHandler = Annotated[
     Union[
         ConsoleHandler,
@@ -1866,7 +1897,7 @@ class BaseAsyncEmit(ABC):
         await self.amit(msg, level="exception")
 
 
-class TraceManager(BaseModel, BaseEmit, BaseAsyncEmit):
+class Trace(BaseModel, BaseEmit, BaseAsyncEmit):
     """Trace Manager model that keep all trance handler and emit log to its
     handler.
     """
@@ -1955,7 +1986,7 @@ class TraceManager(BaseModel, BaseEmit, BaseAsyncEmit):
         any logging level.
 
         Args:
-            msg: A message that want to log.
+            msg (str): A message that want to log.
             level (Level): A logging mode.
         """
         _msg: str = self.make_message(msg)
@@ -2005,10 +2036,12 @@ class TraceManager(BaseModel, BaseEmit, BaseAsyncEmit):
 def get_trace(
     run_id: str,
     *,
+    handlers: list[Union[DictData, Handler]] = None,
     parent_run_id: Optional[str] = None,
     extras: Optional[DictData] = None,
-) -> TraceManager:
-    """Get dynamic TraceManager instance from the core config.
+    auto_pre_process: bool = False,
+) -> Trace:
+    """Get dynamic Trace instance from the core config.
 
     This factory function returns the appropriate trace implementation based on
     configuration. It can be overridden by extras argument and accepts running ID
@@ -2017,16 +2050,27 @@ def get_trace(
     Args:
         run_id (str): A running ID.
         parent_run_id (str | None, default None): A parent running ID.
+        handlers (list):
         extras: An extra parameter that want to override the core
             config values.
+        auto_pre_process (bool, default False)
 
     Returns:
-        TraceManager: The appropriate trace instance.
+        Trace: The appropriate trace instance.
     """
-    handlers = dynamic("trace_handlers", extras=extras)
-    return TraceManager(
-        run_id=run_id,
-        parent_run_id=parent_run_id,
-        handlers=handlers,
-        extras=extras or {},
+    handlers: list[DictData] = dynamic(
+        "trace_handlers", f=handlers, extras=extras
+    )
+    trace: Trace = Trace.model_validate(
+        {
+            "run_id": run_id,
+            "parent_run_id": parent_run_id,
+            "handlers": handlers,
+            "extras": extras or {},
+        }
     )
+    # NOTE: Start pre-process when start create trace.
+    if auto_pre_process:
+        for handler in trace.handlers:
+            handler.pre()
+    return trace

ddeutil/workflow/utils.py

@@ -8,26 +8,6 @@
 This module provides essential utility functions used throughout the workflow
 system for ID generation, datetime handling, string processing, template
 operations, and other common tasks.
-
-Functions:
-    to_train: Convert camel case strings to train case format
-    prepare_newline: Format messages with multiple newlines
-    replace_sec: Replace seconds and microseconds in datetime objects
-    clear_tz: Clear timezone info from datetime objects
-    get_dt_now: Get current datetime with timezone
-    get_d_now: Get current date
-    get_diff_sec: Calculate time difference in seconds
-    reach_next_minute: Check if datetime reaches next minute
-    wait_until_next_minute: Wait until next minute
-    delay: Add random delay to execution
-    gen_id: Generate unique identifiers for workflow components
-    default_gen_id: Generate default running ID
-    make_exec: Make files executable
-    filter_func: Filter function objects from data structures
-    cross_product: Generate cross product of matrix values
-    cut_id: Cut running ID to specified length
-    dump_all: Serialize nested BaseModel objects to dictionaries
-    obj_name: Get object name or class name
 """
 from __future__ import annotations
 
@@ -218,7 +198,10 @@ def gen_id(
     hashing value length to 10 if simple mode is enabled.
 
     Simple Mode Format:
-        YYYYMMDDHHMMSSffffffTxxxxxxxxxx
+
+        The format of ID include full datetime and hashing identity.
+
+        YYYY MM    DD  HH   MM     SS     ffffff      T   **********
         year month day hour minute second microsecond sep simple-id
 
     Args:
@@ -250,6 +233,33 @@ def gen_id(
     ).hexdigest()
 
 
+def extract_id(
+    name: str,
+    run_id: Optional[str] = None,
+    extras: Optional[DictData] = None,
+) -> tuple[str, str]:
+    """Extract the parent ID and running ID. If the `run_id` parameter was
+    passed, it will replace the parent_run_id with this value and re-generate
+    new running ID for it instead.
+
+    Args:
+        name (str): A name for generate hashing value for the `gen_id` function.
+        run_id (str | None, default None):
+        extras:
+
+    Returns:
+        tuple[str, str]: A pair of parent running ID and running ID.
+    """
+    generated = gen_id(name, unique=True, extras=extras)
+    if run_id:
+        parent_run_id: str = run_id
+        run_id: str = generated
+    else:
+        run_id: str = generated
+        parent_run_id: str = run_id
+    return parent_run_id, run_id
+
+
 def default_gen_id() -> str:
     """Return running ID for making default ID for the Result model.
 
@@ -318,12 +328,14 @@ def cross_product(matrix: Matrix) -> Iterator[DictData]:
     )
 
 
-def cut_id(run_id: str, *, num: int = 6) -> str:
+def cut_id(run_id: str, *, num: int = 8) -> str:
     """Cut running ID to specified length.
 
     Example:
         >>> cut_id(run_id='20240101081330000000T1354680202')
         '202401010813680202'
+        >>> cut_id(run_id='20240101081330000000T1354680202')
+        '54680202'
 
     Args:
         run_id: A running ID to cut.
@@ -334,8 +346,8 @@ def cut_id(run_id: str, *, num: int = 6) -> str:
     """
     if "T" in run_id:
         dt, simple = run_id.split("T", maxsplit=1)
-        return dt[:12] + simple[-num:]
-    return run_id[:12] + run_id[-num:]
+        return dt[10:20] + simple[-num:]
+    return run_id[-num:]
 
 
 @overload
@@ -391,3 +403,8 @@ def obj_name(obj: Optional[Union[str, object]] = None) -> Optional[str]:
     else:
         obj_type: str = obj.__class__.__name__
     return obj_type
+
+
+def remove_sys_extras(extras: DictData) -> DictData:
+    """Remove key that starts with `__sys_` from the extra dict parameter."""
+    return {k: extras[k] for k in extras if not k.startswith("__sys_")}