ddeutil-workflow 0.0.59__py3-none-any.whl → 0.0.61__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1 +1 @@
-__version__: str = "0.0.59"
+__version__: str = "0.0.61"

ddeutil/workflow/__types.py

@@ -20,6 +20,7 @@ from typing import Any, Optional, TypedDict, Union
 
 from typing_extensions import Self
 
+StrOrNone = Optional[str]
 StrOrInt = Union[str, int]
 TupleStr = tuple[str, ...]
 DictData = dict[str, Any]
@@ -42,7 +43,7 @@ class CallerRe:
 
     full: str
     caller: str
-    caller_prefix: Optional[str]
+    caller_prefix: StrOrNone
     caller_last: str
     post_filters: str
 
@@ -50,6 +51,9 @@ class CallerRe:
     def from_regex(cls, match: Match[str]) -> Self:
         """Class construct from matching result.
 
+        :param match: A match string object for contract this Caller regex data
+            class.
+
         :rtype: Self
         """
         return cls(full=match.group(0), **match.groupdict())
@@ -121,10 +125,13 @@ class Re:
     )
 
     @classmethod
-    def finditer_caller(cls, value) -> Iterator[CallerRe]:
+    def finditer_caller(cls, value: str) -> Iterator[CallerRe]:
         """Generate CallerRe object that create from matching object that
         extract with re.finditer function.
 
+        :param value: (str) A string value that want to finditer with the caller
+            regular expression.
+
         :rtype: Iterator[CallerRe]
         """
         for found in cls.RE_CALLER.finditer(value):

ddeutil/workflow/event.py

@@ -3,8 +3,9 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
-"""Event module that store all event object. Now, it has only `Crontab` and
-`CrontabYear` model these are schedule with crontab event.
+"""Event module include all event object for trigger the Workflow to release.
+Now, it has only `Crontab` and `CrontabYear` event models on this module because
+I think it is the core event for workflow orchestration.
 """
 from __future__ import annotations
 
@@ -86,7 +87,7 @@ class Crontab(BaseModel):
         CronJob,
         Field(
             description=(
-                "A Cronjob object that use for validate and generate datetime.",
+                "A Cronjob object that use for validate and generate datetime."
             ),
         ),
     ]
@@ -117,7 +118,6 @@ class Crontab(BaseModel):
         passing["cronjob"] = interval2crontab(
             **{v: value[v] for v in value if v in ("interval", "day", "time")}
         )
-        print(passing)
         return cls(extras=extras | passing.pop("extras", {}), **passing)
 
     @classmethod
@@ -170,9 +170,10 @@ class Crontab(BaseModel):
 
     @model_validator(mode="before")
     def __prepare_values(cls, data: Any) -> Any:
-        """Extract tz key from value and change name to timezone key.
+        """Extract a `tz` key from data and change the key name from `tz` to
+        `timezone`.
 
-        :param data: (DictData) A data that want to pass for create an Crontab
+        :param data: (DictData) A data that want to pass for create a Crontab
             model.
 
         :rtype: DictData
@@ -198,7 +199,7 @@ class Crontab(BaseModel):
         "cronjob", mode="before", json_schema_input_type=Union[CronJob, str]
     )
     def __prepare_cronjob(
-        cls, value: str | CronJob, info: ValidationInfo
+        cls, value: Union[str, CronJob], info: ValidationInfo
     ) -> CronJob:
         """Prepare crontab value that able to receive with string type.
         This step will get options kwargs from extras field and pass to the
@@ -234,7 +235,7 @@ class Crontab(BaseModel):
         """
         return str(value)
 
-    def generate(self, start: str | datetime) -> CronRunner:
+    def generate(self, start: Union[str, datetime]) -> CronRunner:
         """Return CronRunner object from an initial datetime.
 
         :param start: (str | datetime) A string or datetime for generate the
@@ -248,7 +249,7 @@ class Crontab(BaseModel):
             raise TypeError("start value should be str or datetime type.")
         return self.cronjob.schedule(date=start, tz=self.tz)
 
-    def next(self, start: str | datetime) -> CronRunner:
+    def next(self, start: Union[str, datetime]) -> CronRunner:
         """Return a next datetime from Cron runner object that start with any
         date that given from input.
 
@@ -277,16 +278,18 @@ class CrontabYear(Crontab):
         CronJobYear,
         Field(
             description=(
-                "A Cronjob object that use for validate and generate datetime.",
+                "A Cronjob object that use for validate and generate datetime."
             ),
         ),
     ]
 
     @field_validator(
-        "cronjob", mode="before", json_schema_input_type=Union[CronJobYear, str]
+        "cronjob",
+        mode="before",
+        json_schema_input_type=Union[CronJobYear, str],
     )
     def __prepare_cronjob(
-        cls, value: str | CronJobYear, info: ValidationInfo
+        cls, value: Union[CronJobYear, str], info: ValidationInfo
     ) -> CronJobYear:
         """Prepare crontab value that able to receive with string type.
         This step will get options kwargs from extras field and pass to the
@@ -311,3 +314,15 @@ class CrontabYear(Crontab):
             if isinstance(value, str)
             else value
         )
+
+
+Event = Annotated[
+    Union[
+        CronJobYear,
+        CronJob,
+    ],
+    Field(
+        union_mode="smart",
+        description="An event models.",
+    ),
+]  # pragma: no cov

ddeutil/workflow/exceptions.py

@@ -9,7 +9,7 @@ annotate for handle error only.
 """
 from __future__ import annotations
 
-from typing import Literal, Optional, TypedDict, overload
+from typing import Literal, Optional, TypedDict, Union, overload
 
 
 class ErrorData(TypedDict):
@@ -55,8 +55,9 @@ class BaseWorkflowException(Exception):
 
     def to_dict(
         self, with_refs: bool = False
-    ) -> ErrorData | dict[str, ErrorData]:
-        """Return ErrorData data from the current exception object.
+    ) -> Union[ErrorData, dict[str, ErrorData]]:
+        """Return ErrorData data from the current exception object. If with_refs
+        flag was set, it will return mapping of refs and itself data.
 
         :rtype: ErrorData
         """

ddeutil/workflow/job.py

@@ -39,7 +39,7 @@ from pydantic import BaseModel, Discriminator, Field, SecretStr, Tag
 from pydantic.functional_validators import field_validator, model_validator
 from typing_extensions import Self
 
-from .__types import DictData, DictStr, Matrix
+from .__types import DictData, DictStr, Matrix, StrOrNone
 from .exceptions import (
     JobException,
     StageException,
@@ -329,14 +329,14 @@ class Job(BaseModel):
         ... }
     """
 
-    id: Optional[str] = Field(
+    id: StrOrNone = Field(
         default=None,
         description=(
             "A job ID that was set from Workflow model after initialize step. "
             "If this model create standalone, it will be None."
         ),
     )
-    desc: Optional[str] = Field(
+    desc: StrOrNone = Field(
         default=None,
         description="A job description that can be markdown syntax.",
     )
@@ -345,7 +345,7 @@ class Job(BaseModel):
         description="A target node for this job to use for execution.",
         alias="runs-on",
     )
-    condition: Optional[str] = Field(
+    condition: StrOrNone = Field(
         default=None,
         description="A job condition statement to allow job executable.",
         alias="if",
@@ -526,7 +526,7 @@ class Job(BaseModel):
         output: DictData,
         to: DictData,
         *,
-        job_id: Optional[str] = None,
+        job_id: StrOrNone = None,
     ) -> DictData:
         """Set an outputs from execution result context to the received context
         with a `to` input parameter. The result context from job strategy
@@ -567,7 +567,7 @@ class Job(BaseModel):
         :param output: (DictData) A result data context that want to extract
             and transfer to the `strategies` key in receive context.
         :param to: (DictData) A received context data.
-        :param job_id: (Optional[str]) A job ID if the `id` field does not set.
+        :param job_id: (StrOrNone) A job ID if the `id` field does not set.
 
         :rtype: DictData
         """
@@ -607,8 +607,8 @@ class Job(BaseModel):
         self,
         params: DictData,
         *,
-        run_id: Optional[str] = None,
-        parent_run_id: Optional[str] = None,
+        run_id: StrOrNone = None,
+        parent_run_id: StrOrNone = None,
         event: Optional[Event] = None,
     ) -> Result:
         """Job execution with passing dynamic parameters from the workflow
@@ -800,8 +800,8 @@ def local_execute(
     job: Job,
     params: DictData,
     *,
-    run_id: Optional[str] = None,
-    parent_run_id: Optional[str] = None,
+    run_id: StrOrNone = None,
+    parent_run_id: StrOrNone = None,
     event: Optional[Event] = None,
 ) -> Result:
     """Local job execution with passing dynamic parameters from the workflow
@@ -919,8 +919,8 @@ def self_hosted_execute(
     job: Job,
     params: DictData,
     *,
-    run_id: Optional[str] = None,
-    parent_run_id: Optional[str] = None,
+    run_id: StrOrNone = None,
+    parent_run_id: StrOrNone = None,
     event: Optional[Event] = None,
 ) -> Result:  # pragma: no cov
     """Self-Hosted job execution with passing dynamic parameters from the
@@ -982,8 +982,8 @@ def azure_batch_execute(
     job: Job,
     params: DictData,
     *,
-    run_id: Optional[str] = None,
-    parent_run_id: Optional[str] = None,
+    run_id: StrOrNone = None,
+    parent_run_id: StrOrNone = None,
     event: Optional[Event] = None,
 ) -> Result:  # pragma: no cov
     """Azure Batch job execution that will run all job's stages on the Azure
@@ -1036,8 +1036,8 @@ def docker_execution(
     job: Job,
     params: DictData,
     *,
-    run_id: Optional[str] = None,
-    parent_run_id: Optional[str] = None,
+    run_id: StrOrNone = None,
+    parent_run_id: StrOrNone = None,
     event: Optional[Event] = None,
 ):  # pragma: no cov
     """Docker job execution.

ddeutil/workflow/logs.py

@@ -4,13 +4,17 @@
 # license information.
 # ------------------------------------------------------------------------------
 # [x] Use fix config for `get_logger`, and Model initialize step.
-"""A Logs module contain Trace dataclass and Audit Pydantic model.
+"""A Logs module contain Trace and Audit Pydantic models for process log from
+the core workflow engine. I separate part of log to 2 types:
+  - Trace: A stdout and stderr log
+  - Audit: An audit release log for tracking incremental running workflow.
 """
 from __future__ import annotations
 
 import json
 import logging
 import os
+import re
 from abc import ABC, abstractmethod
 from collections.abc import Iterator
 from datetime import datetime
@@ -18,7 +22,8 @@ from functools import lru_cache
 from inspect import Traceback, currentframe, getframeinfo
 from pathlib import Path
 from threading import get_ident
-from typing import ClassVar, Literal, Optional, TypeVar, Union
+from types import FrameType
+from typing import ClassVar, Final, Literal, Optional, TypeVar, Union
 
 from pydantic import BaseModel, ConfigDict, Field
 from pydantic.functional_validators import model_validator
@@ -28,12 +33,14 @@ from .__types import DictData
 from .conf import config, dynamic
 from .utils import cut_id, get_dt_now, prepare_newline
 
+METADATA: str = "metadata.json"
+
 
 @lru_cache
 def get_logger(name: str):
     """Return logger object with an input module name.
 
-    :param name: A module name that want to log.
+    :param name: (str) A module name that want to log.
     """
     lg = logging.getLogger(name)
 
@@ -67,14 +74,59 @@ def get_dt_tznow() -> datetime:  # pragma: no cov
     return get_dt_now(tz=config.tz)
 
 
-PREFIX_LOGS: dict[str, dict] = {
-    "CALLER": {"emoji": ""},
-    "STAGE": {"emoji": ""},
-    "JOB": {"emoji": ""},
-    "WORKFLOW": {"emoji": "🏃"},
-    "RELEASE": {"emoji": ""},
-    "POKE": {"emoji": ""},
+PREFIX_LOGS: Final[dict[str, dict]] = {
+    "CALLER": {
+        "emoji": "📍",
+        "desc": "logs from any usage from custom caller function.",
+    },
+    "STAGE": {"emoji": "⚙️", "desc": "logs from stages module."},
+    "JOB": {"emoji": "⛓️", "desc": "logs from job module."},
+    "WORKFLOW": {"emoji": "🏃", "desc": "logs from workflow module."},
+    "RELEASE": {"emoji": "📅", "desc": "logs from release workflow method."},
+    "POKING": {"emoji": "⏰", "desc": "logs from poke workflow method."},
 }  # pragma: no cov
+PREFIX_DEFAULT: Final[str] = "CALLER"
+PREFIX_LOGS_REGEX: re.Pattern[str] = re.compile(
+    rf"(^\[(?P<name>{'|'.join(PREFIX_LOGS)})]:\s?)?(?P<message>.*)",
+    re.MULTILINE | re.DOTALL | re.ASCII | re.VERBOSE,
+)  # pragma: no cov
+
+
+class PrefixMsg(BaseModel):
+    """Prefix Message model for receive grouping dict from searching prefix data
+    from logging message.
+    """
+
+    name: Optional[str] = Field(default=None)
+    message: Optional[str] = Field(default=None)
+
+    def prepare(self, extras: Optional[DictData] = None) -> str:
+        """Prepare message with force add prefix before writing trace log.
+
+        :param extras: (DictData) An extra parameter that want to get the
+            `log_add_emoji` flag.
+
+        :rtype: str
+        """
+        name: str = self.name or PREFIX_DEFAULT
+        emoji: str = (
+            f"{PREFIX_LOGS[name]['emoji']} "
+            if (extras or {}).get("log_add_emoji", True)
+            else ""
+        )
+        return f"{emoji}[{name}]: {self.message}"
+
+
+def extract_msg_prefix(msg: str) -> PrefixMsg:
+    """Extract message prefix from an input message.
+
+    :param msg: A message that want to extract.
+
+    :rtype: PrefixMsg
+    """
+    return PrefixMsg.model_validate(
+        obj=PREFIX_LOGS_REGEX.search(msg).groupdict()
+    )
 
 
 class TraceMeta(BaseModel):  # pragma: no cov
@@ -91,6 +143,28 @@ class TraceMeta(BaseModel):  # pragma: no cov
     filename: str = Field(description="A filename of this log.")
     lineno: int = Field(description="A line number of this log.")
 
+    @classmethod
+    def dynamic_frame(
+        cls, frame: FrameType, *, extras: Optional[DictData] = None
+    ) -> Traceback:
+        """Dynamic Frame information base on the `logs_trace_frame_layer` config
+        value that was set from the extra parameter.
+
+        :param frame: (FrameType) The current frame that want to dynamic.
+        :param extras: (DictData) An extra parameter that want to get the
+            `logs_trace_frame_layer` config value.
+        """
+        extras: DictData = extras or {}
+        layer: int = extras.get("logs_trace_frame_layer", 4)
+        for _ in range(layer):
+            _frame: Optional[FrameType] = frame.f_back
+            if _frame is None:
+                raise ValueError(
+                    f"Layer value does not valid, the maximum frame is: {_ + 1}"
+                )
+            frame: FrameType = _frame
+        return getframeinfo(frame)
+
     @classmethod
     def make(
         cls,
@@ -100,7 +174,8 @@ class TraceMeta(BaseModel):  # pragma: no cov
         *,
         extras: Optional[DictData] = None,
     ) -> Self:
-        """Make the current TraceMeta instance that catching local state.
+        """Make the current metric for contract this TraceMeta model instance
+        that will catch local states like PID, thread identity.
 
         :param mode: (Literal["stdout", "stderr"]) A metadata mode.
         :param message: (str) A message.
@@ -110,9 +185,8 @@ class TraceMeta(BaseModel):  # pragma: no cov
 
         :rtype: Self
         """
-        frame_info: Traceback = getframeinfo(
-            currentframe().f_back.f_back.f_back
-        )
+        frame: FrameType = currentframe()
+        frame_info: Traceback = cls.dynamic_frame(frame, extras=extras)
         extras: DictData = extras or {}
         return cls(
             mode=mode,
@@ -157,13 +231,11 @@ class TraceData(BaseModel):  # pragma: no cov
             if (file / f"{mode}.txt").exists():
                 data[mode] = (file / f"{mode}.txt").read_text(encoding="utf-8")
 
-        if (file / "metadata.json").exists():
+        if (file / METADATA).exists():
             data["meta"] = [
                 json.loads(line)
                 for line in (
-                    (file / "metadata.json")
-                    .read_text(encoding="utf-8")
-                    .splitlines()
+                    (file / METADATA).read_text(encoding="utf-8").splitlines()
                 )
             ]
 
@@ -263,7 +335,7 @@ class BaseTrace(BaseModel, ABC):  # pragma: no cov
 
         :param message: (str) A message that want to log.
         """
-        msg: str = prepare_newline(self.make_message(message))
+        msg: str = self.make_message(message)
 
         if mode != "debug" or (
             mode == "debug" and dynamic("debug", extras=self.extras)
@@ -320,7 +392,7 @@ class BaseTrace(BaseModel, ABC):  # pragma: no cov
 
         :param message: (str) A message that want to log.
         """
-        msg: str = prepare_newline(self.make_message(message))
+        msg: str = self.make_message(message)
 
         if mode != "debug" or (
             mode == "debug" and dynamic("debug", extras=self.extras)
@@ -441,13 +513,15 @@ class FileTrace(BaseTrace):  # pragma: no cov
         return f"{cut_parent_run_id} -> {cut_run_id}"
 
     def make_message(self, message: str) -> str:
-        """Prepare and Make a message before write and log processes.
+        """Prepare and Make a message before write and log steps.
 
         :param message: (str) A message that want to prepare and make before.
 
         :rtype: str
         """
-        return f"({self.cut_id}) {message}"
+        return prepare_newline(
+            f"({self.cut_id}) {extract_msg_prefix(message).prepare(self.extras)}"
+        )
 
     def writer(self, message: str, level: str, is_err: bool = False) -> None:
         """Write a trace message after making to target file and write metadata
@@ -468,7 +542,7 @@ class FileTrace(BaseTrace):  # pragma: no cov
 
         mode: Literal["stdout", "stderr"] = "stderr" if is_err else "stdout"
         trace_meta: TraceMeta = TraceMeta.make(
-            mode=mode, level=level, message=message
+            mode=mode, level=level, message=message, extras=self.extras
         )
 
         with (self.pointer / f"{mode}.txt").open(
@@ -477,9 +551,7 @@ class FileTrace(BaseTrace):  # pragma: no cov
             fmt: str = dynamic("log_format_file", extras=self.extras)
             f.write(f"{fmt}\n".format(**trace_meta.model_dump()))
 
-        with (self.pointer / "metadata.json").open(
-            mode="at", encoding="utf-8"
-        ) as f:
+        with (self.pointer / METADATA).open(mode="at", encoding="utf-8") as f:
             f.write(trace_meta.model_dump_json() + "\n")
 
     async def awriter(
@@ -496,7 +568,7 @@ class FileTrace(BaseTrace):  # pragma: no cov
 
         mode: Literal["stdout", "stderr"] = "stderr" if is_err else "stdout"
         trace_meta: TraceMeta = TraceMeta.make(
-            mode=mode, level=level, message=message
+            mode=mode, level=level, message=message, extras=self.extras
         )
 
         async with aiofiles.open(
@@ -506,7 +578,7 @@ class FileTrace(BaseTrace):  # pragma: no cov
             await f.write(f"{fmt}\n".format(**trace_meta.model_dump()))
 
         async with aiofiles.open(
-            self.pointer / "metadata.json", mode="at", encoding="utf-8"
+            self.pointer / METADATA, mode="at", encoding="utf-8"
         ) as f:
             await f.write(trace_meta.model_dump_json() + "\n")