ddeutil-workflow 0.0.34__py3-none-any.whl → 0.0.36__py3-none-any.whl

ddeutil/workflow/audit.py

@@ -20,7 +20,7 @@ from typing_extensions import Self
 
 from .__types import DictData, TupleStr
 from .conf import config
-from .result import TraceLog
+from .logs import TraceLog, get_trace
 
 __all__: TupleStr = (
     "get_audit",
@@ -112,7 +112,8 @@ class FileAudit(BaseAudit):
         :param release: A release datetime that want to search log.
 
         :raise FileNotFoundError:
-        :raise NotImplementedError:
+        :raise NotImplementedError: If an input release does not pass to this
+            method. Because this method does not implement latest log.
 
         :rtype: Self
         """
@@ -174,14 +175,16 @@ class FileAudit(BaseAudit):
 
         :rtype: Self
         """
-        trace: TraceLog = TraceLog(self.run_id, self.parent_run_id)
+        trace: TraceLog = get_trace(self.run_id, self.parent_run_id)
 
         # NOTE: Check environ variable was set for real writing.
         if not config.enable_write_audit:
             trace.debug("[LOG]: Skip writing log cause config was set")
             return self
 
-        log_file: Path = self.pointer() / f"{self.run_id}.log"
+        log_file: Path = (
+            self.pointer() / f"{self.parent_run_id or self.run_id}.log"
+        )
         log_file.write_text(
             json.dumps(
                 self.model_dump(exclude=excluded),
@@ -196,7 +199,7 @@ class FileAudit(BaseAudit):
 class SQLiteAudit(BaseAudit):  # pragma: no cov
     """SQLite Audit Pydantic Model."""
 
-    table_name: ClassVar[str] = "workflow_log"
+    table_name: ClassVar[str] = "audits"
     schemas: ClassVar[
         str
     ] = """
@@ -214,7 +217,7 @@ class SQLiteAudit(BaseAudit):  # pragma: no cov
         """Save logging data that receive a context data from a workflow
         execution result.
         """
-        trace: TraceLog = TraceLog(self.run_id, self.parent_run_id)
+        trace: TraceLog = get_trace(self.run_id, self.parent_run_id)
 
         # NOTE: Check environ variable was set for real writing.
         if not config.enable_write_audit:

ddeutil/workflow/{call.py → caller.py}

@@ -60,7 +60,7 @@ def tag(
 
         @wraps(func)
         def wrapped(*args: P.args, **kwargs: P.kwargs) -> TagFunc:
-            # NOTE: Able to do anything before calling call function.
+            # NOTE: Able to do anything before calling the call function.
             return func(*args, **kwargs)
 
         return wrapped
@@ -150,7 +150,7 @@ def extract_call(call: str) -> Callable[[], TagFunc]:
     """
     if not (found := Re.RE_TASK_FMT.search(call)):
         raise ValueError(
-            f"Call {call!r} does not match with call format regex."
+            f"Call {call!r} does not match with the call regex format."
         )
 
     # NOTE: Pass the searching call string to `path`, `func`, and `tag`.
@@ -160,13 +160,13 @@ def extract_call(call: str) -> Callable[[], TagFunc]:
     rgt: dict[str, Registry] = make_registry(f"{call.path}")
     if call.func not in rgt:
         raise NotImplementedError(
-            f"``REGISTER-MODULES.{call.path}.registries`` does not "
+            f"`REGISTER-MODULES.{call.path}.registries` does not "
             f"implement registry: {call.func!r}."
         )
 
     if call.tag not in rgt[call.func]:
         raise NotImplementedError(
             f"tag: {call.tag!r} does not found on registry func: "
-            f"``REGISTER-MODULES.{call.path}.registries.{call.func}``"
+            f"`REGISTER-MODULES.{call.path}.registries.{call.func}`"
         )
     return rgt[call.func][call.tag]

ddeutil/workflow/job.py

@@ -5,7 +5,7 @@
 # ------------------------------------------------------------------------------
 """Job Model that use for keeping stages and node that running its stages.
 The job handle the lineage of stages and location of execution of stages that
-mean the job model able to define ``runs-on`` key that allow you to run this
+mean the job model able to define `runs-on` key that allow you to run this
 job.
 
     This module include Strategy Model that use on the job strategy field.
@@ -24,15 +24,15 @@ from enum import Enum
 from functools import lru_cache
 from textwrap import dedent
 from threading import Event
-from typing import Any, Optional, Union
+from typing import Annotated, Any, Literal, Optional, Union
 
 from ddeutil.core import freeze_args
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, ConfigDict, Field
 from pydantic.functional_validators import field_validator, model_validator
 from typing_extensions import Self
 
 from .__types import DictData, DictStr, Matrix, TupleStr
-from .conf import config, get_logger
+from .conf import config
 from .exceptions import (
     JobException,
     StageException,
@@ -48,7 +48,6 @@ from .utils import (
     gen_id,
 )
 
-logger = get_logger("ddeutil.workflow")
 MatrixFilter = list[dict[str, Union[str, int]]]
 
 
@@ -56,6 +55,10 @@ __all__: TupleStr = (
     "Strategy",
     "Job",
     "TriggerRules",
+    "RunsOn",
+    "RunsOnLocal",
+    "RunsOnSelfHosted",
+    "RunsOnK8s",
     "make",
 )
 
@@ -216,13 +219,52 @@ class TriggerRules(str, Enum):
     none_skipped: str = "none_skipped"
 
 
-class RunsOn(str, Enum):
+class RunsOnType(str, Enum):
     """Runs-On enum object."""
 
-    local: str = "local"
-    docker: str = "docker"
-    self_hosted: str = "self_hosted"
-    k8s: str = "k8s"
+    LOCAL: str = "local"
+    SELF_HOSTED: str = "self_hosted"
+    K8S: str = "k8s"
+
+
+class BaseRunsOn(BaseModel):  # pragma: no cov
+    model_config = ConfigDict(use_enum_values=True)
+
+    type: Literal[RunsOnType.LOCAL]
+    args: DictData = Field(
+        default_factory=dict,
+        alias="with",
+    )
+
+
+class RunsOnLocal(BaseRunsOn):  # pragma: no cov
+    """Runs-on local."""
+
+    type: Literal[RunsOnType.LOCAL] = Field(default=RunsOnType.LOCAL)
+
+
+class RunsOnSelfHosted(BaseRunsOn):  # pragma: no cov
+    """Runs-on self-hosted."""
+
+    type: Literal[RunsOnType.SELF_HOSTED] = Field(
+        default=RunsOnType.SELF_HOSTED
+    )
+
+
+class RunsOnK8s(BaseRunsOn):  # pragma: no cov
+    """Runs-on Kubernetes."""
+
+    type: Literal[RunsOnType.K8S] = Field(default=RunsOnType.K8S)
+
+
+RunsOn = Annotated[
+    Union[
+        RunsOnLocal,
+        RunsOnSelfHosted,
+        RunsOnK8s,
+    ],
+    Field(discriminator="type"),
+]
 
 
 class Job(BaseModel):
@@ -234,7 +276,7 @@ class Job(BaseModel):
 
     Data Validate:
         >>> job = {
-        ...     "runs-on": None,
+        ...     "runs-on": {"type": "local"},
         ...     "strategy": {
         ...         "max-parallel": 1,
         ...         "matrix": {
@@ -263,9 +305,9 @@ class Job(BaseModel):
         default=None,
         description="A job description that can be string of markdown content.",
     )
-    runs_on: Optional[str] = Field(
-        default=None,
-        description="A target executor node for this job use to execution.",
+    runs_on: RunsOn = Field(
+        default_factory=RunsOnLocal,
+        description="A target node for this job to use for execution.",
         serialization_alias="runs-on",
     )
     stages: list[Stage] = Field(
@@ -359,7 +401,7 @@ class Job(BaseModel):
 
     def set_outputs(self, output: DictData, to: DictData) -> DictData:
         """Set an outputs from execution process to the received context. The
-        result from execution will pass to value of ``strategies`` key.
+        result from execution will pass to value of `strategies` key.
 
             For example of setting output method, If you receive execute output
         and want to set on the `to` like;
@@ -424,14 +466,14 @@ class Job(BaseModel):
         workflow execution to strategy matrix.
 
             This execution is the minimum level of execution of this job model.
-        It different with ``self.execute`` because this method run only one
+        It different with `self.execute` because this method run only one
         strategy and return with context of this strategy data.
 
             The result of this execution will return result with strategy ID
         that generated from the `gen_id` function with an input strategy value.
 
-        :raise JobException: If it has any error from ``StageException`` or
-            ``UtilException``.
+        :raise JobException: If it has any error from `StageException` or
+            `UtilException`.
 
         :param strategy: A strategy metrix value that use on this execution.
             This value will pass to the `matrix` key for templating.
@@ -510,7 +552,7 @@ class Job(BaseModel):
             #
             #       ... params |= stage.execute(params=params)
             #
-            #   This step will add the stage result to ``stages`` key in
+            #   This step will add the stage result to `stages` key in
             #   that stage id. It will have structure like;
             #
             #   {
@@ -581,7 +623,7 @@ class Job(BaseModel):
     ) -> Result:
         """Job execution with passing dynamic parameters from the workflow
         execution. It will generate matrix values at the first step and run
-        multithread on this metrics to the ``stages`` field of this job.
+        multithread on this metrics to the `stages` field of this job.
 
         :param params: An input parameters that use on job execution.
         :param run_id: A job running ID for this execution.
@@ -591,15 +633,12 @@ class Job(BaseModel):
 
         :rtype: Result
         """
-
-        # NOTE: I use this condition because this method allow passing empty
-        #   params and I do not want to create new dict object.
         if result is None:  # pragma: no cov
             result: Result = Result(
                 run_id=(run_id or gen_id(self.id or "", unique=True)),
                 parent_run_id=parent_run_id,
             )
-        elif parent_run_id:
+        elif parent_run_id:  # pragma: no cov
             result.set_parent_run_id(parent_run_id)
 
         # NOTE: Normal Job execution without parallel strategy matrix. It uses

ddeutil/workflow/logs.py

@@ -0,0 +1,326 @@
+# ------------------------------------------------------------------------------
+# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
+# Licensed under the MIT License. See LICENSE in the project root for
+# license information.
+# ------------------------------------------------------------------------------
+"""A Logs module contain a TraceLog dataclass.
+"""
+from __future__ import annotations
+
+import json
+import os
+from abc import ABC, abstractmethod
+from collections.abc import Iterator
+from datetime import datetime
+from inspect import Traceback, currentframe, getframeinfo
+from pathlib import Path
+from threading import get_ident
+from typing import ClassVar, Literal, Optional, Union
+
+from pydantic import BaseModel, Field
+from pydantic.dataclasses import dataclass
+from typing_extensions import Self
+
+from .__types import DictStr, TupleStr
+from .conf import config, get_logger
+from .utils import cut_id, get_dt_now
+
+logger = get_logger("ddeutil.workflow")
+
+__all__: TupleStr = (
+    "FileTraceLog",
+    "SQLiteTraceLog",
+    "TraceData",
+    "TraceMeda",
+    "TraceLog",
+    "get_dt_tznow",
+    "get_trace",
+    "get_trace_obj",
+)
+
+
+def get_dt_tznow() -> datetime:  # pragma: no cov
+    """Return the current datetime object that passing the config timezone.
+
+    :rtype: datetime
+    """
+    return get_dt_now(tz=config.tz)
+
+
+@dataclass(frozen=True)
+class BaseTraceLog(ABC):  # pragma: no cov
+    """Base Trace Log dataclass object."""
+
+    run_id: str
+    parent_run_id: Optional[str] = None
+
+    @abstractmethod
+    def writer(self, message: str, is_err: bool = False) -> None:
+        """Write a trace message after making to target pointer object. The
+        target can be anything be inherited this class and overwrite this method
+        such as file, console, or database.
+
+        :param message: A message after making.
+        :param is_err: A flag for writing with an error trace or not.
+        """
+        raise NotImplementedError(
+            "Create writer logic for this trace object before using."
+        )
+
+    @abstractmethod
+    def make_message(self, message: str) -> str:
+        """Prepare and Make a message before write and log processes.
+
+        :param message: A message that want to prepare and make before.
+
+        :rtype: str
+        """
+        raise NotImplementedError(
+            "Adjust make message method for this trace object before using."
+        )
+
+    def debug(self, message: str):
+        """Write trace log with append mode and logging this message with the
+        DEBUG level.
+
+        :param message: (str) A message that want to log.
+        """
+        msg: str = self.make_message(message)
+
+        # NOTE: Write file if debug mode was enabled.
+        if config.debug:
+            self.writer(msg)
+
+        logger.debug(msg, stacklevel=2)
+
+    def info(self, message: str) -> None:
+        """Write trace log with append mode and logging this message with the
+        INFO level.
+
+        :param message: (str) A message that want to log.
+        """
+        msg: str = self.make_message(message)
+        self.writer(msg)
+        logger.info(msg, stacklevel=2)
+
+    def warning(self, message: str) -> None:
+        """Write trace log with append mode and logging this message with the
+        WARNING level.
+
+        :param message: (str) A message that want to log.
+        """
+        msg: str = self.make_message(message)
+        self.writer(msg)
+        logger.warning(msg, stacklevel=2)
+
+    def error(self, message: str) -> None:
+        """Write trace log with append mode and logging this message with the
+        ERROR level.
+
+        :param message: (str) A message that want to log.
+        """
+        msg: str = self.make_message(message)
+        self.writer(msg, is_err=True)
+        logger.error(msg, stacklevel=2)
+
+    def exception(self, message: str) -> None:
+        """Write trace log with append mode and logging this message with the
+        EXCEPTION level.
+
+        :param message: (str) A message that want to log.
+        """
+        msg: str = self.make_message(message)
+        self.writer(msg, is_err=True)
+        logger.exception(msg, stacklevel=2)
+
+
+class TraceMeda(BaseModel):  # pragma: no cov
+    mode: Literal["stdout", "stderr"]
+    datetime: str
+    process: int
+    thread: int
+    message: str
+    filename: str
+    lineno: int
+
+
+class TraceData(BaseModel):  # pragma: no cov
+    stdout: str = Field(description="A standard output trace data.")
+    stderr: str = Field(description="A standard error trace data.")
+    meta: list[TraceMeda] = Field(
+        default_factory=list,
+        description=(
+            "A metadata mapping of this output and error before making it to "
+            "standard value."
+        ),
+    )
+
+    @classmethod
+    def from_path(cls, file: Path) -> Self:
+        data: DictStr = {"stdout": "", "stderr": "", "meta": []}
+
+        if (file / "stdout.txt").exists():
+            data["stdout"] = (file / "stdout.txt").read_text(encoding="utf-8")
+
+        if (file / "stderr.txt").exists():
+            data["stderr"] = (file / "stderr.txt").read_text(encoding="utf-8")
+
+        if (file / "metadata.json").exists():
+            data["meta"] = [
+                json.loads(line)
+                for line in (
+                    (file / "metadata.json")
+                    .read_text(encoding="utf-8")
+                    .splitlines()
+                )
+            ]
+
+        return cls.model_validate(data)
+
+
+class FileTraceLog(BaseTraceLog):  # pragma: no cov
+    """Trace Log object that write file to the local storage."""
+
+    @classmethod
+    def find_logs(cls) -> Iterator[TraceData]:  # pragma: no cov
+        for file in sorted(
+            config.log_path.glob("./run_id=*"),
+            key=lambda f: f.lstat().st_mtime,
+        ):
+            yield TraceData.from_path(file)
+
+    @classmethod
+    def find_log_with_id(
+        cls, run_id: str, force_raise: bool = True
+    ) -> TraceData:
+        file: Path = config.log_path / f"run_id={run_id}"
+        if file.exists():
+            return TraceData.from_path(file)
+        elif force_raise:
+            raise FileNotFoundError(
+                f"Trace log on path 'run_id={run_id}' does not found."
+            )
+        return {}
+
+    @property
+    def pointer(self) -> Path:
+        log_file: Path = (
+            config.log_path / f"run_id={self.parent_run_id or self.run_id}"
+        )
+        if not log_file.exists():
+            log_file.mkdir(parents=True)
+        return log_file
+
+    @property
+    def cut_id(self) -> str:
+        """Combine cutting ID of parent running ID if it set.
+
+        :rtype: str
+        """
+        cut_run_id: str = cut_id(self.run_id)
+        if not self.parent_run_id:
+            return f"{cut_run_id} -> {' ' * 6}"
+
+        cut_parent_run_id: str = cut_id(self.parent_run_id)
+        return f"{cut_parent_run_id} -> {cut_run_id}"
+
+    def make_message(self, message: str) -> str:
+        """Prepare and Make a message before write and log processes.
+
+        :param message: A message that want to prepare and make before.
+
+        :rtype: str
+        """
+        return f"({self.cut_id}) {message}"
+
+    def writer(self, message: str, is_err: bool = False) -> None:
+        """ "Write a trace message after making to target file and write metadata
+        in the same path of standard files.
+
+            The path of logging data will store by format:
+
+            ... ./logs/run_id=<run-id>/metadata.json
+            ... ./logs/run_id=<run-id>/stdout.txt
+            ... ./logs/run_id=<run-id>/stderr.txt
+
+        :param message: A message after making.
+        :param is_err: A flag for writing with an error trace or not.
+        """
+        if not config.enable_write_log:
+            return
+
+        frame_info: Traceback = getframeinfo(currentframe().f_back.f_back)
+        filename: str = frame_info.filename.split(os.path.sep)[-1]
+        lineno: int = frame_info.lineno
+
+        # NOTE: set process and thread IDs.
+        process: int = os.getpid()
+        thread: int = get_ident()
+
+        write_file: str = "stderr.txt" if is_err else "stdout.txt"
+        write_data: dict[str, Union[str, int]] = {
+            "datetime": get_dt_tznow().strftime(config.log_datetime_format),
+            "process": process,
+            "thread": thread,
+            "message": message,
+            "filename": filename,
+            "lineno": lineno,
+        }
+
+        with (self.pointer / write_file).open(mode="at", encoding="utf-8") as f:
+            msg_fmt: str = f"{config.log_format_file}\n"
+            f.write(msg_fmt.format(**write_data))
+
+        with (self.pointer / "metadata.json").open(
+            mode="at", encoding="utf-8"
+        ) as f:
+            f.write(
+                json.dumps({"mode": write_file.split(".")[0]} | write_data)
+                + "\n"
+            )
+
+
+class SQLiteTraceLog(BaseTraceLog):  # pragma: no cov
+    """Trace Log object that write trace log to the SQLite database file."""
+
+    table_name: ClassVar[str] = "audits"
+    schemas: ClassVar[
+        str
+    ] = """
+        run_id          int,
+        stdout          str,
+        stderr          str,
+        update          datetime
+        primary key ( run_id )
+        """
+
+    @classmethod
+    def find_logs(cls) -> Iterator[DictStr]: ...
+
+    @classmethod
+    def find_log_with_id(cls, run_id: str) -> DictStr: ...
+
+    def make_message(self, message: str) -> str: ...
+
+    def writer(self, message: str, is_err: bool = False) -> None: ...
+
+
+TraceLog = Union[
+    FileTraceLog,
+    SQLiteTraceLog,
+]
+
+
+def get_trace(
+    run_id: str, parent_run_id: str | None = None
+) -> TraceLog:  # pragma: no cov
+    """Get dynamic TraceLog object from the setting config."""
+    if config.log_path.is_file():
+        return SQLiteTraceLog(run_id, parent_run_id=parent_run_id)
+    return FileTraceLog(run_id, parent_run_id=parent_run_id)
+
+
+def get_trace_obj() -> type[TraceLog]:  # pragma: no cov
+    if config.log_path.is_file():
+        return SQLiteTraceLog
+    return FileTraceLog