ddeutil-workflow 0.0.40__py3-none-any.whl → 0.0.41__py3-none-any.whl

ddeutil/workflow/logs.py

@@ -3,7 +3,7 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
-"""A Logs module contain a TraceLog dataclass.
+"""A Logs module contain TraceLog dataclass and AuditLog model.
 """
 from __future__ import annotations
 
@@ -19,9 +19,10 @@ from typing import ClassVar, Literal, Optional, Union
 
 from pydantic import BaseModel, Field
 from pydantic.dataclasses import dataclass
+from pydantic.functional_validators import model_validator
 from typing_extensions import Self
 
-from .__types import DictStr, TupleStr
+from .__types import DictData, DictStr, TupleStr
 from .conf import config, get_logger
 from .utils import cut_id, get_dt_now
 
@@ -36,6 +37,10 @@ __all__: TupleStr = (
     "get_dt_tznow",
     "get_trace",
     "get_trace_obj",
+    "get_audit",
+    "FileAudit",
+    "SQLiteAudit",
+    "Audit",
 )
 
 
@@ -47,6 +52,66 @@ def get_dt_tznow() -> datetime:  # pragma: no cov
     return get_dt_now(tz=config.tz)
 
 
+class TraceMeda(BaseModel):  # pragma: no cov
+    mode: Literal["stdout", "stderr"]
+    datetime: str
+    process: int
+    thread: int
+    message: str
+    filename: str
+    lineno: int
+
+    @classmethod
+    def make(cls, mode: Literal["stdout", "stderr"], message: str) -> Self:
+        """Make a TraceMeda instance."""
+        frame_info: Traceback = getframeinfo(
+            currentframe().f_back.f_back.f_back
+        )
+        return cls(
+            mode=mode,
+            datetime=get_dt_tznow().strftime(config.log_datetime_format),
+            process=os.getpid(),
+            thread=get_ident(),
+            message=message,
+            filename=frame_info.filename.split(os.path.sep)[-1],
+            lineno=frame_info.lineno,
+        )
+
+
+class TraceData(BaseModel):  # pragma: no cov
+    stdout: str = Field(description="A standard output trace data.")
+    stderr: str = Field(description="A standard error trace data.")
+    meta: list[TraceMeda] = Field(
+        default_factory=list,
+        description=(
+            "A metadata mapping of this output and error before making it to "
+            "standard value."
+        ),
+    )
+
+    @classmethod
+    def from_path(cls, file: Path) -> Self:
+        data: DictStr = {"stdout": "", "stderr": "", "meta": []}
+
+        if (file / "stdout.txt").exists():
+            data["stdout"] = (file / "stdout.txt").read_text(encoding="utf-8")
+
+        if (file / "stderr.txt").exists():
+            data["stderr"] = (file / "stderr.txt").read_text(encoding="utf-8")
+
+        if (file / "metadata.json").exists():
+            data["meta"] = [
+                json.loads(line)
+                for line in (
+                    (file / "metadata.json")
+                    .read_text(encoding="utf-8")
+                    .splitlines()
+                )
+            ]
+
+        return cls.model_validate(data)
+
+
 @dataclass(frozen=True)
 class BaseTraceLog(ABC):  # pragma: no cov
     """Base Trace Log dataclass object."""
@@ -67,6 +132,14 @@ class BaseTraceLog(ABC):  # pragma: no cov
             "Create writer logic for this trace object before using."
         )
 
+    @abstractmethod
+    async def awriter(self, message: str, is_err: bool = False) -> None:
+        """Async Write a trace message after making to target pointer object.
+
+        :param message:
+        :param is_err:
+        """
+
     @abstractmethod
     def make_message(self, message: str) -> str:
         """Prepare and Make a message before write and log processes.
@@ -134,50 +207,6 @@ class BaseTraceLog(ABC):  # pragma: no cov
         logger.exception(msg, stacklevel=2)
 
 
-class TraceMeda(BaseModel):  # pragma: no cov
-    mode: Literal["stdout", "stderr"]
-    datetime: str
-    process: int
-    thread: int
-    message: str
-    filename: str
-    lineno: int
-
-
-class TraceData(BaseModel):  # pragma: no cov
-    stdout: str = Field(description="A standard output trace data.")
-    stderr: str = Field(description="A standard error trace data.")
-    meta: list[TraceMeda] = Field(
-        default_factory=list,
-        description=(
-            "A metadata mapping of this output and error before making it to "
-            "standard value."
-        ),
-    )
-
-    @classmethod
-    def from_path(cls, file: Path) -> Self:
-        data: DictStr = {"stdout": "", "stderr": "", "meta": []}
-
-        if (file / "stdout.txt").exists():
-            data["stdout"] = (file / "stdout.txt").read_text(encoding="utf-8")
-
-        if (file / "stderr.txt").exists():
-            data["stderr"] = (file / "stderr.txt").read_text(encoding="utf-8")
-
-        if (file / "metadata.json").exists():
-            data["meta"] = [
-                json.loads(line)
-                for line in (
-                    (file / "metadata.json")
-                    .read_text(encoding="utf-8")
-                    .splitlines()
-                )
-            ]
-
-        return cls.model_validate(data)
-
-
 class FileTraceLog(BaseTraceLog):  # pragma: no cov
     """Trace Log object that write file to the local storage."""
 
@@ -227,7 +256,7 @@ class FileTraceLog(BaseTraceLog):  # pragma: no cov
     def make_message(self, message: str) -> str:
         """Prepare and Make a message before write and log processes.
 
-        :param message: A message that want to prepare and make before.
+        :param message: (str) A message that want to prepare and make before.
 
         :rtype: str
         """
@@ -249,40 +278,44 @@ class FileTraceLog(BaseTraceLog):  # pragma: no cov
         if not config.enable_write_log:
             return
 
-        frame_info: Traceback = getframeinfo(currentframe().f_back.f_back)
-        filename: str = frame_info.filename.split(os.path.sep)[-1]
-        lineno: int = frame_info.lineno
-
-        # NOTE: set process and thread IDs.
-        process: int = os.getpid()
-        thread: int = get_ident()
-
-        write_file: str = "stderr.txt" if is_err else "stdout.txt"
-        write_data: dict[str, Union[str, int]] = {
-            "datetime": get_dt_tznow().strftime(config.log_datetime_format),
-            "process": process,
-            "thread": thread,
-            "message": message,
-            "filename": filename,
-            "lineno": lineno,
-        }
+        write_file: str = "stderr" if is_err else "stdout"
+        trace_meta: TraceMeda = TraceMeda.make(mode=write_file, message=message)
 
-        with (self.pointer / write_file).open(mode="at", encoding="utf-8") as f:
-            msg_fmt: str = f"{config.log_format_file}\n"
-            f.write(msg_fmt.format(**write_data))
-
-        with (self.pointer / "metadata.json").open(
+        with (self.pointer / f"{write_file}.txt").open(
             mode="at", encoding="utf-8"
         ) as f:
             f.write(
-                json.dumps({"mode": write_file.split(".")[0]} | write_data)
-                + "\n"
+                f"{config.log_format_file}\n".format(**trace_meta.model_dump())
             )
 
+        with (self.pointer / "metadata.json").open(
+            mode="at", encoding="utf-8"
+        ) as f:
+            f.write(trace_meta.model_dump_json() + "\n")
+
     async def awriter(
         self, message: str, is_err: bool = False
-    ):  # pragma: no cov
+    ) -> None:  # pragma: no cov
         """TODO: Use `aiofiles` for make writer method support async."""
+        if not config.enable_write_log:
+            return
+
+        import aiofiles
+
+        write_file: str = "stderr" if is_err else "stdout"
+        trace_meta: TraceMeda = TraceMeda.make(mode=write_file, message=message)
+
+        async with aiofiles.open(
+            self.pointer / f"{write_file}.txt", mode="at", encoding="utf-8"
+        ) as f:
+            await f.write(
+                f"{config.log_format_file}\n".format(**trace_meta.model_dump())
+            )
+
+        async with aiofiles.open(
+            self.pointer / "metadata.json", mode="at", encoding="utf-8"
+        ) as f:
+            await f.write(trace_meta.model_dump_json() + "\n")
 
 
 class SQLiteTraceLog(BaseTraceLog):  # pragma: no cov
@@ -309,6 +342,8 @@ class SQLiteTraceLog(BaseTraceLog):  # pragma: no cov
 
     def writer(self, message: str, is_err: bool = False) -> None: ...
 
+    def awriter(self, message: str, is_err: bool = False) -> None: ...
+
 
 TraceLog = Union[
     FileTraceLog,
@@ -329,3 +364,230 @@ def get_trace_obj() -> type[TraceLog]:  # pragma: no cov
     if config.log_path.is_file():
         return SQLiteTraceLog
     return FileTraceLog
+
+
+class BaseAudit(BaseModel, ABC):
+    """Base Audit Pydantic Model with abstraction class property that implement
+    only model fields. This model should to use with inherit to logging
+    subclass like file, sqlite, etc.
+    """
+
+    name: str = Field(description="A workflow name.")
+    release: datetime = Field(description="A release datetime.")
+    type: str = Field(description="A running type before logging.")
+    context: DictData = Field(
+        default_factory=dict,
+        description="A context that receive from a workflow execution result.",
+    )
+    parent_run_id: Optional[str] = Field(
+        default=None, description="A parent running ID."
+    )
+    run_id: str = Field(description="A running ID")
+    update: datetime = Field(default_factory=get_dt_tznow)
+    execution_time: float = Field(default=0, description="An execution time.")
+
+    @model_validator(mode="after")
+    def __model_action(self) -> Self:
+        """Do before the Audit action with WORKFLOW_AUDIT_ENABLE_WRITE env variable.
+
+        :rtype: Self
+        """
+        if config.enable_write_audit:
+            self.do_before()
+        return self
+
+    def do_before(self) -> None:  # pragma: no cov
+        """To something before end up of initial log model."""
+
+    @abstractmethod
+    def save(self, excluded: list[str] | None) -> None:  # pragma: no cov
+        """Save this model logging to target logging store."""
+        raise NotImplementedError("Audit should implement ``save`` method.")
+
+
+class FileAudit(BaseAudit):
+    """File Audit Pydantic Model that use to saving log data from result of
+    workflow execution. It inherits from BaseAudit model that implement the
+    ``self.save`` method for file.
+    """
+
+    filename_fmt: ClassVar[str] = (
+        "workflow={name}/release={release:%Y%m%d%H%M%S}"
+    )
+
+    def do_before(self) -> None:
+        """Create directory of release before saving log file."""
+        self.pointer().mkdir(parents=True, exist_ok=True)
+
+    @classmethod
+    def find_audits(cls, name: str) -> Iterator[Self]:
+        """Generate the audit data that found from logs path with specific a
+        workflow name.
+
+        :param name: A workflow name that want to search release logging data.
+
+        :rtype: Iterator[Self]
+        """
+        pointer: Path = config.audit_path / f"workflow={name}"
+        if not pointer.exists():
+            raise FileNotFoundError(f"Pointer: {pointer.absolute()}.")
+
+        for file in pointer.glob("./release=*/*.log"):
+            with file.open(mode="r", encoding="utf-8") as f:
+                yield cls.model_validate(obj=json.load(f))
+
+    @classmethod
+    def find_audit_with_release(
+        cls,
+        name: str,
+        release: datetime | None = None,
+    ) -> Self:
+        """Return the audit data that found from logs path with specific
+        workflow name and release values. If a release does not pass to an input
+        argument, it will return the latest release from the current log path.
+
+        :param name: A workflow name that want to search log.
+        :param release: A release datetime that want to search log.
+
+        :raise FileNotFoundError:
+        :raise NotImplementedError: If an input release does not pass to this
+            method. Because this method does not implement latest log.
+
+        :rtype: Self
+        """
+        if release is None:
+            raise NotImplementedError("Find latest log does not implement yet.")
+
+        pointer: Path = (
+            config.audit_path
+            / f"workflow={name}/release={release:%Y%m%d%H%M%S}"
+        )
+        if not pointer.exists():
+            raise FileNotFoundError(
+                f"Pointer: ./logs/workflow={name}/"
+                f"release={release:%Y%m%d%H%M%S} does not found."
+            )
+
+        with max(pointer.glob("./*.log"), key=os.path.getctime).open(
+            mode="r", encoding="utf-8"
+        ) as f:
+            return cls.model_validate(obj=json.load(f))
+
+    @classmethod
+    def is_pointed(cls, name: str, release: datetime) -> bool:
+        """Check the release log already pointed or created at the destination
+        log path.
+
+        :param name: A workflow name.
+        :param release: A release datetime.
+
+        :rtype: bool
+        :return: Return False if the release log was not pointed or created.
+        """
+        # NOTE: Return False if enable writing log flag does not set.
+        if not config.enable_write_audit:
+            return False
+
+        # NOTE: create pointer path that use the same logic of pointer method.
+        pointer: Path = config.audit_path / cls.filename_fmt.format(
+            name=name, release=release
+        )
+
+        return pointer.exists()
+
+    def pointer(self) -> Path:
+        """Return release directory path that was generated from model data.
+
+        :rtype: Path
+        """
+        return config.audit_path / self.filename_fmt.format(
+            name=self.name, release=self.release
+        )
+
+    def save(self, excluded: list[str] | None) -> Self:
+        """Save logging data that receive a context data from a workflow
+        execution result.
+
+        :param excluded: An excluded list of key name that want to pass in the
+            model_dump method.
+
+        :rtype: Self
+        """
+        trace: TraceLog = get_trace(self.run_id, self.parent_run_id)
+
+        # NOTE: Check environ variable was set for real writing.
+        if not config.enable_write_audit:
+            trace.debug("[LOG]: Skip writing log cause config was set")
+            return self
+
+        log_file: Path = (
+            self.pointer() / f"{self.parent_run_id or self.run_id}.log"
+        )
+        log_file.write_text(
+            json.dumps(
+                self.model_dump(exclude=excluded),
+                default=str,
+                indent=2,
+            ),
+            encoding="utf-8",
+        )
+        return self
+
+
+class SQLiteAudit(BaseAudit):  # pragma: no cov
+    """SQLite Audit Pydantic Model."""
+
+    table_name: ClassVar[str] = "audits"
+    schemas: ClassVar[
+        str
+    ] = """
+        workflow        str,
+        release         int,
+        type            str,
+        context         json,
+        parent_run_id   int,
+        run_id          int,
+        update          datetime
+        primary key ( run_id )
+        """
+
+    def save(self, excluded: list[str] | None) -> SQLiteAudit:
+        """Save logging data that receive a context data from a workflow
+        execution result.
+        """
+        trace: TraceLog = get_trace(self.run_id, self.parent_run_id)
+
+        # NOTE: Check environ variable was set for real writing.
+        if not config.enable_write_audit:
+            trace.debug("[LOG]: Skip writing log cause config was set")
+            return self
+
+        raise NotImplementedError("SQLiteAudit does not implement yet.")
+
+
+class RemoteFileAudit(FileAudit):  # pragma: no cov
+    """Remote File Audit Pydantic Model."""
+
+    def save(self, excluded: list[str] | None) -> RemoteFileAudit: ...
+
+
+class RedisAudit(BaseAudit):  # pragma: no cov
+    """Redis Audit Pydantic Model."""
+
+    def save(self, excluded: list[str] | None) -> RedisAudit: ...
+
+
+Audit = Union[
+    FileAudit,
+    SQLiteAudit,
+]
+
+
+def get_audit() -> type[Audit]:  # pragma: no cov
+    """Get an audit class that dynamic base on the config audit path value.
+
+    :rtype: type[Audit]
+    """
+    if config.audit_path.is_file():
+        return SQLiteAudit
+    return FileAudit

ddeutil/workflow/params.py

@@ -3,6 +3,7 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
+# [ ] Use config
 """This module include all Param Pydantic Models that use for parsing an
 incoming parameters that was passed to the Workflow and Schedule objects before
 execution or release methods.

ddeutil/workflow/result.py

@@ -11,7 +11,6 @@ from __future__ import annotations
 from dataclasses import field
 from datetime import datetime
 from enum import IntEnum
-from threading import Event
 from typing import Optional
 
 from pydantic import ConfigDict
@@ -21,30 +20,21 @@ from typing_extensions import Self
 
 from .__types import DictData, TupleStr
 from .logs import TraceLog, get_dt_tznow, get_trace
-from .utils import gen_id
+from .utils import default_gen_id, gen_id
 
 __all__: TupleStr = (
     "Result",
     "Status",
-    "default_gen_id",
 )
 
 
-def default_gen_id() -> str:
-    """Return running ID which use for making default ID for the Result model if
-    a run_id field initializes at the first time.
-
-    :rtype: str
-    """
-    return gen_id("manual", unique=True)
-
-
 class Status(IntEnum):
     """Status Int Enum object."""
 
     SUCCESS: int = 0
     FAILED: int = 1
     WAIT: int = 2
+    SKIP: int = 3
 
 
 @dataclass(
@@ -65,7 +55,6 @@ class Result:
     parent_run_id: Optional[str] = field(default=None, compare=False)
     ts: datetime = field(default_factory=get_dt_tznow, compare=False)
 
-    event: Event = field(default_factory=Event, compare=False, repr=False)
     trace: Optional[TraceLog] = field(default=None, compare=False, repr=False)
 
     @classmethod
@@ -90,10 +79,12 @@ class Result:
 
     @model_validator(mode="after")
     def __prepare_trace(self) -> Self:
-        """Prepare trace field that want to pass after its initialize step."""
+        """Prepare trace field that want to pass after its initialize step.
+
+        :rtype: Self
+        """
         if self.trace is None:  # pragma: no cov
             self.trace: TraceLog = get_trace(self.run_id, self.parent_run_id)
-
         return self
 
     def set_parent_run_id(self, running_id: str) -> Self: