ddeutil-workflow 0.0.32__py3-none-any.whl → 0.0.34__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1 +1 @@
-__version__: str = "0.0.32"
+__version__: str = "0.0.34"

ddeutil/workflow/__init__.py

@@ -5,13 +5,22 @@
 # ------------------------------------------------------------------------------
 from .__cron import CronJob, CronRunner
 from .__types import Re
+from .audit import (
+    Audit,
+    get_audit,
+)
+from .call import (
+    ReturnTagFunc,
+    TagFunc,
+    extract_call,
+    make_registry,
+    tag,
+)
 from .conf import (
     Config,
     Loader,
-    Log,
     config,
     env,
-    get_log,
     get_logger,
 )
 from .cron import (
@@ -26,13 +35,6 @@ from .exceptions import (
     UtilException,
     WorkflowException,
 )
-from .hook import (
-    ReturnTagFunc,
-    TagFunc,
-    extract_hook,
-    make_registry,
-    tag,
-)
 from .job import (
     Job,
     Strategy,
@@ -44,7 +46,13 @@ from .params import (
     Param,
     StrParam,
 )
-from .result import Result
+from .result import (
+    Result,
+    Status,
+    TraceLog,
+    default_gen_id,
+    get_dt_tznow,
+)
 from .scheduler import (
     Schedule,
     ScheduleWorkflow,
@@ -52,10 +60,10 @@ from .scheduler import (
     schedule_runner,
     schedule_task,
 )
-from .stage import (
+from .stages import (
     BashStage,
+    CallStage,
     EmptyStage,
-    HookStage,
     PyStage,
     Stage,
     TriggerStage,

ddeutil/workflow/api/api.py

@@ -86,7 +86,7 @@ if config.enable_route_workflow:
 
 # NOTE: Enable the schedule route.
 if config.enable_route_schedule:
-    from ..conf import get_log
+    from ..audit import get_audit
     from ..scheduler import schedule_task
     from .route import schedule_route
 
@@ -106,7 +106,7 @@ if config.enable_route_schedule:
                 stop=datetime.now(config.tz) + timedelta(minutes=1),
                 queue=app.state.workflow_queue,
                 threads=app.state.workflow_threads,
-                log=get_log(),
+                log=get_audit(),
             )
 
     @schedule_route.on_event("startup")

ddeutil/workflow/api/route.py

@@ -16,7 +16,8 @@ from fastapi.responses import UJSONResponse
 from pydantic import BaseModel
 
 from ..__types import DictData
-from ..conf import FileLog, Loader, config, get_logger
+from ..audit import Audit, get_audit
+from ..conf import Loader, config, get_logger
 from ..result import Result
 from ..scheduler import Schedule
 from ..workflow import Workflow
@@ -109,7 +110,7 @@ async def get_workflow_logs(name: str):
                     exclude_unset=True,
                     exclude_defaults=True,
                 )
-                for log in FileLog.find_logs(name=name)
+                for log in get_audit().find_audits(name=name)
             ],
         }
     except FileNotFoundError:
@@ -122,7 +123,7 @@ async def get_workflow_logs(name: str):
 @workflow_route.get(path="/{name}/logs/{release}")
 async def get_workflow_release_log(name: str, release: str):
     try:
-        log: FileLog = FileLog.find_log_with_release(
+        log: Audit = get_audit().find_audit_with_release(
             name=name, release=datetime.strptime(release, "%Y%m%d%H%M%S")
         )
     except FileNotFoundError:

ddeutil/workflow/audit.py

@@ -0,0 +1,252 @@
+# ------------------------------------------------------------------------------
+# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
+# Licensed under the MIT License. See LICENSE in the project root for
+# license information.
+# ------------------------------------------------------------------------------
+"""Audit Log module."""
+from __future__ import annotations
+
+import json
+import os
+from abc import ABC, abstractmethod
+from collections.abc import Iterator
+from datetime import datetime
+from pathlib import Path
+from typing import ClassVar, Optional, Union
+
+from pydantic import BaseModel, Field
+from pydantic.functional_validators import model_validator
+from typing_extensions import Self
+
+from .__types import DictData, TupleStr
+from .conf import config
+from .result import TraceLog
+
+__all__: TupleStr = (
+    "get_audit",
+    "FileAudit",
+    "SQLiteAudit",
+    "Audit",
+)
+
+
+class BaseAudit(BaseModel, ABC):
+    """Base Audit Pydantic Model with abstraction class property that implement
+    only model fields. This model should to use with inherit to logging
+    subclass like file, sqlite, etc.
+    """
+
+    name: str = Field(description="A workflow name.")
+    release: datetime = Field(description="A release datetime.")
+    type: str = Field(description="A running type before logging.")
+    context: DictData = Field(
+        default_factory=dict,
+        description="A context that receive from a workflow execution result.",
+    )
+    parent_run_id: Optional[str] = Field(default=None)
+    run_id: str
+    update: datetime = Field(default_factory=datetime.now)
+    execution_time: float = Field(default=0)
+
+    @model_validator(mode="after")
+    def __model_action(self) -> Self:
+        """Do before the Audit action with WORKFLOW_AUDIT_ENABLE_WRITE env variable.
+
+        :rtype: Self
+        """
+        if config.enable_write_audit:
+            self.do_before()
+        return self
+
+    def do_before(self) -> None:  # pragma: no cov
+        """To something before end up of initial log model."""
+
+    @abstractmethod
+    def save(self, excluded: list[str] | None) -> None:  # pragma: no cov
+        """Save this model logging to target logging store."""
+        raise NotImplementedError("Audit should implement ``save`` method.")
+
+
+class FileAudit(BaseAudit):
+    """File Audit Pydantic Model that use to saving log data from result of
+    workflow execution. It inherits from BaseAudit model that implement the
+    ``self.save`` method for file.
+    """
+
+    filename_fmt: ClassVar[str] = (
+        "workflow={name}/release={release:%Y%m%d%H%M%S}"
+    )
+
+    def do_before(self) -> None:
+        """Create directory of release before saving log file."""
+        self.pointer().mkdir(parents=True, exist_ok=True)
+
+    @classmethod
+    def find_audits(cls, name: str) -> Iterator[Self]:
+        """Generate the audit data that found from logs path with specific a
+        workflow name.
+
+        :param name: A workflow name that want to search release logging data.
+
+        :rtype: Iterator[Self]
+        """
+        pointer: Path = config.audit_path / f"workflow={name}"
+        if not pointer.exists():
+            raise FileNotFoundError(f"Pointer: {pointer.absolute()}.")
+
+        for file in pointer.glob("./release=*/*.log"):
+            with file.open(mode="r", encoding="utf-8") as f:
+                yield cls.model_validate(obj=json.load(f))
+
+    @classmethod
+    def find_audit_with_release(
+        cls,
+        name: str,
+        release: datetime | None = None,
+    ) -> Self:
+        """Return the audit data that found from logs path with specific
+        workflow name and release values. If a release does not pass to an input
+        argument, it will return the latest release from the current log path.
+
+        :param name: A workflow name that want to search log.
+        :param release: A release datetime that want to search log.
+
+        :raise FileNotFoundError:
+        :raise NotImplementedError:
+
+        :rtype: Self
+        """
+        if release is None:
+            raise NotImplementedError("Find latest log does not implement yet.")
+
+        pointer: Path = (
+            config.audit_path
+            / f"workflow={name}/release={release:%Y%m%d%H%M%S}"
+        )
+        if not pointer.exists():
+            raise FileNotFoundError(
+                f"Pointer: ./logs/workflow={name}/"
+                f"release={release:%Y%m%d%H%M%S} does not found."
+            )
+
+        with max(pointer.glob("./*.log"), key=os.path.getctime).open(
+            mode="r", encoding="utf-8"
+        ) as f:
+            return cls.model_validate(obj=json.load(f))
+
+    @classmethod
+    def is_pointed(cls, name: str, release: datetime) -> bool:
+        """Check the release log already pointed or created at the destination
+        log path.
+
+        :param name: A workflow name.
+        :param release: A release datetime.
+
+        :rtype: bool
+        :return: Return False if the release log was not pointed or created.
+        """
+        # NOTE: Return False if enable writing log flag does not set.
+        if not config.enable_write_audit:
+            return False
+
+        # NOTE: create pointer path that use the same logic of pointer method.
+        pointer: Path = config.audit_path / cls.filename_fmt.format(
+            name=name, release=release
+        )
+
+        return pointer.exists()
+
+    def pointer(self) -> Path:
+        """Return release directory path that was generated from model data.
+
+        :rtype: Path
+        """
+        return config.audit_path / self.filename_fmt.format(
+            name=self.name, release=self.release
+        )
+
+    def save(self, excluded: list[str] | None) -> Self:
+        """Save logging data that receive a context data from a workflow
+        execution result.
+
+        :param excluded: An excluded list of key name that want to pass in the
+            model_dump method.
+
+        :rtype: Self
+        """
+        trace: TraceLog = TraceLog(self.run_id, self.parent_run_id)
+
+        # NOTE: Check environ variable was set for real writing.
+        if not config.enable_write_audit:
+            trace.debug("[LOG]: Skip writing log cause config was set")
+            return self
+
+        log_file: Path = self.pointer() / f"{self.run_id}.log"
+        log_file.write_text(
+            json.dumps(
+                self.model_dump(exclude=excluded),
+                default=str,
+                indent=2,
+            ),
+            encoding="utf-8",
+        )
+        return self
+
+
+class SQLiteAudit(BaseAudit):  # pragma: no cov
+    """SQLite Audit Pydantic Model."""
+
+    table_name: ClassVar[str] = "workflow_log"
+    schemas: ClassVar[
+        str
+    ] = """
+        workflow        str,
+        release         int,
+        type            str,
+        context         json,
+        parent_run_id   int,
+        run_id          int,
+        update          datetime
+        primary key ( run_id )
+        """
+
+    def save(self, excluded: list[str] | None) -> SQLiteAudit:
+        """Save logging data that receive a context data from a workflow
+        execution result.
+        """
+        trace: TraceLog = TraceLog(self.run_id, self.parent_run_id)
+
+        # NOTE: Check environ variable was set for real writing.
+        if not config.enable_write_audit:
+            trace.debug("[LOG]: Skip writing log cause config was set")
+            return self
+
+        raise NotImplementedError("SQLiteAudit does not implement yet.")
+
+
+class RemoteFileAudit(FileAudit):  # pragma: no cov
+    """Remote File Audit Pydantic Model."""
+
+    def save(self, excluded: list[str] | None) -> RemoteFileAudit: ...
+
+
+class RedisAudit(BaseAudit):  # pragma: no cov
+    """Redis Audit Pydantic Model."""
+
+    def save(self, excluded: list[str] | None) -> RedisAudit: ...
+
+
+Audit = Union[
+    FileAudit,
+    SQLiteAudit,
+]
+
+
+def get_audit() -> type[Audit]:  # pragma: no cov
+    """Get an audit class that dynamic base on the config audit path value.
+
+    :rtype: type[Audit]
+    """
+    if config.audit_path.is_file():
+        return SQLiteAudit
+    return FileAudit

ddeutil/workflow/{hook.py → call.py}

@@ -60,7 +60,7 @@ def tag(
 
         @wraps(func)
         def wrapped(*args: P.args, **kwargs: P.kwargs) -> TagFunc:
-            # NOTE: Able to do anything before calling hook function.
+            # NOTE: Able to do anything before calling call function.
             return func(*args, **kwargs)
 
         return wrapped
@@ -79,9 +79,9 @@ def make_registry(submodule: str) -> dict[str, Registry]:
     :rtype: dict[str, Registry]
     """
     rs: dict[str, Registry] = {}
-    regis_hooks: list[str] = config.regis_hook
-    regis_hooks.extend(["ddeutil.vendors"])
-    for module in regis_hooks:
+    regis_calls: list[str] = config.regis_call
+    regis_calls.extend(["ddeutil.vendors"])
+    for module in regis_calls:
         # NOTE: try to sequential import task functions
         try:
             importer = import_module(f"{module}.{submodule}")
@@ -114,9 +114,9 @@ def make_registry(submodule: str) -> dict[str, Registry]:
 
 
 @dataclass(frozen=True)
-class HookSearchData:
-    """Hook Search dataclass that use for receive regular expression grouping
-    dict from searching hook string value.
+class CallSearchData:
+    """Call Search dataclass that use for receive regular expression grouping
+    dict from searching call string value.
     """
 
     path: str
@@ -124,49 +124,49 @@ class HookSearchData:
     tag: str
 
 
-def extract_hook(hook: str) -> Callable[[], TagFunc]:
-    """Extract Hook function from string value to hook partial function that
+def extract_call(call: str) -> Callable[[], TagFunc]:
+    """Extract Call function from string value to call partial function that
     does run it at runtime.
 
-    :raise NotImplementedError: When the searching hook's function result does
+    :raise NotImplementedError: When the searching call's function result does
         not exist in the registry.
-    :raise NotImplementedError: When the searching hook's tag result does not
+    :raise NotImplementedError: When the searching call's tag result does not
         exist in the registry with its function key.
 
-    :param hook: A hook value that able to match with Task regex.
+    :param call: A call value that able to match with Task regex.
 
-        The format of hook value should contain 3 regular expression groups
+        The format of call value should contain 3 regular expression groups
     which match with the below config format:
 
         >>> "^(?P<path>[^/@]+)/(?P<func>[^@]+)@(?P<tag>.+)$"
 
     Examples:
-        >>> extract_hook("tasks/el-postgres-to-delta@polars")
+        >>> extract_call("tasks/el-postgres-to-delta@polars")
         ...
-        >>> extract_hook("tasks/return-type-not-valid@raise")
+        >>> extract_call("tasks/return-type-not-valid@raise")
         ...
 
     :rtype: Callable[[], TagFunc]
     """
-    if not (found := Re.RE_TASK_FMT.search(hook)):
+    if not (found := Re.RE_TASK_FMT.search(call)):
         raise ValueError(
-            f"Hook {hook!r} does not match with hook format regex."
+            f"Call {call!r} does not match with call format regex."
         )
 
-    # NOTE: Pass the searching hook string to `path`, `func`, and `tag`.
-    hook: HookSearchData = HookSearchData(**found.groupdict())
+    # NOTE: Pass the searching call string to `path`, `func`, and `tag`.
+    call: CallSearchData = CallSearchData(**found.groupdict())
 
     # NOTE: Registry object should implement on this package only.
-    rgt: dict[str, Registry] = make_registry(f"{hook.path}")
-    if hook.func not in rgt:
+    rgt: dict[str, Registry] = make_registry(f"{call.path}")
+    if call.func not in rgt:
         raise NotImplementedError(
-            f"``REGISTER-MODULES.{hook.path}.registries`` does not "
-            f"implement registry: {hook.func!r}."
+            f"``REGISTER-MODULES.{call.path}.registries`` does not "
+            f"implement registry: {call.func!r}."
         )
 
-    if hook.tag not in rgt[hook.func]:
+    if call.tag not in rgt[call.func]:
         raise NotImplementedError(
-            f"tag: {hook.tag!r} does not found on registry func: "
-            f"``REGISTER-MODULES.{hook.path}.registries.{hook.func}``"
+            f"tag: {call.tag!r} does not found on registry func: "
+            f"``REGISTER-MODULES.{call.path}.registries.{call.func}``"
         )
-    return rgt[hook.func][hook.tag]
+    return rgt[call.func][call.tag]