ddeutil-workflow 0.0.31__py3-none-any.whl → 0.0.33__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1 +1 @@
-__version__: str = "0.0.31"
+__version__: str = "0.0.33"

ddeutil/workflow/__init__.py

@@ -5,13 +5,15 @@
 # ------------------------------------------------------------------------------
 from .__cron import CronJob, CronRunner
 from .__types import Re
+from .audit import (
+    Audit,
+    get_audit,
+)
 from .conf import (
     Config,
     Loader,
-    Log,
     config,
     env,
-    get_log,
     get_logger,
 )
 from .cron import (

ddeutil/workflow/api/api.py

@@ -43,9 +43,7 @@ async def lifespan(a: FastAPI) -> AsyncIterator[State]:
     a.state.workflow_queue = {}
 
     yield {
-        "upper_queue": a.state.upper_queue,
-        "upper_result": a.state.upper_result,
-        # NOTE: Scheduler value should be contain a key of workflow workflow and
+        # NOTE: Scheduler value should be contained a key of workflow and
         #   list of datetime of queue and running.
         #
         #   ... {
@@ -88,7 +86,7 @@ if config.enable_route_workflow:
 
 # NOTE: Enable the schedule route.
 if config.enable_route_schedule:
-    from ..conf import FileLog
+    from ..audit import get_audit
     from ..scheduler import schedule_task
     from .route import schedule_route
 
@@ -108,11 +106,11 @@ if config.enable_route_schedule:
                 stop=datetime.now(config.tz) + timedelta(minutes=1),
                 queue=app.state.workflow_queue,
                 threads=app.state.workflow_threads,
-                log=FileLog,
+                log=get_audit(),
             )
 
     @schedule_route.on_event("startup")
-    @repeat_at(cron="*/5 * * * *")
+    @repeat_at(cron="*/5 * * * *", delay=10)
     def monitoring():
         logger.debug("[MONITOR]: Start monitoring threading.")
         snapshot_threads: list[str] = list(app.state.workflow_threads.keys())

ddeutil/workflow/api/route.py

@@ -16,7 +16,8 @@ from fastapi.responses import UJSONResponse
 from pydantic import BaseModel
 
 from ..__types import DictData
-from ..conf import FileLog, Loader, config, get_logger
+from ..audit import Audit, get_audit
+from ..conf import Loader, config, get_logger
 from ..result import Result
 from ..scheduler import Schedule
 from ..workflow import Workflow
@@ -109,7 +110,7 @@ async def get_workflow_logs(name: str):
                     exclude_unset=True,
                     exclude_defaults=True,
                 )
-                for log in FileLog.find_logs(name=name)
+                for log in get_audit().find_logs(name=name)
             ],
         }
     except FileNotFoundError:
@@ -122,7 +123,7 @@ async def get_workflow_logs(name: str):
 @workflow_route.get(path="/{name}/logs/{release}")
 async def get_workflow_release_log(name: str, release: str):
     try:
-        log: FileLog = FileLog.find_log_with_release(
+        log: Audit = get_audit().find_log_with_release(
             name=name, release=datetime.strptime(release, "%Y%m%d%H%M%S")
         )
     except FileNotFoundError:
@@ -169,7 +170,7 @@ async def get_schedules(name: str):
     )
 
 
-@schedule_route.get(path="/deploy")
+@schedule_route.get(path="/deploy/")
 async def get_deploy_schedulers(request: Request):
     snapshot = copy.deepcopy(request.state.scheduler)
     return {"schedule": snapshot}
@@ -178,9 +179,9 @@ async def get_deploy_schedulers(request: Request):
 @schedule_route.get(path="/deploy/{name}")
 async def get_deploy_scheduler(request: Request, name: str):
     if name in request.state.scheduler:
-        sch = Schedule.from_loader(name)
+        schedule = Schedule.from_loader(name)
         getter: list[dict[str, dict[str, list[datetime]]]] = []
-        for workflow in sch.workflows:
+        for workflow in schedule.workflows:
             getter.append(
                 {
                     workflow.name: {
@@ -219,7 +220,7 @@ async def add_deploy_scheduler(request: Request, name: str):
         second=0, microsecond=0
     )
 
-    # NOTE: Create pair of workflow and on from schedule model.
+    # NOTE: Create a pair of workflow and on from schedule model.
     try:
         schedule: Schedule = Schedule.from_loader(name)
     except ValueError as err:

ddeutil/workflow/audit.py

@@ -0,0 +1,261 @@
+# ------------------------------------------------------------------------------
+# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
+# Licensed under the MIT License. See LICENSE in the project root for
+# license information.
+# ------------------------------------------------------------------------------
+"""Audit Log module."""
+from __future__ import annotations
+
+import json
+import os
+from abc import ABC, abstractmethod
+from collections.abc import Iterator
+from datetime import datetime
+from pathlib import Path
+from typing import Any, ClassVar, Optional, Union
+
+from pydantic import BaseModel, Field
+from pydantic.functional_validators import model_validator
+from typing_extensions import Self
+
+from .__types import DictData, TupleStr
+from .conf import config, get_logger
+
+logger = get_logger("ddeutil.workflow")
+
+__all__: TupleStr = (
+    "get_audit",
+    "FileAudit",
+    "SQLiteAudit",
+    "Audit",
+)
+
+
+class BaseAudit(BaseModel, ABC):
+    """Base Audit Pydantic Model with abstraction class property that implement
+    only model fields. This model should to use with inherit to logging
+    subclass like file, sqlite, etc.
+    """
+
+    name: str = Field(description="A workflow name.")
+    release: datetime = Field(description="A release datetime.")
+    type: str = Field(description="A running type before logging.")
+    context: DictData = Field(
+        default_factory=dict,
+        description="A context that receive from a workflow execution result.",
+    )
+    parent_run_id: Optional[str] = Field(default=None)
+    run_id: str
+    update: datetime = Field(default_factory=datetime.now)
+    execution_time: float = Field(default=0)
+
+    @model_validator(mode="after")
+    def __model_action(self) -> Self:
+        """Do before the Audit action with WORKFLOW_AUDIT_ENABLE_WRITE env variable.
+
+        :rtype: Self
+        """
+        if config.enable_write_log:
+            self.do_before()
+        return self
+
+    def do_before(self) -> None:  # pragma: no cov
+        """To something before end up of initial log model."""
+
+    @abstractmethod
+    def save(self, excluded: list[str] | None) -> None:  # pragma: no cov
+        """Save this model logging to target logging store."""
+        raise NotImplementedError("Audit should implement ``save`` method.")
+
+
+class FileAudit(BaseAudit):
+    """File Audit Pydantic Model that use to saving log data from result of
+    workflow execution. It inherits from BaseAudit model that implement the
+    ``self.save`` method for file.
+    """
+
+    filename_fmt: ClassVar[str] = (
+        "workflow={name}/release={release:%Y%m%d%H%M%S}"
+    )
+
+    def do_before(self) -> None:
+        """Create directory of release before saving log file."""
+        self.pointer().mkdir(parents=True, exist_ok=True)
+
+    @classmethod
+    def find_logs(cls, name: str) -> Iterator[Self]:
+        """Generate the logging data that found from logs path with specific a
+        workflow name.
+
+        :param name: A workflow name that want to search release logging data.
+
+        :rtype: Iterator[Self]
+        """
+        pointer: Path = config.audit_path / f"workflow={name}"
+        if not pointer.exists():
+            raise FileNotFoundError(f"Pointer: {pointer.absolute()}.")
+
+        for file in pointer.glob("./release=*/*.log"):
+            with file.open(mode="r", encoding="utf-8") as f:
+                yield cls.model_validate(obj=json.load(f))
+
+    @classmethod
+    def find_log_with_release(
+        cls,
+        name: str,
+        release: datetime | None = None,
+    ) -> Self:
+        """Return the logging data that found from logs path with specific
+        workflow name and release values. If a release does not pass to an input
+        argument, it will return the latest release from the current log path.
+
+        :param name: A workflow name that want to search log.
+        :param release: A release datetime that want to search log.
+
+        :raise FileNotFoundError:
+        :raise NotImplementedError:
+
+        :rtype: Self
+        """
+        if release is None:
+            raise NotImplementedError("Find latest log does not implement yet.")
+
+        pointer: Path = (
+            config.audit_path
+            / f"workflow={name}/release={release:%Y%m%d%H%M%S}"
+        )
+        if not pointer.exists():
+            raise FileNotFoundError(
+                f"Pointer: ./logs/workflow={name}/"
+                f"release={release:%Y%m%d%H%M%S} does not found."
+            )
+
+        with max(pointer.glob("./*.log"), key=os.path.getctime).open(
+            mode="r", encoding="utf-8"
+        ) as f:
+            return cls.model_validate(obj=json.load(f))
+
+    @classmethod
+    def is_pointed(cls, name: str, release: datetime) -> bool:
+        """Check the release log already pointed or created at the destination
+        log path.
+
+        :param name: A workflow name.
+        :param release: A release datetime.
+
+        :rtype: bool
+        :return: Return False if the release log was not pointed or created.
+        """
+        # NOTE: Return False if enable writing log flag does not set.
+        if not config.enable_write_log:
+            return False
+
+        # NOTE: create pointer path that use the same logic of pointer method.
+        pointer: Path = config.audit_path / cls.filename_fmt.format(
+            name=name, release=release
+        )
+
+        return pointer.exists()
+
+    def pointer(self) -> Path:
+        """Return release directory path that was generated from model data.
+
+        :rtype: Path
+        """
+        return config.audit_path / self.filename_fmt.format(
+            name=self.name, release=self.release
+        )
+
+    def save(self, excluded: list[str] | None) -> Self:
+        """Save logging data that receive a context data from a workflow
+        execution result.
+
+        :param excluded: An excluded list of key name that want to pass in the
+            model_dump method.
+
+        :rtype: Self
+        """
+        from .utils import cut_id
+
+        # NOTE: Check environ variable was set for real writing.
+        if not config.enable_write_log:
+            logger.debug(
+                f"({cut_id(self.run_id)}) [LOG]: Skip writing log cause "
+                f"config was set"
+            )
+            return self
+
+        log_file: Path = self.pointer() / f"{self.run_id}.log"
+        log_file.write_text(
+            json.dumps(
+                self.model_dump(exclude=excluded),
+                default=str,
+                indent=2,
+            ),
+            encoding="utf-8",
+        )
+        return self
+
+
+class SQLiteAudit(BaseAudit):  # pragma: no cov
+    """SQLite Audit Pydantic Model."""
+
+    @staticmethod
+    def meta() -> dict[str, Any]:
+        return {
+            "table": "workflow_log",
+            "ddl": """
+                workflow        str,
+                release         int,
+                type            str,
+                context         json,
+                parent_run_id   int,
+                run_id          int,
+                update          datetime
+                primary key ( run_id )
+                """,
+        }
+
+    def save(self, excluded: list[str] | None) -> SQLiteAudit:
+        """Save logging data that receive a context data from a workflow
+        execution result.
+        """
+        from .utils import cut_id
+
+        # NOTE: Check environ variable was set for real writing.
+        if not config.enable_write_log:
+            logger.debug(
+                f"({cut_id(self.run_id)}) [LOG]: Skip writing log cause "
+                f"config was set"
+            )
+            return self
+
+        raise NotImplementedError("SQLiteAudit does not implement yet.")
+
+
+class RemoteFileAudit(FileAudit):  # pragma: no cov
+    """Remote File Audit Pydantic Model."""
+
+    def save(self, excluded: list[str] | None) -> RemoteFileAudit: ...
+
+
+class RedisAudit(BaseAudit):  # pragma: no cov
+    """Redis Audit Pydantic Model."""
+
+    def save(self, excluded: list[str] | None) -> RedisAudit: ...
+
+
+Audit = Union[
+    FileAudit,
+    SQLiteAudit,
+]
+
+
+def get_audit() -> type[Audit]:  # pragma: no cov
+    """Get an audit class that dynamic base on the config audit path value.
+
+    :rtype: type[Audit]
+    """
+    if config.audit_path.is_file():
+        return SQLiteAudit
+    return FileAudit