ddeutil-workflow 0.0.68__py3-none-any.whl → 0.0.69__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1 +1 @@
-__version__: str = "0.0.68"
+__version__: str = "0.0.69"

ddeutil/workflow/__init__.py

@@ -5,23 +5,16 @@
 # ------------------------------------------------------------------------------
 from .__cron import CronJob, CronRunner
 from .__types import DictData, DictStr, Matrix, Re, TupleStr
-from .conf import *
-from .errors import *
-from .event import *
-from .job import *
-from .logs import (
+from .audits import (
     Audit,
     AuditModel,
     FileAudit,
-    FileTrace,
-    Trace,
-    TraceData,
-    TraceMeta,
-    TraceModel,
     get_audit,
-    get_dt_tznow,
-    get_trace,
 )
+from .conf import *
+from .errors import *
+from .event import *
+from .job import *
 from .params import *
 from .result import (
     CANCEL,
@@ -34,5 +27,14 @@ from .result import (
 )
 from .reusables import *
 from .stages import *
+from .traces import (
+    ConsoleTrace,
+    FileTrace,
+    Trace,
+    TraceData,
+    TraceMeta,
+    TraceModel,
+    get_trace,
+)
 from .utils import *
 from .workflow import *

ddeutil/workflow/api/log_conf.py

@@ -1,15 +1,16 @@
 from typing import Any
 
+from uvicorn.config import LOGGING_CONFIG as LOGGING_CONFIG_UVICORN
+
 from ..conf import config
 
 LOGGING_CONFIG: dict[str, Any] = {  # pragma: no cov
     "version": 1,
     "disable_existing_loggers": False,
     "formatters": {
-        "standard": {
-            "format": "%(asctime)s [%(levelname)s] %(name)s: %(message)s"
-        },
-        "custom_formatter": {
+        "default": LOGGING_CONFIG_UVICORN["formatters"]["default"],
+        "access": LOGGING_CONFIG_UVICORN["formatters"]["access"],
+        "custom": {
             "format": config.log_format,
             "datefmt": config.log_datetime_format,
         },
@@ -18,14 +19,10 @@ LOGGING_CONFIG: dict[str, Any] = {  # pragma: no cov
         "level": "DEBUG" if config.debug else "INFO",
     },
     "handlers": {
-        "default": {
-            "formatter": "standard",
-            "class": "logging.StreamHandler",
-            "stream": "ext://sys.stderr",
-        },
-        "stream_handler": {
-            # "formatter": "standard",
-            "formatter": "custom_formatter",
+        "default": LOGGING_CONFIG_UVICORN["handlers"]["default"],
+        "access": LOGGING_CONFIG_UVICORN["handlers"]["access"],
+        "stream_custom": {
+            "formatter": "custom",
             "class": "logging.StreamHandler",
             "stream": "ext://sys.stdout",
         },
@@ -39,34 +36,24 @@ LOGGING_CONFIG: dict[str, Any] = {  # pragma: no cov
     },
     "loggers": {
         "uvicorn": {
-            # "handlers": ["default", "file_handler"],
             "handlers": ["default"],
             "level": "DEBUG" if config.debug else "INFO",
             "propagate": False,
         },
         "uvicorn.access": {
-            # "handlers": ["stream_handler", "file_handler"],
-            "handlers": ["stream_handler"],
+            "handlers": ["access"],
             "level": "DEBUG" if config.debug else "INFO",
             "propagate": False,
         },
         "uvicorn.error": {
-            # "handlers": ["stream_handler", "file_handler"],
-            "handlers": ["stream_handler"],
+            "handlers": ["default"],
             "level": "DEBUG" if config.debug else "INFO",
-            "propagate": False,
         },
-        "uvicorn.asgi": {
-            # "handlers": ["stream_handler", "file_handler"],
-            "handlers": ["stream_handler"],
-            "level": "TRACE",
-            "propagate": False,
+        "ddeutil.workflow": {
+            "handlers": ["stream_custom"],
+            "level": "INFO",
+            # "propagate": False,
+            "propagate": True,
         },
-        # "ddeutil.workflow": {
-        #     "handlers": ["stream_handler"],
-        #     "level": "INFO",
-        #     # "propagate": False,
-        #     "propagate": True,
-        # },
     },
 }

ddeutil/workflow/api/routes/logs.py

@@ -10,7 +10,7 @@ from fastapi import APIRouter, Path, Query
 from fastapi import status as st
 from fastapi.responses import UJSONResponse
 
-from ...logs import get_audit
+from ...audits import get_audit
 from ...result import Result
 
 router = APIRouter(

ddeutil/workflow/api/routes/workflows.py

@@ -16,8 +16,8 @@ from fastapi.responses import UJSONResponse
 from pydantic import BaseModel
 
 from ...__types import DictData
-from ...conf import Loader
-from ...logs import AuditModel, get_audit
+from ...audits import AuditModel, get_audit
+from ...conf import YamlParser
 from ...result import Result
 from ...workflow import Workflow
 
@@ -32,7 +32,7 @@ router = APIRouter(
 @router.get(path="/", status_code=st.HTTP_200_OK)
 async def get_workflows() -> DictData:
     """Return all workflow workflows that exists in config path."""
-    workflows: DictData = dict(Loader.finds(Workflow))
+    workflows: DictData = dict(YamlParser.finds(Workflow))
     return {
         "message": f"Getting all workflows: {len(workflows)}",
         "count": len(workflows),

ddeutil/workflow/audits.py

@@ -0,0 +1,374 @@
+# ------------------------------------------------------------------------------
+# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
+# Licensed under the MIT License. See LICENSE in the project root for
+# license information.
+# ------------------------------------------------------------------------------
+from __future__ import annotations
+
+import json
+import logging
+import os
+from abc import ABC, abstractmethod
+from collections.abc import Iterator
+from datetime import datetime
+from pathlib import Path
+from typing import ClassVar, Optional, TypeVar, Union
+
+from pydantic import BaseModel, Field
+from pydantic.functional_validators import model_validator
+from typing_extensions import Self
+
+from .__types import DictData
+from .conf import dynamic
+from .traces import TraceModel, get_trace, set_logging
+
+logger = logging.getLogger("ddeutil.workflow")
+
+
+class BaseAudit(BaseModel, ABC):
+    """Base Audit Pydantic Model with abstraction class property that implement
+    only model fields. This model should to use with inherit to logging
+    subclass like file, sqlite, etc.
+    """
+
+    extras: DictData = Field(
+        default_factory=dict,
+        description="An extras parameter that want to override core config",
+    )
+    name: str = Field(description="A workflow name.")
+    release: datetime = Field(description="A release datetime.")
+    type: str = Field(description="A running type before logging.")
+    context: DictData = Field(
+        default_factory=dict,
+        description="A context that receive from a workflow execution result.",
+    )
+    parent_run_id: Optional[str] = Field(
+        default=None, description="A parent running ID."
+    )
+    run_id: str = Field(description="A running ID")
+    execution_time: float = Field(default=0, description="An execution time.")
+
+    @model_validator(mode="after")
+    def __model_action(self) -> Self:
+        """Do before the Audit action with WORKFLOW_AUDIT_ENABLE_WRITE env variable.
+
+        :rtype: Self
+        """
+        if dynamic("enable_write_audit", extras=self.extras):
+            self.do_before()
+
+        # NOTE: Start setting log config in this line with cache.
+        set_logging("ddeutil.workflow")
+        return self
+
+    @classmethod
+    @abstractmethod
+    def is_pointed(
+        cls,
+        name: str,
+        release: datetime,
+        *,
+        extras: Optional[DictData] = None,
+    ) -> bool:
+        raise NotImplementedError(
+            "Audit should implement `is_pointed` class-method"
+        )
+
+    @classmethod
+    @abstractmethod
+    def find_audits(
+        cls,
+        name: str,
+        *,
+        extras: Optional[DictData] = None,
+    ) -> Iterator[Self]:
+        raise NotImplementedError(
+            "Audit should implement `find_audits` class-method"
+        )
+
+    @classmethod
+    @abstractmethod
+    def find_audit_with_release(
+        cls,
+        name: str,
+        release: Optional[datetime] = None,
+        *,
+        extras: Optional[DictData] = None,
+    ) -> Self:
+        raise NotImplementedError(
+            "Audit should implement `find_audit_with_release` class-method"
+        )
+
+    def do_before(self) -> None:  # pragma: no cov
+        """To something before end up of initial log model."""
+
+    @abstractmethod
+    def save(self, excluded: Optional[list[str]]) -> None:  # pragma: no cov
+        """Save this model logging to target logging store."""
+        raise NotImplementedError("Audit should implement `save` method.")
+
+
+class NullAudit(BaseAudit):
+
+    @classmethod
+    def is_pointed(
+        cls,
+        name: str,
+        release: datetime,
+        *,
+        extras: Optional[DictData] = None,
+    ) -> bool:
+        return False
+
+    @classmethod
+    def find_audits(
+        cls,
+        name: str,
+        *,
+        extras: Optional[DictData] = None,
+    ) -> Iterator[Self]:
+        raise NotImplementedError()
+
+    @classmethod
+    def find_audit_with_release(
+        cls,
+        name: str,
+        release: Optional[datetime] = None,
+        *,
+        extras: Optional[DictData] = None,
+    ) -> Self:
+        raise NotImplementedError()
+
+    def save(self, excluded: Optional[list[str]]) -> None:
+        """Do nothing when do not set audit."""
+        return
+
+
+class FileAudit(BaseAudit):
+    """File Audit Pydantic Model that use to saving log data from result of
+    workflow execution. It inherits from BaseAudit model that implement the
+    ``self.save`` method for file.
+    """
+
+    filename_fmt: ClassVar[str] = (
+        "workflow={name}/release={release:%Y%m%d%H%M%S}"
+    )
+
+    def do_before(self) -> None:
+        """Create directory of release before saving log file."""
+        self.pointer().mkdir(parents=True, exist_ok=True)
+
+    @classmethod
+    def find_audits(
+        cls, name: str, *, extras: Optional[DictData] = None
+    ) -> Iterator[Self]:
+        """Generate the audit data that found from logs path with specific a
+        workflow name.
+
+        :param name: A workflow name that want to search release logging data.
+        :param extras: An extra parameter that want to override core config.
+
+        :rtype: Iterator[Self]
+        """
+        pointer: Path = (
+            dynamic("audit_path", extras=extras) / f"workflow={name}"
+        )
+        if not pointer.exists():
+            raise FileNotFoundError(f"Pointer: {pointer.absolute()}.")
+
+        for file in pointer.glob("./release=*/*.log"):
+            with file.open(mode="r", encoding="utf-8") as f:
+                yield cls.model_validate(obj=json.load(f))
+
+    @classmethod
+    def find_audit_with_release(
+        cls,
+        name: str,
+        release: Optional[datetime] = None,
+        *,
+        extras: Optional[DictData] = None,
+    ) -> Self:
+        """Return the audit data that found from logs path with specific
+        workflow name and release values. If a release does not pass to an input
+        argument, it will return the latest release from the current log path.
+
+        :param name: (str) A workflow name that want to search log.
+        :param release: (datetime) A release datetime that want to search log.
+        :param extras: An extra parameter that want to override core config.
+
+        :raise FileNotFoundError:
+        :raise NotImplementedError: If an input release does not pass to this
+            method. Because this method does not implement latest log.
+
+        :rtype: Self
+        """
+        if release is None:
+            raise NotImplementedError("Find latest log does not implement yet.")
+
+        pointer: Path = (
+            dynamic("audit_path", extras=extras)
+            / f"workflow={name}/release={release:%Y%m%d%H%M%S}"
+        )
+        if not pointer.exists():
+            raise FileNotFoundError(
+                f"Pointer: ./logs/workflow={name}/"
+                f"release={release:%Y%m%d%H%M%S} does not found."
+            )
+
+        latest_file: Path = max(pointer.glob("./*.log"), key=os.path.getctime)
+        with latest_file.open(mode="r", encoding="utf-8") as f:
+            return cls.model_validate(obj=json.load(f))
+
+    @classmethod
+    def is_pointed(
+        cls,
+        name: str,
+        release: datetime,
+        *,
+        extras: Optional[DictData] = None,
+    ) -> bool:
+        """Check the release log already pointed or created at the destination
+        log path.
+
+        :param name: (str) A workflow name.
+        :param release: (datetime) A release datetime.
+        :param extras: An extra parameter that want to override core config.
+
+        :rtype: bool
+        :return: Return False if the release log was not pointed or created.
+        """
+        # NOTE: Return False if enable writing log flag does not set.
+        if not dynamic("enable_write_audit", extras=extras):
+            return False
+
+        # NOTE: create pointer path that use the same logic of pointer method.
+        pointer: Path = dynamic(
+            "audit_path", extras=extras
+        ) / cls.filename_fmt.format(name=name, release=release)
+
+        return pointer.exists()
+
+    def pointer(self) -> Path:
+        """Return release directory path that was generated from model data.
+
+        :rtype: Path
+        """
+        return dynamic(
+            "audit_path", extras=self.extras
+        ) / self.filename_fmt.format(name=self.name, release=self.release)
+
+    def save(self, excluded: Optional[list[str]] = None) -> Self:
+        """Save logging data that receive a context data from a workflow
+        execution result.
+
+        :param excluded: An excluded list of key name that want to pass in the
+            model_dump method.
+
+        :rtype: Self
+        """
+        trace: TraceModel = get_trace(
+            self.run_id,
+            parent_run_id=self.parent_run_id,
+            extras=self.extras,
+        )
+
+        # NOTE: Check environ variable was set for real writing.
+        if not dynamic("enable_write_audit", extras=self.extras):
+            trace.debug("[AUDIT]: Skip writing log cause config was set")
+            return self
+
+        log_file: Path = (
+            self.pointer() / f"{self.parent_run_id or self.run_id}.log"
+        )
+        log_file.write_text(
+            json.dumps(
+                self.model_dump(exclude=excluded),
+                default=str,
+                indent=2,
+            ),
+            encoding="utf-8",
+        )
+        return self
+
+
+class SQLiteAudit(BaseAudit):  # pragma: no cov
+    """SQLite Audit Pydantic Model."""
+
+    table_name: ClassVar[str] = "audits"
+    schemas: ClassVar[
+        str
+    ] = """
+        workflow        str,
+        release         int,
+        type            str,
+        context         json,
+        parent_run_id   int,
+        run_id          int,
+        update          datetime
+        primary key ( run_id )
+        """
+
+    @classmethod
+    def is_pointed(
+        cls,
+        name: str,
+        release: datetime,
+        *,
+        extras: Optional[DictData] = None,
+    ) -> bool: ...
+
+    @classmethod
+    def find_audits(
+        cls,
+        name: str,
+        *,
+        extras: Optional[DictData] = None,
+    ) -> Iterator[Self]: ...
+
+    @classmethod
+    def find_audit_with_release(
+        cls,
+        name: str,
+        release: Optional[datetime] = None,
+        *,
+        extras: Optional[DictData] = None,
+    ) -> Self: ...
+
+    def save(self, excluded: Optional[list[str]]) -> SQLiteAudit:
+        """Save logging data that receive a context data from a workflow
+        execution result.
+        """
+        trace: TraceModel = get_trace(
+            self.run_id,
+            parent_run_id=self.parent_run_id,
+            extras=self.extras,
+        )
+
+        # NOTE: Check environ variable was set for real writing.
+        if not dynamic("enable_write_audit", extras=self.extras):
+            trace.debug("[AUDIT]: Skip writing log cause config was set")
+            return self
+
+        raise NotImplementedError("SQLiteAudit does not implement yet.")
+
+
+Audit = TypeVar("Audit", bound=BaseAudit)
+AuditModel = Union[
+    NullAudit,
+    FileAudit,
+    SQLiteAudit,
+]
+
+
+def get_audit(
+    extras: Optional[DictData] = None,
+) -> type[AuditModel]:  # pragma: no cov
+    """Get an audit class that dynamic base on the config audit path value.
+
+    :param extras: An extra parameter that want to override the core config.
+
+    :rtype: type[Audit]
+    """
+    if dynamic("audit_path", extras=extras).is_file():
+        return SQLiteAudit
+    return FileAudit

ddeutil/workflow/cli.py

@@ -19,22 +19,22 @@ app = typer.Typer(
 
 
 @app.callback()
-def callback():
-    """Manage Workflow CLI app.
+def callback() -> None:
+    """Manage Workflow Orchestration CLI.
 
     Use it with the interface workflow engine.
     """
 
 
 @app.command()
-def version():
+def version() -> None:
     """Get the ddeutil-workflow package version."""
     typer.echo(f"ddeutil-workflow=={__version__}")
     typer.echo(f"python-version=={python_version()}")
 
 
-@app.command()
-def job(
+@app.command(name="job")
+def execute_job(
     params: Annotated[str, typer.Option(help="A job execute parameters")],
     job: Annotated[str, typer.Option(help="A job model")],
     parent_run_id: Annotated[str, typer.Option(help="A parent running ID")],
@@ -115,5 +115,19 @@ def make(
     typer.echo(f"Start create YAML template filename: {name.resolve()}")
 
 
+workflow_app = typer.Typer()
+app.add_typer(workflow_app, name="workflow", help="An Only Workflow CLI.")
+
+
+@workflow_app.callback()
+def workflow_callback():
+    """Manage Only Workflow CLI."""
+
+
+@workflow_app.command(name="execute")
+def workflow_execute():
+    """"""
+
+
 if __name__ == "__main__":
     app()

ddeutil/workflow/conf.py

@@ -7,12 +7,11 @@ from __future__ import annotations
 
 import copy
 import os
-from abc import ABC, abstractmethod
 from collections.abc import Iterator
 from functools import cached_property
 from inspect import isclass
 from pathlib import Path
-from typing import Final, Optional, Protocol, TypeVar, Union
+from typing import Final, Optional, TypeVar, Union
 from zoneinfo import ZoneInfo
 
 from ddeutil.core import str2bool
@@ -145,18 +144,6 @@ class Config:  # pragma: no cov
     def stage_default_id(self) -> bool:
         return str2bool(env("CORE_STAGE_DEFAULT_ID", "false"))
 
-    @property
-    def max_cron_per_workflow(self) -> int:
-        """The maximum on value that store in workflow model.
-
-        :rtype: int
-        """
-        return int(env("CORE_MAX_CRON_PER_WORKFLOW", "5"))
-
-    @property
-    def max_queue_complete_hist(self) -> int:
-        return int(env("CORE_MAX_QUEUE_COMPLETE_HIST", "16"))
-
 
 class APIConfig:
     """API Config object."""
@@ -170,23 +157,7 @@ class APIConfig:
         return env("API_PREFIX_PATH", f"/api/v{self.version}")
 
 
-class BaseLoad(ABC):  # pragma: no cov
-    """Base Load object is the abstraction object for any Load object that
-    should to inherit from this base class.
-    """
-
-    @classmethod
-    @abstractmethod
-    def find(cls, name: str, *args, **kwargs) -> DictData: ...
-
-    @classmethod
-    @abstractmethod
-    def finds(
-        cls, obj: object, *args, **kwargs
-    ) -> Iterator[tuple[str, DictData]]: ...
-
-
-class FileLoad(BaseLoad):
+class YamlParser:
     """Base Load object that use to search config data by given some identity
     value like name of `Workflow` or `Crontab` templates.
 
@@ -428,24 +399,6 @@ def dynamic(
     return extra
 
 
-class Loader(Protocol):  # pragma: no cov
-    type: str
-    path: Path
-    data: DictData
-    extras: DictData
-    externals: DictData
-
-    def __init__(self, *args, **kwargs) -> None: ...
-
-    @classmethod
-    def find(cls, name: str, *args, **kwargs) -> DictData: ...
-
-    @classmethod
-    def finds(
-        cls, obj: object, *args, **kwargs
-    ) -> Iterator[tuple[str, DictData]]: ...
-
-
 def pass_env(value: T) -> T:  # pragma: no cov
     """Passing environment variable to an input value.
 

ddeutil/workflow/event.py

@@ -23,7 +23,7 @@ from typing_extensions import Self
 
 from .__cron import WEEKDAYS, CronJob, CronJobYear, CronRunner, Options
 from .__types import DictData, DictStr
-from .conf import FileLoad
+from .conf import YamlParser
 
 Interval = Literal["daily", "weekly", "monthly"]
 
@@ -139,7 +139,7 @@ class Crontab(BaseModel):
         :rtype: Self
         """
         extras: DictData = extras or {}
-        loader: FileLoad = FileLoad(name, extras=extras)
+        loader: YamlParser = YamlParser(name, extras=extras)
 
         # NOTE: Validate the config type match with current connection model
         if loader.type != cls.__name__: