ddeutil-workflow 0.0.67__py3-none-any.whl → 0.0.69__py3-none-any.whl

ddeutil/workflow/audits.py

@@ -0,0 +1,374 @@
+# ------------------------------------------------------------------------------
+# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
+# Licensed under the MIT License. See LICENSE in the project root for
+# license information.
+# ------------------------------------------------------------------------------
+from __future__ import annotations
+
+import json
+import logging
+import os
+from abc import ABC, abstractmethod
+from collections.abc import Iterator
+from datetime import datetime
+from pathlib import Path
+from typing import ClassVar, Optional, TypeVar, Union
+
+from pydantic import BaseModel, Field
+from pydantic.functional_validators import model_validator
+from typing_extensions import Self
+
+from .__types import DictData
+from .conf import dynamic
+from .traces import TraceModel, get_trace, set_logging
+
+logger = logging.getLogger("ddeutil.workflow")
+
+
+class BaseAudit(BaseModel, ABC):
+    """Base Audit Pydantic Model with abstraction class property that implement
+    only model fields. This model should to use with inherit to logging
+    subclass like file, sqlite, etc.
+    """
+
+    extras: DictData = Field(
+        default_factory=dict,
+        description="An extras parameter that want to override core config",
+    )
+    name: str = Field(description="A workflow name.")
+    release: datetime = Field(description="A release datetime.")
+    type: str = Field(description="A running type before logging.")
+    context: DictData = Field(
+        default_factory=dict,
+        description="A context that receive from a workflow execution result.",
+    )
+    parent_run_id: Optional[str] = Field(
+        default=None, description="A parent running ID."
+    )
+    run_id: str = Field(description="A running ID")
+    execution_time: float = Field(default=0, description="An execution time.")
+
+    @model_validator(mode="after")
+    def __model_action(self) -> Self:
+        """Do before the Audit action with WORKFLOW_AUDIT_ENABLE_WRITE env variable.
+
+        :rtype: Self
+        """
+        if dynamic("enable_write_audit", extras=self.extras):
+            self.do_before()
+
+        # NOTE: Start setting log config in this line with cache.
+        set_logging("ddeutil.workflow")
+        return self
+
+    @classmethod
+    @abstractmethod
+    def is_pointed(
+        cls,
+        name: str,
+        release: datetime,
+        *,
+        extras: Optional[DictData] = None,
+    ) -> bool:
+        raise NotImplementedError(
+            "Audit should implement `is_pointed` class-method"
+        )
+
+    @classmethod
+    @abstractmethod
+    def find_audits(
+        cls,
+        name: str,
+        *,
+        extras: Optional[DictData] = None,
+    ) -> Iterator[Self]:
+        raise NotImplementedError(
+            "Audit should implement `find_audits` class-method"
+        )
+
+    @classmethod
+    @abstractmethod
+    def find_audit_with_release(
+        cls,
+        name: str,
+        release: Optional[datetime] = None,
+        *,
+        extras: Optional[DictData] = None,
+    ) -> Self:
+        raise NotImplementedError(
+            "Audit should implement `find_audit_with_release` class-method"
+        )
+
+    def do_before(self) -> None:  # pragma: no cov
+        """To something before end up of initial log model."""
+
+    @abstractmethod
+    def save(self, excluded: Optional[list[str]]) -> None:  # pragma: no cov
+        """Save this model logging to target logging store."""
+        raise NotImplementedError("Audit should implement `save` method.")
+
+
+class NullAudit(BaseAudit):
+
+    @classmethod
+    def is_pointed(
+        cls,
+        name: str,
+        release: datetime,
+        *,
+        extras: Optional[DictData] = None,
+    ) -> bool:
+        return False
+
+    @classmethod
+    def find_audits(
+        cls,
+        name: str,
+        *,
+        extras: Optional[DictData] = None,
+    ) -> Iterator[Self]:
+        raise NotImplementedError()
+
+    @classmethod
+    def find_audit_with_release(
+        cls,
+        name: str,
+        release: Optional[datetime] = None,
+        *,
+        extras: Optional[DictData] = None,
+    ) -> Self:
+        raise NotImplementedError()
+
+    def save(self, excluded: Optional[list[str]]) -> None:
+        """Do nothing when do not set audit."""
+        return
+
+
+class FileAudit(BaseAudit):
+    """File Audit Pydantic Model that use to saving log data from result of
+    workflow execution. It inherits from BaseAudit model that implement the
+    ``self.save`` method for file.
+    """
+
+    filename_fmt: ClassVar[str] = (
+        "workflow={name}/release={release:%Y%m%d%H%M%S}"
+    )
+
+    def do_before(self) -> None:
+        """Create directory of release before saving log file."""
+        self.pointer().mkdir(parents=True, exist_ok=True)
+
+    @classmethod
+    def find_audits(
+        cls, name: str, *, extras: Optional[DictData] = None
+    ) -> Iterator[Self]:
+        """Generate the audit data that found from logs path with specific a
+        workflow name.
+
+        :param name: A workflow name that want to search release logging data.
+        :param extras: An extra parameter that want to override core config.
+
+        :rtype: Iterator[Self]
+        """
+        pointer: Path = (
+            dynamic("audit_path", extras=extras) / f"workflow={name}"
+        )
+        if not pointer.exists():
+            raise FileNotFoundError(f"Pointer: {pointer.absolute()}.")
+
+        for file in pointer.glob("./release=*/*.log"):
+            with file.open(mode="r", encoding="utf-8") as f:
+                yield cls.model_validate(obj=json.load(f))
+
+    @classmethod
+    def find_audit_with_release(
+        cls,
+        name: str,
+        release: Optional[datetime] = None,
+        *,
+        extras: Optional[DictData] = None,
+    ) -> Self:
+        """Return the audit data that found from logs path with specific
+        workflow name and release values. If a release does not pass to an input
+        argument, it will return the latest release from the current log path.
+
+        :param name: (str) A workflow name that want to search log.
+        :param release: (datetime) A release datetime that want to search log.
+        :param extras: An extra parameter that want to override core config.
+
+        :raise FileNotFoundError:
+        :raise NotImplementedError: If an input release does not pass to this
+            method. Because this method does not implement latest log.
+
+        :rtype: Self
+        """
+        if release is None:
+            raise NotImplementedError("Find latest log does not implement yet.")
+
+        pointer: Path = (
+            dynamic("audit_path", extras=extras)
+            / f"workflow={name}/release={release:%Y%m%d%H%M%S}"
+        )
+        if not pointer.exists():
+            raise FileNotFoundError(
+                f"Pointer: ./logs/workflow={name}/"
+                f"release={release:%Y%m%d%H%M%S} does not found."
+            )
+
+        latest_file: Path = max(pointer.glob("./*.log"), key=os.path.getctime)
+        with latest_file.open(mode="r", encoding="utf-8") as f:
+            return cls.model_validate(obj=json.load(f))
+
+    @classmethod
+    def is_pointed(
+        cls,
+        name: str,
+        release: datetime,
+        *,
+        extras: Optional[DictData] = None,
+    ) -> bool:
+        """Check the release log already pointed or created at the destination
+        log path.
+
+        :param name: (str) A workflow name.
+        :param release: (datetime) A release datetime.
+        :param extras: An extra parameter that want to override core config.
+
+        :rtype: bool
+        :return: Return False if the release log was not pointed or created.
+        """
+        # NOTE: Return False if enable writing log flag does not set.
+        if not dynamic("enable_write_audit", extras=extras):
+            return False
+
+        # NOTE: create pointer path that use the same logic of pointer method.
+        pointer: Path = dynamic(
+            "audit_path", extras=extras
+        ) / cls.filename_fmt.format(name=name, release=release)
+
+        return pointer.exists()
+
+    def pointer(self) -> Path:
+        """Return release directory path that was generated from model data.
+
+        :rtype: Path
+        """
+        return dynamic(
+            "audit_path", extras=self.extras
+        ) / self.filename_fmt.format(name=self.name, release=self.release)
+
+    def save(self, excluded: Optional[list[str]] = None) -> Self:
+        """Save logging data that receive a context data from a workflow
+        execution result.
+
+        :param excluded: An excluded list of key name that want to pass in the
+            model_dump method.
+
+        :rtype: Self
+        """
+        trace: TraceModel = get_trace(
+            self.run_id,
+            parent_run_id=self.parent_run_id,
+            extras=self.extras,
+        )
+
+        # NOTE: Check environ variable was set for real writing.
+        if not dynamic("enable_write_audit", extras=self.extras):
+            trace.debug("[AUDIT]: Skip writing log cause config was set")
+            return self
+
+        log_file: Path = (
+            self.pointer() / f"{self.parent_run_id or self.run_id}.log"
+        )
+        log_file.write_text(
+            json.dumps(
+                self.model_dump(exclude=excluded),
+                default=str,
+                indent=2,
+            ),
+            encoding="utf-8",
+        )
+        return self
+
+
+class SQLiteAudit(BaseAudit):  # pragma: no cov
+    """SQLite Audit Pydantic Model."""
+
+    table_name: ClassVar[str] = "audits"
+    schemas: ClassVar[
+        str
+    ] = """
+        workflow        str,
+        release         int,
+        type            str,
+        context         json,
+        parent_run_id   int,
+        run_id          int,
+        update          datetime
+        primary key ( run_id )
+        """
+
+    @classmethod
+    def is_pointed(
+        cls,
+        name: str,
+        release: datetime,
+        *,
+        extras: Optional[DictData] = None,
+    ) -> bool: ...
+
+    @classmethod
+    def find_audits(
+        cls,
+        name: str,
+        *,
+        extras: Optional[DictData] = None,
+    ) -> Iterator[Self]: ...
+
+    @classmethod
+    def find_audit_with_release(
+        cls,
+        name: str,
+        release: Optional[datetime] = None,
+        *,
+        extras: Optional[DictData] = None,
+    ) -> Self: ...
+
+    def save(self, excluded: Optional[list[str]]) -> SQLiteAudit:
+        """Save logging data that receive a context data from a workflow
+        execution result.
+        """
+        trace: TraceModel = get_trace(
+            self.run_id,
+            parent_run_id=self.parent_run_id,
+            extras=self.extras,
+        )
+
+        # NOTE: Check environ variable was set for real writing.
+        if not dynamic("enable_write_audit", extras=self.extras):
+            trace.debug("[AUDIT]: Skip writing log cause config was set")
+            return self
+
+        raise NotImplementedError("SQLiteAudit does not implement yet.")
+
+
+Audit = TypeVar("Audit", bound=BaseAudit)
+AuditModel = Union[
+    NullAudit,
+    FileAudit,
+    SQLiteAudit,
+]
+
+
+def get_audit(
+    extras: Optional[DictData] = None,
+) -> type[AuditModel]:  # pragma: no cov
+    """Get an audit class that dynamic base on the config audit path value.
+
+    :param extras: An extra parameter that want to override the core config.
+
+    :rtype: type[Audit]
+    """
+    if dynamic("audit_path", extras=extras).is_file():
+        return SQLiteAudit
+    return FileAudit

ddeutil/workflow/cli.py

@@ -1,12 +1,17 @@
 import json
-from typing import Annotated, Any
+from pathlib import Path
+from platform import python_version
+from typing import Annotated, Any, Optional
 
 import typer
 import uvicorn
 
 from .__about__ import __version__
+from .__types import DictData
 from .api import app as fastapp
-from .api.logs import LOGGING_CONFIG
+from .errors import JobError
+from .job import Job
+from .result import Result
 
 app = typer.Typer(
     pretty_exceptions_enable=True,
@@ -14,23 +19,27 @@ app = typer.Typer(
 
 
 @app.callback()
-def callback():
-    """
-    Awesome Portal Gun
+def callback() -> None:
+    """Manage Workflow Orchestration CLI.
+
+    Use it with the interface workflow engine.
     """
-    typer.echo("Start call from callback function")
 
 
 @app.command()
-def version():
+def version() -> None:
     """Get the ddeutil-workflow package version."""
-    typer.echo(__version__)
+    typer.echo(f"ddeutil-workflow=={__version__}")
+    typer.echo(f"python-version=={python_version()}")
 
 
-@app.command()
-def job(
+@app.command(name="job")
+def execute_job(
     params: Annotated[str, typer.Option(help="A job execute parameters")],
-):
+    job: Annotated[str, typer.Option(help="A job model")],
+    parent_run_id: Annotated[str, typer.Option(help="A parent running ID")],
+    run_id: Annotated[Optional[str], typer.Option(help="A running ID")] = None,
+) -> None:
     """Job execution on the local.
 
     Example:
@@ -39,8 +48,32 @@ def job(
     try:
         params_dict: dict[str, Any] = json.loads(params)
     except json.JSONDecodeError as e:
-        raise ValueError(f"params does not support format: {params!r}.") from e
+        raise ValueError(f"Params does not support format: {params!r}.") from e
+
+    try:
+        job_dict: dict[str, Any] = json.loads(job)
+        _job: Job = Job.model_validate(obj=job_dict)
+    except json.JSONDecodeError as e:
+        raise ValueError(f"Params does not support format: {params!r}.") from e
+
     typer.echo(f"Job params: {params_dict}")
+    rs: Result = Result(
+        run_id=run_id,
+        parent_run_id=parent_run_id,
+    )
+
+    context: DictData = {}
+    try:
+        _job.set_outputs(
+            _job.execute(
+                params=params_dict,
+                run_id=rs.run_id,
+                parent_run_id=rs.parent_run_id,
+            ).context,
+            to=context,
+        )
+    except JobError as err:
+        rs.trace.error(f"[JOB]: {err.__class__.__name__}: {err}")
 
 
 @app.command()
@@ -48,21 +81,53 @@ def api(
     host: Annotated[str, typer.Option(help="A host url.")] = "0.0.0.0",
     port: Annotated[int, typer.Option(help="A port url.")] = 80,
     debug: Annotated[bool, typer.Option(help="A debug mode flag")] = True,
-    worker: Annotated[int, typer.Option(help="A worker number")] = None,
+    workers: Annotated[int, typer.Option(help="A worker number")] = None,
+    reload: Annotated[bool, typer.Option(help="A reload flag")] = False,
 ):
     """
     Provision API application from the FastAPI.
     """
+    from .api.log_conf import LOGGING_CONFIG
+
+    # LOGGING_CONFIG = {}
 
     uvicorn.run(
         fastapp,
         host=host,
         port=port,
         log_config=uvicorn.config.LOGGING_CONFIG | LOGGING_CONFIG,
-        log_level=("DEBUG" if debug else "INFO"),
-        workers=worker,
+        # NOTE: Logging level of uvicorn should be lowered case.
+        log_level=("debug" if debug else "info"),
+        workers=workers,
+        reload=reload,
     )
 
 
+@app.command()
+def make(
+    name: Annotated[Path, typer.Argument()],
+) -> None:
+    """
+    Create Workflow YAML template.
+
+    :param name:
+    """
+    typer.echo(f"Start create YAML template filename: {name.resolve()}")
+
+
+workflow_app = typer.Typer()
+app.add_typer(workflow_app, name="workflow", help="An Only Workflow CLI.")
+
+
+@workflow_app.callback()
+def workflow_callback():
+    """Manage Only Workflow CLI."""
+
+
+@workflow_app.command(name="execute")
+def workflow_execute():
+    """"""
+
+
 if __name__ == "__main__":
     app()

ddeutil/workflow/conf.py

@@ -7,12 +7,11 @@ from __future__ import annotations
 
 import copy
 import os
-from abc import ABC, abstractmethod
 from collections.abc import Iterator
 from functools import cached_property
 from inspect import isclass
 from pathlib import Path
-from typing import Final, Optional, Protocol, TypeVar, Union
+from typing import Final, Optional, TypeVar, Union
 from zoneinfo import ZoneInfo
 
 from ddeutil.core import str2bool
@@ -109,9 +108,9 @@ class Config:  # pragma: no cov
         return env(
             "LOG_FORMAT",
             (
-                "%(asctime)s.%(msecs)03d (%(name)-10s, %(process)-5d, "
+                "%(asctime)s.%(msecs)03d (%(process)-5d, "
                 "%(thread)-5d) [%(levelname)-7s] %(message)-120s "
-                "(%(filename)s:%(lineno)s)"
+                "(%(filename)s:%(lineno)s) (%(name)-10s)"
             ),
         )
 
@@ -145,44 +144,20 @@ class Config:  # pragma: no cov
     def stage_default_id(self) -> bool:
         return str2bool(env("CORE_STAGE_DEFAULT_ID", "false"))
 
-    @property
-    def max_cron_per_workflow(self) -> int:
-        """The maximum on value that store in workflow model.
-
-        :rtype: int
-        """
-        return int(env("CORE_MAX_CRON_PER_WORKFLOW", "5"))
-
-    @property
-    def max_queue_complete_hist(self) -> int:
-        return int(env("CORE_MAX_QUEUE_COMPLETE_HIST", "16"))
-
 
 class APIConfig:
     """API Config object."""
 
     @property
-    def prefix_path(self) -> str:
-        return env("API_PREFIX_PATH", "/api/v1")
-
-
-class BaseLoad(ABC):  # pragma: no cov
-    """Base Load object is the abstraction object for any Load object that
-    should to inherit from this base class.
-    """
-
-    @classmethod
-    @abstractmethod
-    def find(cls, name: str, *args, **kwargs) -> DictData: ...
+    def version(self) -> str:
+        return env("API_VERSION", "1")
 
-    @classmethod
-    @abstractmethod
-    def finds(
-        cls, obj: object, *args, **kwargs
-    ) -> Iterator[tuple[str, DictData]]: ...
+    @property
+    def prefix_path(self) -> str:
+        return env("API_PREFIX_PATH", f"/api/v{self.version}")
 
 
-class FileLoad(BaseLoad):
+class YamlParser:
     """Base Load object that use to search config data by given some identity
     value like name of `Workflow` or `Crontab` templates.
 
@@ -424,24 +399,6 @@ def dynamic(
     return extra
 
 
-class Loader(Protocol):  # pragma: no cov
-    type: str
-    path: Path
-    data: DictData
-    extras: DictData
-    externals: DictData
-
-    def __init__(self, *args, **kwargs) -> None: ...
-
-    @classmethod
-    def find(cls, name: str, *args, **kwargs) -> DictData: ...
-
-    @classmethod
-    def finds(
-        cls, obj: object, *args, **kwargs
-    ) -> Iterator[tuple[str, DictData]]: ...
-
-
 def pass_env(value: T) -> T:  # pragma: no cov
     """Passing environment variable to an input value.
 

ddeutil/workflow/event.py

@@ -3,8 +3,9 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
-"""Event module include all event object for trigger the Workflow to release.
-Now, it has only `Crontab` and `CrontabYear` event models on this module because
+"""An Event module keep all triggerable object to the Workflow model. The simple
+event trigger that use to run workflow is `Crontab` model.
+Now, it has only `Crontab` and `CrontabYear` event models in this module because
 I think it is the core event for workflow orchestration.
 """
 from __future__ import annotations
@@ -22,7 +23,7 @@ from typing_extensions import Self
 
 from .__cron import WEEKDAYS, CronJob, CronJobYear, CronRunner, Options
 from .__types import DictData, DictStr
-from .conf import FileLoad
+from .conf import YamlParser
 
 Interval = Literal["daily", "weekly", "monthly"]
 
@@ -95,7 +96,7 @@ class Crontab(BaseModel):
     tz: Annotated[
         TimeZoneName,
         Field(
-            description="A timezone string value",
+            description="A timezone string value.",
             alias="timezone",
         ),
     ] = "UTC"
@@ -138,7 +139,7 @@ class Crontab(BaseModel):
         :rtype: Self
         """
         extras: DictData = extras or {}
-        loader: FileLoad = FileLoad(name, extras=extras)
+        loader: YamlParser = YamlParser(name, extras=extras)
 
         # NOTE: Validate the config type match with current connection model
         if loader.type != cls.__name__:

ddeutil/workflow/result.py

@@ -13,6 +13,7 @@ from dataclasses import field
 from datetime import datetime
 from enum import Enum
 from typing import Optional, Union
+from zoneinfo import ZoneInfo
 
 from pydantic import ConfigDict
 from pydantic.dataclasses import dataclass
@@ -30,12 +31,20 @@ from . import (
     WorkflowError,
 )
 from .__types import DictData
+from .audits import TraceModel, get_trace
 from .conf import dynamic
 from .errors import ResultError
-from .logs import TraceModel, get_dt_tznow, get_trace
 from .utils import default_gen_id, gen_id, get_dt_now
 
 
+def get_dt_tznow(tz: Optional[ZoneInfo] = None) -> datetime:  # pragma: no cov
+    """Return the current datetime object that passing the config timezone.
+
+    :rtype: datetime
+    """
+    return get_dt_now(tz=dynamic("tz", f=tz))
+
+
 class Status(str, Enum):
     """Status Int Enum object that use for tracking execution status to the
     Result dataclass object.