ddeutil-workflow 0.0.48__py3-none-any.whl → 0.0.49__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1 +1 @@
-__version__: str = "0.0.48"
+__version__: str = "0.0.49"

ddeutil/workflow/__init__.py

@@ -32,8 +32,11 @@ from .job import (
 )
 from .logs import (
     Audit,
+    AuditModel,
+    Trace,
     TraceData,
-    TraceLog,
+    TraceMeta,
+    TraceModel,
     get_audit,
     get_dt_tznow,
     get_trace,

ddeutil/workflow/api/routes/logs.py

@@ -10,7 +10,8 @@ from fastapi import APIRouter, Path, Query
 from fastapi import status as st
 from fastapi.responses import UJSONResponse
 
-from ...logs import get_audit, get_trace_obj
+from ...logs import get_audit
+from ...result import Result
 
 log_route = APIRouter(
     prefix="/logs",
@@ -33,6 +34,7 @@ async def get_traces(
     """Return all trace logs from the current trace log path that config with
     `WORKFLOW_LOG_PATH` environment variable name.
     """
+    result = Result()
     return {
         "message": (
             f"Getting trace logs with offset: {offset} and limit: {limit}"
@@ -44,7 +46,7 @@ async def get_traces(
                 exclude_unset=True,
                 exclude_defaults=True,
             )
-            for trace in get_trace_obj().find_logs()
+            for trace in result.trace.find_traces()
         ],
     }
 
@@ -63,12 +65,11 @@ async def get_trace_with_id(run_id: str):
     - **run_id**: A running ID that want to search a trace log from the log
         path.
     """
+    result = Result()
     return {
         "message": f"Getting trace log with specific running ID: {run_id}",
         "trace": (
-            get_trace_obj()
-            .find_log_with_id(run_id)
-            .model_dump(
+            result.trace.find_trace_with_id(run_id).model_dump(
                 by_alias=True,
                 exclude_none=True,
                 exclude_unset=True,

ddeutil/workflow/conf.py

@@ -11,7 +11,7 @@ from collections.abc import Iterator
 from datetime import timedelta
 from functools import cached_property
 from pathlib import Path
-from typing import Optional, TypeVar
+from typing import Final, Optional, TypeVar
 from zoneinfo import ZoneInfo
 
 from ddeutil.core import str2bool
@@ -21,12 +21,15 @@ from ddeutil.io.paths import glob_files, is_ignored, read_ignore
 from .__types import DictData, TupleStr
 
 T = TypeVar("T")
-PREFIX: str = "WORKFLOW"
+PREFIX: Final[str] = "WORKFLOW"
 
 
 def env(var: str, default: str | None = None) -> str | None:  # pragma: no cov
     """Get environment variable with uppercase and adding prefix string.
 
+    :param var: (str) A env variable name.
+    :param default: (str | None) A default value if an env var does not set.
+
     :rtype: str | None
     """
     return os.getenv(f"{PREFIX}_{var.upper().replace(' ', '_')}", default)
@@ -51,22 +54,13 @@ class Config:  # pragma: no cov
     """
 
     # NOTE: Core
-    @property
-    def root_path(self) -> Path:
-        """Root path or the project path for this workflow engine that use for
-        combine with `conf_path` value.
-
-        :rtype: Path
-        """
-        return Path(env("CORE_ROOT_PATH", "."))
-
     @property
     def conf_path(self) -> Path:
         """Config path that keep all workflow template YAML files.
 
         :rtype: Path
         """
-        return self.root_path / env("CORE_CONF_PATH", "conf")
+        return Path(env("CORE_CONF_PATH", "./conf"))
 
     @property
     def tz(self) -> ZoneInfo:
@@ -78,12 +72,12 @@ class Config:  # pragma: no cov
         return ZoneInfo(env("CORE_TIMEZONE", "UTC"))
 
     @property
-    def gen_id_simple_mode(self) -> bool:
+    def generate_id_simple_mode(self) -> bool:
         return str2bool(env("CORE_GENERATE_ID_SIMPLE_MODE", "true"))
 
     # NOTE: Register
     @property
-    def regis_call(self) -> list[str]:
+    def registry_caller(self) -> list[str]:
         """Register Caller that is a list of importable string for the call
         stage model can get.
 
@@ -93,7 +87,7 @@ class Config:  # pragma: no cov
         return [r.strip() for r in regis_call_str.split(",")]
 
     @property
-    def regis_filter(self) -> list[str]:
+    def registry_filter(self) -> list[str]:
         """Register Filter that is a list of importable string for the filter
         template.
 
@@ -106,7 +100,7 @@ class Config:  # pragma: no cov
 
     # NOTE: Log
     @property
-    def log_path(self) -> Path:
+    def trace_path(self) -> Path:
         return Path(env("LOG_TRACE_PATH", "./logs"))
 
     @property
@@ -165,11 +159,7 @@ class Config:  # pragma: no cov
         return str2bool(env("CORE_JOB_RAISE_ERROR", "true"))
 
     @property
-    def job_default_id(self) -> bool:
-        return str2bool(env("CORE_JOB_DEFAULT_ID", "false"))
-
-    @property
-    def max_on_per_workflow(self) -> int:
+    def max_cron_per_workflow(self) -> int:
         """The maximum on value that store in workflow model.
 
         :rtype: int
@@ -305,23 +295,31 @@ class SimLoad:
                     )
 
     @classmethod
-    def is_ignore(cls, file: Path, conf_path: Path) -> bool:
+    def is_ignore(
+        cls,
+        file: Path,
+        conf_path: Path,
+        *,
+        ignore_filename: Optional[str] = None,
+    ) -> bool:
         """Check this file was ignored.
 
         :param file: (Path) A file path that want to check.
         :param conf_path: (Path) A config path that want to read the config
             ignore file.
+        :param ignore_filename: (str) An ignore filename.
 
         :rtype: bool
         """
-        return is_ignored(file, read_ignore(conf_path / ".confignore"))
+        ignore_filename: str = ignore_filename or ".confignore"
+        return is_ignored(file, read_ignore(conf_path / ignore_filename))
 
     @classmethod
     def filter_yaml(cls, file: Path, name: str | None = None) -> DictData:
         """Read a YAML file context from an input file path and specific name.
 
-        :param file: (Path)
-        :param name: (str)
+        :param file: (Path) A file path that want to extract YAML context.
+        :param name: (str) A key name that search on a YAML context.
 
         :rtype: DictData
         """
@@ -374,8 +372,8 @@ def dynamic(
 class Loader(SimLoad):
     """Loader Object that get the config `yaml` file from current path.
 
-    :param name: A name of config data that will read by Yaml Loader object.
-    :param externals: An external parameters
+    :param name: (str) A name of config data that will read by Yaml Loader object.
+    :param externals: (DictData) An external parameters
     """
 
     @classmethod
@@ -383,17 +381,19 @@ class Loader(SimLoad):
         cls,
         obj: object,
         *,
+        path: Path | None = None,
         included: list[str] | None = None,
         excluded: list[str] | None = None,
-        path: Path | None = None,
         **kwargs,
     ) -> Iterator[tuple[str, DictData]]:
         """Override the find class method from the Simple Loader object.
 
         :param obj: An object that want to validate matching before return.
-        :param included:
-        :param excluded:
-        :param path:
+        :param path: (Path) A override config path.
+        :param included: An excluded list of data key that want to reject this
+            data if any key exist.
+        :param excluded: An included list of data key that want to filter from
+            data.
 
         :rtype: Iterator[tuple[str, DictData]]
         """

ddeutil/workflow/job.py

@@ -483,7 +483,13 @@ class Job(BaseModel):
         except Exception as err:
             raise JobException(f"{err.__class__.__name__}: {err}") from err
 
-    def set_outputs(self, output: DictData, to: DictData) -> DictData:
+    def set_outputs(
+        self,
+        output: DictData,
+        to: DictData,
+        *,
+        job_id: Optional[None] = None,
+    ) -> DictData:
         """Set an outputs from execution process to the received context. The
         result from execution will pass to value of `strategies` key.
 
@@ -511,22 +517,21 @@ class Job(BaseModel):
 
         :param output: An output context.
         :param to: A context data that want to add output result.
+        :param job_id: A job ID if the id field does not set.
 
         :rtype: DictData
         """
         if "jobs" not in to:
             to["jobs"] = {}
 
-        if self.id is None and not dynamic(
-            "job_default_id", extras=self.extras
-        ):
+        if self.id is None and job_id is None:
             raise JobException(
                 "This job do not set the ID before setting execution output."
             )
 
         # NOTE: If the job ID did not set, it will use index of jobs key
         #   instead.
-        _id: str = self.id or str(len(to["jobs"]) + 1)
+        _id: str = self.id or job_id
 
         errors: DictData = (
             {"errors": output.pop("errors", {})} if "errors" in output else {}

ddeutil/workflow/logs.py

@@ -3,8 +3,9 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
-# [x] Use fix config
-"""A Logs module contain TraceLog dataclass and AuditLog model.
+# [x] Use dynamic config
+# [x] Use fix config for `get_logger`, and Model initialize step.
+"""A Logs module contain Trace dataclass and Audit Pydantic model.
 """
 from __future__ import annotations
 
@@ -19,14 +20,14 @@ from functools import lru_cache
 from inspect import Traceback, currentframe, getframeinfo
 from pathlib import Path
 from threading import get_ident
-from typing import ClassVar, Literal, Optional, Union
+from typing import ClassVar, Literal, Optional, TypeVar, Union
 
-from pydantic import BaseModel, Field, ValidationInfo
+from pydantic import BaseModel, Field
 from pydantic.dataclasses import dataclass
 from pydantic.functional_validators import model_validator
 from typing_extensions import Self
 
-from .__types import DictData, DictStr, TupleStr
+from .__types import DictData, DictStr
 from .conf import config, dynamic
 from .utils import cut_id, get_dt_now
 
@@ -60,22 +61,6 @@ def get_logger(name: str):
 
 logger = get_logger("ddeutil.workflow")
 
-__all__: TupleStr = (
-    "FileTraceLog",
-    "SQLiteTraceLog",
-    "TraceData",
-    "TraceMeta",
-    "TraceLog",
-    "get_dt_tznow",
-    "get_logger",
-    "get_trace",
-    "get_trace_obj",
-    "get_audit",
-    "FileAudit",
-    "SQLiteAudit",
-    "Audit",
-)
-
 
 def get_dt_tznow() -> datetime:  # pragma: no cov
     """Return the current datetime object that passing the config timezone.
@@ -170,13 +155,39 @@ class TraceData(BaseModel):  # pragma: no cov
 
 
 @dataclass(frozen=True)
-class BaseTraceLog(ABC):  # pragma: no cov
-    """Base Trace Log dataclass object."""
+class BaseTrace(ABC):  # pragma: no cov
+    """Base Trace dataclass with abstraction class property."""
 
     run_id: str
     parent_run_id: Optional[str] = field(default=None)
     extras: DictData = field(default_factory=dict, compare=False, repr=False)
 
+    @classmethod
+    @abstractmethod
+    def find_traces(
+        cls,
+        path: Path | None = None,
+        extras: Optional[DictData] = None,
+    ) -> Iterator[TraceData]:  # pragma: no cov
+        raise NotImplementedError(
+            "Trace dataclass should implement `find_traces` class-method."
+        )
+
+    @classmethod
+    @abstractmethod
+    def find_trace_with_id(
+        cls,
+        run_id: str,
+        force_raise: bool = True,
+        *,
+        path: Path | None = None,
+        extras: Optional[DictData] = None,
+    ) -> TraceData:
+        raise NotImplementedError(
+            "Trace dataclass should implement `find_trace_with_id` "
+            "class-method."
+        )
+
     @abstractmethod
     def writer(self, message: str, is_err: bool = False) -> None:
         """Write a trace message after making to target pointer object. The
@@ -318,11 +329,11 @@ class BaseTraceLog(ABC):  # pragma: no cov
         logger.exception(msg, stacklevel=2)
 
 
-class FileTraceLog(BaseTraceLog):  # pragma: no cov
-    """Trace Log object that write file to the local storage."""
+class FileTrace(BaseTrace):  # pragma: no cov
+    """File Trace dataclass that write file to the local storage."""
 
     @classmethod
-    def find_logs(
+    def find_traces(
         cls,
         path: Path | None = None,
         extras: Optional[DictData] = None,
@@ -333,13 +344,13 @@ class FileTraceLog(BaseTraceLog):  # pragma: no cov
         :param extras: An extra parameter that want to override core config.
         """
         for file in sorted(
-            (path or dynamic("log_path", extras=extras)).glob("./run_id=*"),
+            (path or dynamic("trace_path", extras=extras)).glob("./run_id=*"),
             key=lambda f: f.lstat().st_mtime,
         ):
             yield TraceData.from_path(file)
 
     @classmethod
-    def find_log_with_id(
+    def find_trace_with_id(
         cls,
         run_id: str,
         force_raise: bool = True,
@@ -354,7 +365,7 @@ class FileTraceLog(BaseTraceLog):  # pragma: no cov
         :param path:
         :param extras: An extra parameter that want to override core config.
         """
-        base_path: Path = path or dynamic("log_path", extras=extras)
+        base_path: Path = path or dynamic("trace_path", extras=extras)
         file: Path = base_path / f"run_id={run_id}"
         if file.exists():
             return TraceData.from_path(file)
@@ -368,7 +379,7 @@ class FileTraceLog(BaseTraceLog):  # pragma: no cov
     @property
     def pointer(self) -> Path:
         log_file: Path = (
-            dynamic("log_path", extras=self.extras)
+            dynamic("trace_path", extras=self.extras)
             / f"run_id={self.parent_run_id or self.run_id}"
         )
         if not log_file.exists():
@@ -453,8 +464,8 @@ class FileTraceLog(BaseTraceLog):  # pragma: no cov
             await f.write(trace_meta.model_dump_json() + "\n")
 
 
-class SQLiteTraceLog(BaseTraceLog):  # pragma: no cov
-    """Trace Log object that write trace log to the SQLite database file."""
+class SQLiteTrace(BaseTrace):  # pragma: no cov
+    """SQLite Trace dataclass that write trace log to the SQLite database file."""
 
     table_name: ClassVar[str] = "audits"
     schemas: ClassVar[
@@ -468,10 +479,21 @@ class SQLiteTraceLog(BaseTraceLog):  # pragma: no cov
         """
 
     @classmethod
-    def find_logs(cls) -> Iterator[DictStr]: ...
+    def find_traces(
+        cls,
+        path: Path | None = None,
+        extras: Optional[DictData] = None,
+    ) -> Iterator[TraceData]: ...
 
     @classmethod
-    def find_log_with_id(cls, run_id: str) -> DictStr: ...
+    def find_trace_with_id(
+        cls,
+        run_id: str,
+        force_raise: bool = True,
+        *,
+        path: Path | None = None,
+        extras: Optional[DictData] = None,
+    ) -> TraceData: ...
 
     def make_message(self, message: str) -> str: ...
 
@@ -480,47 +502,34 @@ class SQLiteTraceLog(BaseTraceLog):  # pragma: no cov
     def awriter(self, message: str, is_err: bool = False) -> None: ...
 
 
-TraceLog = Union[
-    FileTraceLog,
-    SQLiteTraceLog,
+Trace = TypeVar("Trace", bound=BaseTrace)
+TraceModel = Union[
+    FileTrace,
+    SQLiteTrace,
 ]
 
 
 def get_trace(
     run_id: str,
-    parent_run_id: str | None = None,
     *,
+    parent_run_id: str | None = None,
     extras: Optional[DictData] = None,
-) -> TraceLog:  # pragma: no cov
-    """Get dynamic TraceLog object from the setting config.
+) -> TraceModel:  # pragma: no cov
+    """Get dynamic Trace instance from the core config (it can override by an
+    extras argument) that passing running ID and parent running ID.
 
-    :param run_id: A running ID.
-    :param parent_run_id: A parent running ID.
-    :param extras: An extra parameter that want to override the core config.
+    :param run_id: (str) A running ID.
+    :param parent_run_id: (str) A parent running ID.
+    :param extras: (DictData) An extra parameter that want to override the core
+        config values.
 
     :rtype: TraceLog
     """
-    if dynamic("log_path", extras=extras).is_file():
-        return SQLiteTraceLog(
+    if dynamic("trace_path", extras=extras).is_file():
+        return SQLiteTrace(
             run_id, parent_run_id=parent_run_id, extras=(extras or {})
         )
-    return FileTraceLog(
-        run_id, parent_run_id=parent_run_id, extras=(extras or {})
-    )
-
-
-def get_trace_obj(
-    extras: Optional[DictData] = None,
-) -> type[TraceLog]:  # pragma: no cov
-    """Get trace object.
-
-    :param extras: An extra parameter that want to override the core config.
-
-    :rtype: type[TraceLog]
-    """
-    if dynamic("log_path", extras=extras).is_file():
-        return SQLiteTraceLog
-    return FileTraceLog
+    return FileTrace(run_id, parent_run_id=parent_run_id, extras=(extras or {}))
 
 
 class BaseAudit(BaseModel, ABC):
@@ -548,15 +557,50 @@ class BaseAudit(BaseModel, ABC):
     execution_time: float = Field(default=0, description="An execution time.")
 
     @model_validator(mode="after")
-    def __model_action(self, info: ValidationInfo) -> Self:
+    def __model_action(self) -> Self:
         """Do before the Audit action with WORKFLOW_AUDIT_ENABLE_WRITE env variable.
 
         :rtype: Self
         """
-        if dynamic("enable_write_audit", extras=info.data.get("extras")):
+        if dynamic("enable_write_audit", extras=self.extras):
             self.do_before()
         return self
 
+    @classmethod
+    @abstractmethod
+    def is_pointed(
+        cls,
+        name: str,
+        release: datetime,
+        *,
+        extras: Optional[DictData] = None,
+    ) -> bool:
+        raise NotImplementedError(
+            "Audit should implement `is_pointed` class-method"
+        )
+
+    @classmethod
+    @abstractmethod
+    def find_audits(
+        cls, name: str, *, extras: Optional[DictData] = None
+    ) -> Iterator[Self]:
+        raise NotImplementedError(
+            "Audit should implement `find_audits` class-method"
+        )
+
+    @classmethod
+    @abstractmethod
+    def find_audit_with_release(
+        cls,
+        name: str,
+        release: datetime | None = None,
+        *,
+        extras: Optional[DictData] = None,
+    ) -> Self:
+        raise NotImplementedError(
+            "Audit should implement `find_audit_with_release` class-method"
+        )
+
     def do_before(self) -> None:  # pragma: no cov
         """To something before end up of initial log model."""
 
@@ -688,7 +732,11 @@ class FileAudit(BaseAudit):
 
         :rtype: Self
         """
-        trace: TraceLog = get_trace(self.run_id, self.parent_run_id)
+        trace: Trace = get_trace(
+            self.run_id,
+            parent_run_id=self.parent_run_id,
+            extras=self.extras,
+        )
 
         # NOTE: Check environ variable was set for real writing.
         if not dynamic("enable_write_audit", extras=self.extras):
@@ -726,11 +774,38 @@ class SQLiteAudit(BaseAudit):  # pragma: no cov
         primary key ( run_id )
         """
 
+    @classmethod
+    def is_pointed(
+        cls,
+        name: str,
+        release: datetime,
+        *,
+        extras: Optional[DictData] = None,
+    ) -> bool: ...
+
+    @classmethod
+    def find_audits(
+        cls, name: str, *, extras: Optional[DictData] = None
+    ) -> Iterator[Self]: ...
+
+    @classmethod
+    def find_audit_with_release(
+        cls,
+        name: str,
+        release: datetime | None = None,
+        *,
+        extras: Optional[DictData] = None,
+    ) -> Self: ...
+
     def save(self, excluded: list[str] | None) -> SQLiteAudit:
         """Save logging data that receive a context data from a workflow
         execution result.
         """
-        trace: TraceLog = get_trace(self.run_id, self.parent_run_id)
+        trace: Trace = get_trace(
+            self.run_id,
+            parent_run_id=self.parent_run_id,
+            extras=self.extras,
+        )
 
         # NOTE: Check environ variable was set for real writing.
         if not dynamic("enable_write_audit", extras=self.extras):
@@ -740,19 +815,8 @@ class SQLiteAudit(BaseAudit):  # pragma: no cov
         raise NotImplementedError("SQLiteAudit does not implement yet.")
 
 
-class RemoteFileAudit(FileAudit):  # pragma: no cov
-    """Remote File Audit Pydantic Model."""
-
-    def save(self, excluded: list[str] | None) -> RemoteFileAudit: ...
-
-
-class RedisAudit(BaseAudit):  # pragma: no cov
-    """Redis Audit Pydantic Model."""
-
-    def save(self, excluded: list[str] | None) -> RedisAudit: ...
-
-
-Audit = Union[
+Audit = TypeVar("Audit", bound=BaseAudit)
+AuditModel = Union[
     FileAudit,
     SQLiteAudit,
 ]
@@ -760,7 +824,7 @@ Audit = Union[
 
 def get_audit(
     extras: Optional[DictData] = None,
-) -> type[Audit]:  # pragma: no cov
+) -> type[AuditModel]:  # pragma: no cov
     """Get an audit class that dynamic base on the config audit path value.
 
     :param extras: An extra parameter that want to override the core config.

ddeutil/workflow/result.py

@@ -21,7 +21,7 @@ from typing_extensions import Self
 
 from .__types import DictData
 from .conf import dynamic
-from .logs import TraceLog, get_dt_tznow, get_trace
+from .logs import Trace, get_dt_tznow, get_trace
 from .utils import default_gen_id, gen_id, get_dt_now
 
 
@@ -68,7 +68,7 @@ class Result:
     parent_run_id: Optional[str] = field(default=None, compare=False)
     ts: datetime = field(default_factory=get_dt_tznow, compare=False)
 
-    trace: Optional[TraceLog] = field(default=None, compare=False, repr=False)
+    trace: Optional[Trace] = field(default=None, compare=False, repr=False)
     extras: DictData = field(default_factory=dict, compare=False, repr=False)
 
     @classmethod
@@ -113,8 +113,10 @@ class Result:
         :rtype: Self
         """
         if self.trace is None:  # pragma: no cov
-            self.trace: TraceLog = get_trace(
-                self.run_id, self.parent_run_id, extras=self.extras
+            self.trace: Trace = get_trace(
+                self.run_id,
+                parent_run_id=self.parent_run_id,
+                extras=self.extras,
             )
         return self
 
@@ -126,8 +128,8 @@ class Result:
         :rtype: Self
         """
         self.parent_run_id: str = running_id
-        self.trace: TraceLog = get_trace(
-            self.run_id, running_id, extras=self.extras
+        self.trace: Trace = get_trace(
+            self.run_id, parent_run_id=running_id, extras=self.extras
         )
         return self