ddeutil-workflow 0.0.73__py3-none-any.whl → 0.0.75__py3-none-any.whl

ddeutil/workflow/audits.py

@@ -3,6 +3,37 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
+"""Audit and Execution Tracking Module.
+
+This module provides comprehensive audit capabilities for workflow execution
+tracking and monitoring. It supports multiple audit backends for capturing
+execution metadata, status information, and detailed logging.
+
+The audit system tracks workflow, job, and stage executions with configurable
+storage backends including file-based JSON storage and database persistence.
+
+Classes:
+    Audit: Pydantic model for audit data validation
+    FileAudit: File-based audit storage implementation
+
+Functions:
+    get_audit_model: Factory function for creating audit instances
+
+Example:
+
+    ```python
+    from ddeutil.workflow.audits import get_audit_model
+
+    # NOTE: Create file-based Audit.
+    audit = get_audit_model(run_id="run-123")
+    audit.info("Workflow execution started")
+    audit.success("Workflow completed successfully")
+    ```
+
+Note:
+    Audit instances are automatically configured based on the workflow
+    configuration and provide detailed execution tracking capabilities.
+"""
 from __future__ import annotations
 
 import json
@@ -12,15 +43,17 @@ from abc import ABC, abstractmethod
 from collections.abc import Iterator
 from datetime import datetime
 from pathlib import Path
-from typing import ClassVar, Optional, TypeVar, Union
+from typing import ClassVar, Optional, Union
+from urllib.parse import ParseResult
 
 from pydantic import BaseModel, Field
+from pydantic.functional_serializers import field_serializer
 from pydantic.functional_validators import model_validator
 from typing_extensions import Self
 
 from .__types import DictData
 from .conf import dynamic
-from .traces import TraceModel, get_trace, set_logging
+from .traces import Trace, get_trace, set_logging
 
 logger = logging.getLogger("ddeutil.workflow")
 
@@ -108,42 +141,6 @@ class BaseAudit(BaseModel, ABC):
         raise NotImplementedError("Audit should implement `save` method.")
 
 
-class NullAudit(BaseAudit):
-
-    @classmethod
-    def is_pointed(
-        cls,
-        name: str,
-        release: datetime,
-        *,
-        extras: Optional[DictData] = None,
-    ) -> bool:
-        return False
-
-    @classmethod
-    def find_audits(
-        cls,
-        name: str,
-        *,
-        extras: Optional[DictData] = None,
-    ) -> Iterator[Self]:
-        raise NotImplementedError()
-
-    @classmethod
-    def find_audit_with_release(
-        cls,
-        name: str,
-        release: Optional[datetime] = None,
-        *,
-        extras: Optional[DictData] = None,
-    ) -> Self:
-        raise NotImplementedError()
-
-    def save(self, excluded: Optional[list[str]]) -> None:
-        """Do nothing when do not set audit."""
-        return
-
-
 class FileAudit(BaseAudit):
     """File Audit Pydantic Model that use to saving log data from result of
     workflow execution. It inherits from BaseAudit model that implement the
@@ -154,6 +151,13 @@ class FileAudit(BaseAudit):
         "workflow={name}/release={release:%Y%m%d%H%M%S}"
     )
 
+    @field_serializer("extras")
+    def __serialize_extras(self, value: DictData) -> DictData:
+        return {
+            k: (v.geturl() if isinstance(v, ParseResult) else v)
+            for k, v in value.items()
+        }
+
     def do_before(self) -> None:
         """Create directory of release before saving log file."""
         self.pointer().mkdir(parents=True, exist_ok=True)
@@ -171,7 +175,7 @@ class FileAudit(BaseAudit):
         :rtype: Iterator[Self]
         """
         pointer: Path = (
-            dynamic("audit_path", extras=extras) / f"workflow={name}"
+            Path(dynamic("audit_url", extras=extras).path) / f"workflow={name}"
         )
         if not pointer.exists():
             raise FileNotFoundError(f"Pointer: {pointer.absolute()}.")
@@ -206,7 +210,7 @@ class FileAudit(BaseAudit):
             raise NotImplementedError("Find latest log does not implement yet.")
 
         pointer: Path = (
-            dynamic("audit_path", extras=extras)
+            Path(dynamic("audit_url", extras=extras).path)
             / f"workflow={name}/release={release:%Y%m%d%H%M%S}"
         )
         if not pointer.exists():
@@ -242,8 +246,8 @@ class FileAudit(BaseAudit):
             return False
 
         # NOTE: create pointer path that use the same logic of pointer method.
-        pointer: Path = dynamic(
-            "audit_path", extras=extras
+        pointer: Path = Path(
+            dynamic("audit_url", extras=extras).path
         ) / cls.filename_fmt.format(name=name, release=release)
 
         return pointer.exists()
@@ -253,8 +257,8 @@ class FileAudit(BaseAudit):
 
         :rtype: Path
         """
-        return dynamic(
-            "audit_path", extras=self.extras
+        return Path(
+            dynamic("audit_url", extras=self.extras).path
         ) / self.filename_fmt.format(name=self.name, release=self.release)
 
     def save(self, excluded: Optional[list[str]] = None) -> Self:
@@ -266,7 +270,7 @@ class FileAudit(BaseAudit):
 
         :rtype: Self
         """
-        trace: TraceModel = get_trace(
+        trace: Trace = get_trace(
             self.run_id,
             parent_run_id=self.parent_run_id,
             extras=self.extras,
@@ -338,7 +342,7 @@ class SQLiteAudit(BaseAudit):  # pragma: no cov
         """Save logging data that receive a context data from a workflow
         execution result.
         """
-        trace: TraceModel = get_trace(
+        trace: Trace = get_trace(
             self.run_id,
             parent_run_id=self.parent_run_id,
             extras=self.extras,
@@ -352,23 +356,34 @@ class SQLiteAudit(BaseAudit):  # pragma: no cov
         raise NotImplementedError("SQLiteAudit does not implement yet.")
 
 
-Audit = TypeVar("Audit", bound=BaseAudit)
-AuditModel = Union[
-    NullAudit,
+Audit = Union[
     FileAudit,
     SQLiteAudit,
+    BaseAudit,
 ]
 
 
-def get_audit(
+def get_audit_model(
     extras: Optional[DictData] = None,
-) -> type[AuditModel]:  # pragma: no cov
-    """Get an audit class that dynamic base on the config audit path value.
+) -> type[Audit]:  # pragma: no cov
+    """Get an audit model that dynamic base on the config audit path value.
 
     :param extras: An extra parameter that want to override the core config.
 
     :rtype: type[Audit]
     """
-    if dynamic("audit_path", extras=extras).is_file():
-        return SQLiteAudit
-    return FileAudit
+    # NOTE: Allow you to override trace model by the extra parameter.
+    map_audit_models: dict[str, type[Trace]] = extras.get(
+        "audit_model_mapping", {}
+    )
+    url: ParseResult
+    if (url := dynamic("audit_url", extras=extras)).scheme and (
+        url.scheme == "sqlite"
+        or (url.scheme == "file" and Path(url.path).is_file())
+    ):
+        return map_audit_models.get("sqlite", FileAudit)
+    elif url.scheme and url.scheme != "file":
+        raise NotImplementedError(
+            f"Does not implement the audit model support for URL: {url}"
+        )
+    return map_audit_models.get("file", FileAudit)

ddeutil/workflow/cli.py

@@ -16,7 +16,6 @@ from pydantic import Field, TypeAdapter
 from .__about__ import __version__
 from .__types import DictData
 from .errors import JobError
-from .event import Crontab
 from .job import Job
 from .params import Param
 from .result import Result
@@ -153,19 +152,6 @@ class WorkflowSchema(Workflow):
         default_factory=dict,
         description="A parameters that need to use on this workflow.",
     )
-    on: Union[list[Union[Crontab, str]], str] = Field(
-        default_factory=list,
-        description="A list of Crontab instance for this workflow schedule.",
-    )
-
-
-CRONTAB_TYPE = Literal["Crontab"]
-
-
-class CrontabSchema(Crontab):
-    """Override crontab model fields for generate JSON schema file."""
-
-    type: CRONTAB_TYPE = Field(description="A type of crontab template.")
 
 
 @workflow_app.command(name="json-schema")
@@ -176,7 +162,7 @@ def workflow_json_schema(
     ] = Path("./json-schema.json"),
 ) -> None:
     """Generate JSON schema file from the Workflow model."""
-    template = dict[str, Union[WorkflowSchema, CrontabSchema]]
+    template = dict[str, WorkflowSchema]
     json_schema = TypeAdapter(template).json_schema(by_alias=True)
     template_schema: dict[str, str] = {
         "$schema": "http://json-schema.org/draft-07/schema#",

ddeutil/workflow/conf.py

@@ -3,14 +3,49 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
-from __future__ import annotations
-
+"""Configuration Management for Workflow System.
+
+This module provides comprehensive configuration management for the workflow
+system, including YAML parsing, dynamic configuration loading, environment
+variable handling, and configuration validation.
+
+The configuration system supports hierarchical configuration files, environment
+variable substitution, and dynamic parameter resolution for flexible workflow
+deployment across different environments.
+
+Classes:
+    Config: Main configuration class with validation
+    YamlParser: YAML configuration file parser and loader
+
+Functions:
+    dynamic: Get dynamic configuration values with fallbacks
+    pass_env: Process environment variable substitution
+    api_config: Get API-specific configuration settings
+
+Example:
+    ```python
+    from ddeutil.workflow.conf import Config, YamlParser
+
+    # Load workflow configuration
+    parser = YamlParser("my-workflow")
+    workflow_config = parser.data
+
+    # Access dynamic configuration
+    from ddeutil.workflow.conf import dynamic
+    log_level = dynamic("log_level", default="INFO")
+    ```
+
+Note:
+    Configuration files support environment variable substitution using
+    ${VAR_NAME} syntax and provide extensive validation capabilities.
+"""
 import copy
 import os
 from collections.abc import Iterator
 from functools import cached_property
 from pathlib import Path
 from typing import Final, Optional, TypeVar, Union
+from urllib.parse import ParseResult, urlparse
 from zoneinfo import ZoneInfo
 
 from ddeutil.core import str2bool
@@ -28,10 +63,12 @@ PREFIX: Final[str] = "WORKFLOW"
 def env(var: str, default: Optional[str] = None) -> Optional[str]:
     """Get environment variable with uppercase and adding prefix string.
 
-    :param var: (str) A env variable name.
-    :param default: (Optional[str]) A default value if an env var does not set.
+    Args:
+        var: A env variable name.
+        default: A default value if an env var does not set.
 
-    :rtype: Optional[str]
+    Returns:
+        Optional[str]: The environment variable value or default.
     """
     return os.getenv(f"{PREFIX}_{var.upper().replace(' ', '_')}", default)
 
@@ -47,25 +84,18 @@ class Config:  # pragma: no cov
     def conf_path(self) -> Path:
         """Config path that keep all workflow template YAML files.
 
-        :rtype: Path
+        Returns:
+            Path: The configuration path for workflow templates.
         """
         return Path(env("CORE_CONF_PATH", "./conf"))
 
-    @property
-    def tz(self) -> ZoneInfo:
-        """Timezone value that return with the `ZoneInfo` object and use for all
-        datetime object in this workflow engine.
-
-        :rtype: ZoneInfo
-        """
-        return ZoneInfo(env("CORE_TIMEZONE", "UTC"))
-
     @property
     def generate_id_simple_mode(self) -> bool:
         """Flag for generate running ID with simple mode. That does not use
         `md5` function after generate simple mode.
 
-        :rtype: bool
+        Returns:
+            bool: True if simple mode ID generation is enabled.
         """
         return str2bool(env("CORE_GENERATE_ID_SIMPLE_MODE", "true"))
 
@@ -92,8 +122,8 @@ class Config:  # pragma: no cov
         return [r.strip() for r in regis_filter_str.split(",")]
 
     @property
-    def trace_path(self) -> Path:
-        return Path(env("LOG_TRACE_PATH", "./logs"))
+    def trace_url(self) -> ParseResult:
+        return urlparse(env("LOG_TRACE_URL", "file:./logs"))
 
     @property
     def debug(self) -> bool:
@@ -103,6 +133,16 @@ class Config:  # pragma: no cov
         """
         return str2bool(env("LOG_DEBUG_MODE", "true"))
 
+    @property
+    def log_tz(self) -> ZoneInfo:
+        """Timezone value that return with the `ZoneInfo` object and use for all
+        datetime object in this workflow engine.
+
+        Returns:
+            ZoneInfo: The timezone configuration for the workflow engine.
+        """
+        return ZoneInfo(env("LOG_TIMEZONE", "UTC"))
+
     @property
     def log_format(self) -> str:
         return env(
@@ -129,8 +169,8 @@ class Config:  # pragma: no cov
         return str2bool(env("LOG_TRACE_ENABLE_WRITE", "false"))
 
     @property
-    def audit_path(self) -> Path:
-        return Path(env("LOG_AUDIT_PATH", "./audits"))
+    def audit_url(self) -> ParseResult:
+        return urlparse(env("LOG_AUDIT_URL", "file:./audits"))
 
     @property
     def enable_write_audit(self) -> bool:
@@ -190,8 +230,8 @@ class YamlParser:
         name: str,
         *,
         path: Optional[Union[str, Path]] = None,
-        externals: DictData | None = None,
-        extras: DictData | None = None,
+        externals: Optional[DictData] = None,
+        extras: Optional[DictData] = None,
         obj: Optional[Union[object, str]] = None,
     ) -> None:
         self.path: Path = Path(dynamic("conf_path", f=path, extras=extras))
@@ -260,10 +300,13 @@ class YamlParser:
                     continue
 
                 if data := cls.filter_yaml(file, name=name):
+                    file_stat: os.stat_result = file.lstat()
+                    data["created_at"] = file_stat.st_ctime
+                    data["updated_at"] = file_stat.st_mtime
                     if not obj_type:
-                        all_data.append((file.lstat().st_mtime, data))
+                        all_data.append((file_stat.st_mtime, data))
                     elif (t := data.get("type")) and t == obj_type:
-                        all_data.append((file.lstat().st_mtime, data))
+                        all_data.append((file_stat.st_mtime, data))
                     else:
                         continue
 
@@ -435,10 +478,12 @@ def pass_env(value: T) -> T:  # pragma: no cov
 
 
 class CallerSecret(SecretStr):  # pragma: no cov
-    """Workflow Secret String model."""
+    """Workflow Secret String model that was inherited from the SecretStr model
+    and override the `get_secret_value` method only.
+    """
 
     def get_secret_value(self) -> str:
-        """Override get_secret_value by adding pass_env before return the
+        """Override the `get_secret_value` by adding pass_env before return the
         real-value.
 
         :rtype: str

ddeutil/workflow/errors.py

@@ -3,9 +3,22 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
-"""Exception objects for this package do not do anything because I want to
-create the lightweight workflow package. So, this module do just an exception
-annotate for handle error only.
+"""Exception Classes for Workflow Orchestration.
+
+This module provides a comprehensive exception hierarchy for the workflow system.
+The exceptions are designed to be lightweight while providing sufficient context
+for error handling and debugging.
+
+Classes:
+    BaseError: Base exception class with context support
+    StageError: Exceptions related to stage execution
+    JobError: Exceptions related to job execution
+    WorkflowError: Exceptions related to workflow execution
+    ParamError: Exceptions related to parameter validation
+    ResultError: Exceptions related to result processing
+
+Functions:
+    to_dict: Convert exception instances to dictionary format
 """
 from __future__ import annotations
 
@@ -15,8 +28,14 @@ from .__types import DictData, StrOrInt
 
 
 class ErrorData(TypedDict):
-    """Error data type dict for typing necessary keys of return of to_dict func
-    and method.
+    """Error data structure for exception serialization.
+
+    This TypedDict defines the standard structure for converting exceptions
+    to dictionary format for consistent error handling across the system.
+
+    Attributes:
+        name: Exception class name
+        message: Exception message content
     """
 
     name: str
@@ -24,11 +43,26 @@ class ErrorData(TypedDict):
 
 
 def to_dict(exception: Exception, **kwargs) -> ErrorData:  # pragma: no cov
-    """Create dict data from exception instance.
-
-    :param exception: An exception object.
-
-    :rtype: ErrorData
+    """Create dictionary data from exception instance.
+
+    Converts an exception object to a standardized dictionary format
+    for consistent error handling and serialization.
+
+    Args:
+        exception: Exception object to convert
+        **kwargs: Additional key-value pairs to include in result
+
+    Returns:
+        ErrorData: Dictionary containing exception name and message
+
+    Example:
+        ```python
+        try:
+            raise ValueError("Something went wrong")
+        except Exception as e:
+            error_data = to_dict(e, context="workflow_execution")
+            # Returns: {"name": "ValueError", "message": "Something went wrong", "context": "workflow_execution"}
+        ```
     """
     return {
         "name": exception.__class__.__name__,
@@ -38,14 +72,27 @@ def to_dict(exception: Exception, **kwargs) -> ErrorData:  # pragma: no cov
 
 
 class BaseError(Exception):
-    """Base Workflow exception class will implement the ``refs`` argument for
-    making an error context to the result context.
+    """Base exception class for all workflow-related errors.
 
-    Attributes:
-        refs: (:obj:str, optional)
-        context: (:obj:DictData)
-        params: (:obj:DictData)
+    BaseError provides the foundation for all workflow exceptions, offering
+    enhanced context management and error tracking capabilities. It supports
+    reference IDs for error correlation and maintains context information
+    for debugging purposes.
 
+    Attributes:
+        refs: Optional reference identifier for error correlation
+        context: Additional context data related to the error
+        params: Parameter data that was being processed when error occurred
+
+    Example:
+        ```python
+        try:
+            # Some workflow operation
+            pass
+        except BaseError as e:
+            error_dict = e.to_dict(with_refs=True)
+            print(f"Error in {e.refs}: {error_dict}")
+        ```
     """
 
     def __init__(
@@ -76,10 +123,30 @@ class BaseError(Exception):
         with_refs: bool = False,
         **kwargs,
     ) -> Union[ErrorData, dict[str, ErrorData]]:
-        """Return ErrorData data from the current exception object. If with_refs
-        flag was set, it will return mapping of refs and itself data.
+        """Convert exception to dictionary format.
+
+        Serializes the exception to a standardized dictionary format.
+        Optionally includes reference mapping for error correlation.
+
+        Args:
+            with_refs: Include reference ID mapping in result
+            **kwargs: Additional key-value pairs to include
+
+        Returns:
+            ErrorData or dict: Exception data, optionally mapped by reference ID
+
+        Example:
+            ```python
+            error = BaseError("Something failed", refs="stage-1")
+
+            # Simple format
+            data = error.to_dict()
+            # Returns: {"name": "BaseError", "message": "Something failed"}
 
-        :rtype: ErrorData
+            # With reference mapping
+            ref_data = error.to_dict(with_refs=True)
+            # Returns: {"stage-1": {"name": "BaseError", "message": "Something failed"}}
+            ```
         """
         data: ErrorData = to_dict(self)
         if with_refs and (self.refs is not None and self.refs != "EMPTY"):