ddeutil-workflow 0.0.80__tar.gz → 0.0.82__tar.gz

{ddeutil_workflow-0.0.80/src/ddeutil_workflow.egg-info → ddeutil_workflow-0.0.82}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddeutil-workflow
-Version: 0.0.80
+Version: 0.0.82
 Summary: Lightweight workflow orchestration with YAML template
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT

{ddeutil_workflow-0.0.80 → ddeutil_workflow-0.0.82}/pyproject.toml

@@ -91,7 +91,6 @@ omit = [
     "src/ddeutil/workflow/__cron.py",
     "src/ddeutil/workflow/__main__.py",
     "src/ddeutil/workflow/__types.py",
-    "src/ddeutil/workflow/cli.py",
     "src/ddeutil/workflow/api/__init__.py",
     "src/ddeutil/workflow/api/log_conf.py",
     "src/ddeutil/workflow/api/routes/__init__.py",

ddeutil_workflow-0.0.82/src/ddeutil/workflow/__about__.py

@@ -0,0 +1,2 @@
+__version__: str = "0.0.82"
+__python_version__: str = "3.9"

{ddeutil_workflow-0.0.80 → ddeutil_workflow-0.0.82}/src/ddeutil/workflow/__init__.py

@@ -50,11 +50,25 @@ Note:
 from .__cron import CronRunner
 from .__types import DictData, DictStr, Matrix, Re, TupleStr
 from .audits import (
+    EVENT,
+    FORCE,
+    NORMAL,
+    RERUN,
     Audit,
     FileAudit,
     get_audit,
 )
-from .conf import *
+from .conf import (
+    PREFIX,
+    CallerSecret,
+    Config,
+    YamlParser,
+    api_config,
+    config,
+    dynamic,
+    env,
+    pass_env,
+)
 from .errors import (
     BaseError,
     JobCancelError,
@@ -63,6 +77,9 @@ from .errors import (
     ResultError,
     StageCancelError,
     StageError,
+    StageNestedCancelError,
+    StageNestedError,
+    StageNestedSkipError,
     StageSkipError,
     UtilError,
     WorkflowCancelError,
@@ -132,15 +149,11 @@ from .stages import (
     VirtualPyStage,
 )
 from .traces import (
-    TraceManager,
+    Trace,
     get_trace,
 )
 from .utils import *
 from .workflow import (
-    EVENT,
-    FORCE,
-    NORMAL,
-    RERUN,
     ReleaseType,
     Workflow,
 )

ddeutil_workflow-0.0.80/src/ddeutil/workflow/cli.py → ddeutil_workflow-0.0.82/src/ddeutil/workflow/__main__.py

@@ -54,29 +54,30 @@ def init() -> None:
         dedent(
             """
         # Example workflow template.
-        wf-example:
-          type: Workflow
-          desc: |
-            An example workflow template that provide the demo of workflow.
-          params:
-            name:
-                type: str
-                default: "World"
-          jobs:
-            first-job:
-              stages:
-
-                - name: "Hello Stage"
-                  echo: "Start say hi to the console"
-
-                - name: "Call tasks"
-                  uses: tasks/say-hello-func@example
-                  with:
-                    name: ${{ params.name }}
-            second-job:
-
-                - name: "Hello Env"
-                  echo: "Start say hi with ${ WORKFLOW_DEMO_HELLO }"
+        name: wf-example:
+        type: Workflow
+        desc: |
+          An example workflow template that provide the demo of workflow.
+        params:
+          name:
+              type: str
+              default: "World"
+        jobs:
+          first-job:
+            stages:
+
+              - name: "Hello Stage"
+                echo: "Start say hi to the console"
+
+              - name: "Call tasks"
+                uses: tasks/say-hello-func@example
+                with:
+                  name: ${{ params.name }}
+
+          second-job:
+
+              - name: "Hello Env"
+                echo: "Start say hi with ${ WORKFLOW_DEMO_HELLO }"
         """
         ).lstrip("\n")
     )
@@ -89,12 +90,20 @@ def init() -> None:
         dummy_tasks_path.write_text(
             dedent(
                 """
+            from typing import Any, Optional
+
             from ddeutil.workflow import Result, tag
 
             @tag(name="example", alias="say-hello-func")
-            def hello_world_task(name: str, rs: Result) -> dict[str, str]:
+            def hello_world_task(name: str, rs: Result, extras: Optional[dict[str, Any]] = None) -> dict[str, str]:
                 \"\"\"Logging hello task function\"\"\"
-                rs.trace.info(f"Hello, {name}")
+                _extras = extras or {}
+                # NOTE: I will use custom newline logging if you pass `||`.
+                rs.trace.info(
+                    f"Hello, {name}||"
+                    f"> running ID: {rs.run_id}"
+                    f"> extras: {_extras}"
+                )
                 return {"name": name}
             """
             ).lstrip("\n")
@@ -106,18 +115,19 @@ def init() -> None:
     dotenv_file = Path(".env")
     mode: str = "a" if dotenv_file.exists() else "w"
     with dotenv_file.open(mode=mode) as f:
-        f.write("\n# Workflow env vars\n")
+        f.write("\n# Workflow Environment Variables\n")
         f.write(
             "WORKFLOW_DEMO_HELLO=foo\n"
             "WORKFLOW_CORE_DEBUG_MODE=true\n"
             "WORKFLOW_LOG_TIMEZONE=Asia/Bangkok\n"
-            "WORKFLOW_LOG_TRACE_ENABLE_WRITE=false\n"
+            'WORKFLOW_LOG_TRACE_HANDLERS=\'[{"type": "console"}]\'\n'
+            'WORKFLOW_LOG_AUDIT_CONF=\'{"type": "file", "path": "./audits"}\''
             "WORKFLOW_LOG_AUDIT_ENABLE_WRITE=true\n"
         )
 
     typer.echo("Starter command:")
     typer.echo(
-        "> `source .env && workflow-cli workflows execute --name=wf-example`"
+        ">>> `source .env && workflow-cli workflows execute --name=wf-example`"
     )
 
 
@@ -163,7 +173,7 @@ def api(
     debug: Annotated[bool, typer.Option(help="A debug mode flag")] = True,
     workers: Annotated[int, typer.Option(help="A worker number")] = None,
     reload: Annotated[bool, typer.Option(help="A reload flag")] = False,
-):
+) -> None:
     """
     Provision API application from the FastAPI.
     """
@@ -228,13 +238,12 @@ def workflow_execute(
     typer.echo(f"... with params: {params_dict}")
 
 
-WORKFLOW_TYPE = Literal["Workflow"]
-
-
 class WorkflowSchema(Workflow):
     """Override workflow model fields for generate JSON schema file."""
 
-    type: WORKFLOW_TYPE = Field(description="A type of workflow template.")
+    type: Literal["Workflow"] = Field(
+        description="A type of workflow template that should be `Workflow`."
+    )
     name: Optional[str] = Field(default=None, description="A workflow name.")
     params: dict[str, Union[Param, str]] = Field(
         default_factory=dict,

{ddeutil_workflow-0.0.80 → ddeutil_workflow-0.0.82}/src/ddeutil/workflow/api/routes/job.py

@@ -15,7 +15,7 @@ from fastapi.responses import UJSONResponse
 from ...__types import DictData
 from ...errors import JobError
 from ...job import Job
-from ...traces import TraceManager, get_trace
+from ...traces import Trace, get_trace
 from ...utils import gen_id
 
 logger = logging.getLogger("uvicorn.error")
@@ -41,7 +41,7 @@ async def job_execute(
     if extras:
         job.extras = extras
 
-    trace: TraceManager = get_trace(
+    trace: Trace = get_trace(
         run_id, parent_run_id=parent_run_id, extras=job.extras
     )
 

{ddeutil_workflow-0.0.80 → ddeutil_workflow-0.0.82}/src/ddeutil/workflow/api/routes/logs.py

@@ -3,7 +3,7 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
-"""This route include audit and trace log paths."""
+"""This route include audit log path."""
 from __future__ import annotations
 
 from fastapi import APIRouter, Path, Query
@@ -11,7 +11,6 @@ from fastapi import status as st
 from fastapi.responses import UJSONResponse
 
 from ...audits import get_audit
-from ...result import Result
 
 router = APIRouter(
     prefix="/logs",
@@ -20,63 +19,6 @@ router = APIRouter(
 )
 
 
-@router.get(
-    path="/traces/",
-    response_class=UJSONResponse,
-    status_code=st.HTTP_200_OK,
-    summary="Read all trace logs.",
-    tags=["trace"],
-)
-async def get_traces(
-    offset: int = Query(default=0, gt=0),
-    limit: int = Query(default=100, gt=0),
-):
-    """Return all trace logs from the current trace log path that config with
-    `WORKFLOW_LOG_PATH` environment variable name.
-    """
-    result = Result()
-    return {
-        "message": (
-            f"Getting trace logs with offset: {offset} and limit: {limit}"
-        ),
-        "traces": [
-            trace.model_dump(
-                by_alias=True,
-                exclude_none=True,
-                exclude_unset=True,
-            )
-            for trace in result.trace.find_traces()
-        ],
-    }
-
-
-@router.get(
-    path="/traces/{run_id}",
-    response_class=UJSONResponse,
-    status_code=st.HTTP_200_OK,
-    summary="Read trace log with specific running ID.",
-    tags=["trace"],
-)
-async def get_trace_with_id(run_id: str):
-    """Return trace log with specific running ID from the current trace log path
-    that config with `WORKFLOW_LOG_PATH` environment variable name.
-
-    - **run_id**: A running ID that want to search a trace log from the log
-        path.
-    """
-    result = Result()
-    return {
-        "message": f"Getting trace log with specific running ID: {run_id}",
-        "trace": (
-            result.trace.find_trace_with_id(run_id).model_dump(
-                by_alias=True,
-                exclude_none=True,
-                exclude_unset=True,
-            )
-        ),
-    }
-
-
 @router.get(
     path="/audits/",
     response_class=UJSONResponse,
@@ -84,12 +26,17 @@ async def get_trace_with_id(run_id: str):
     summary="Read all audit logs.",
     tags=["audit"],
 )
-async def get_audits():
+async def get_audits(
+    offset: int = Query(default=0, gt=0),
+    limit: int = Query(default=100, gt=0),
+):
     """Return all audit logs from the current audit log path that config with
     `WORKFLOW_AUDIT_URL` environment variable name.
     """
     return {
-        "message": "Getting audit logs",
+        "message": (
+            f"Getting audit logs with offset: {offset} and limit: {limit}",
+        ),
         "audits": list(get_audit().find_audits(name="demo")),
     }
 

{ddeutil_workflow-0.0.80 → ddeutil_workflow-0.0.82}/src/ddeutil/workflow/audits.py

@@ -49,33 +49,65 @@ import zlib
 from abc import ABC, abstractmethod
 from collections.abc import Iterator
 from datetime import datetime, timedelta
+from enum import Enum
 from pathlib import Path
 from typing import Annotated, Any, ClassVar, Literal, Optional, Union
 from urllib.parse import ParseResult, urlparse
 
-from pydantic import BaseModel, Field, TypeAdapter
+from pydantic import BaseModel, ConfigDict, Field, TypeAdapter
 from pydantic.functional_validators import field_validator, model_validator
 from typing_extensions import Self
 
 from .__types import DictData
 from .conf import dynamic
-from .traces import TraceManager, get_trace, set_logging
+from .traces import Trace, get_trace, set_logging
 
 logger = logging.getLogger("ddeutil.workflow")
 
 
+class ReleaseType(str, Enum):
+    """Release type enumeration for workflow execution modes.
+
+    This enum defines the different types of workflow releases that can be
+    triggered, each with specific behavior and use cases.
+
+    Attributes:
+        NORMAL: Standard workflow release execution
+        RERUN: Re-execution of previously failed workflow
+        EVENT: Event-triggered workflow execution
+        FORCE: Forced execution bypassing normal conditions
+    """
+
+    NORMAL = "normal"
+    RERUN = "rerun"
+    EVENT = "event"
+    FORCE = "force"
+
+
+NORMAL = ReleaseType.NORMAL
+RERUN = ReleaseType.RERUN
+EVENT = ReleaseType.EVENT
+FORCE = ReleaseType.FORCE
+
+
 class AuditData(BaseModel):
+    """Audit Data model."""
+
+    model_config = ConfigDict(use_enum_values=True)
+
     name: str = Field(description="A workflow name.")
     release: datetime = Field(description="A release datetime.")
-    type: str = Field(description="A running type before logging.")
+    type: ReleaseType = Field(
+        default=NORMAL, description="A running type before logging."
+    )
     context: DictData = Field(
         default_factory=dict,
         description="A context that receive from a workflow execution result.",
     )
+    run_id: str = Field(description="A running ID")
     parent_run_id: Optional[str] = Field(
         default=None, description="A parent running ID."
     )
-    run_id: str = Field(description="A running ID")
     runs_metadata: DictData = Field(
         default_factory=dict,
         description="A runs metadata that will use to tracking this audit log.",
@@ -122,7 +154,7 @@ class BaseAudit(BaseModel, ABC):
     @abstractmethod
     def is_pointed(
         self,
-        data: AuditData,
+        data: Any,
         *,
         extras: Optional[DictData] = None,
     ) -> bool:
@@ -328,21 +360,21 @@ class FileAudit(BaseAudit):
             return AuditData.model_validate(obj=json.load(f))
 
     def is_pointed(
-        self, data: AuditData, *, extras: Optional[DictData] = None
+        self,
+        data: Any,
+        *,
+        extras: Optional[DictData] = None,
     ) -> bool:
         """Check if the release log already exists at the destination log path.
 
         Args:
-            data: The workflow name.
+            data (str):
             extras: Optional extra parameters to override core config.
 
         Returns:
             bool: True if the release log exists, False otherwise.
         """
-        # NOTE: Return False if enable writing log flag does not set.
-        if not dynamic("enable_write_audit", extras=extras):
-            return False
-        return self.pointer(data).exists()
+        return self.pointer(AuditData.model_validate(data)).exists()
 
     def pointer(self, data: AuditData) -> Path:
         """Return release directory path generated from model data.
@@ -365,7 +397,7 @@ class FileAudit(BaseAudit):
             Self: The audit instance after saving.
         """
         audit = AuditData.model_validate(data)
-        trace: TraceManager = get_trace(
+        trace: Trace = get_trace(
             audit.run_id,
             parent_run_id=audit.parent_run_id,
             extras=self.extras,
@@ -655,7 +687,7 @@ class SQLiteAudit(BaseAudit):  # pragma: no cov
             ValueError: If SQLite database is not properly configured.
         """
         audit = AuditData.model_validate(data)
-        trace: TraceManager = get_trace(
+        trace: Trace = get_trace(
             audit.run_id,
             parent_run_id=audit.parent_run_id,
             extras=self.extras,
@@ -748,10 +780,7 @@ Audit = Annotated[
 ]
 
 
-def get_audit(
-    *,
-    extras: Optional[DictData] = None,
-) -> Audit:  # pragma: no cov
+def get_audit(extras: Optional[DictData] = None) -> Audit:  # pragma: no cov
     """Get an audit model dynamically based on the config audit path value.
 
     Args:

{ddeutil_workflow-0.0.80 → ddeutil_workflow-0.0.82}/src/ddeutil/workflow/conf.py

@@ -22,19 +22,6 @@ Functions:
     pass_env: Process environment variable substitution
     api_config: Get API-specific configuration settings
 
-Example:
-    ```python
-    from ddeutil.workflow.conf import Config, YamlParser
-
-    # Load workflow configuration
-    parser = YamlParser("my-workflow")
-    workflow_config = parser.data
-
-    # Access dynamic configuration
-    from ddeutil.workflow.conf import dynamic
-    log_level = dynamic("log_level", default="INFO")
-    ```
-
 Note:
     Configuration files support environment variable substitution using
     ${VAR_NAME} syntax and provide extensive validation capabilities.
@@ -155,7 +142,7 @@ class Config:  # pragma: no cov
         )
 
     @property
-    def audit_conf(self) -> str:
+    def audit_conf(self) -> dict[str, Any]:
         return json.loads(
             env("LOG_AUDIT_URL", '{"type": "file", "path": "./audits"}')
         )
@@ -189,17 +176,6 @@ class YamlParser:
     """Base Load object that use to search config data by given some identity
     value like name of `Workflow` or `Crontab` templates.
 
-    :param name: (str) A name of key of config data that read with YAML
-        Environment object.
-    :param path: (Path) A config path object.
-    :param externals: (DictData) An external config data that want to add to
-        loaded config data.
-    :param extras: (DictDdata) An extra parameters that use to override core
-        config values.
-
-    :raise ValueError: If the data does not find on the config path with the
-        name parameter.
-
     Noted:
         The config data should have `type` key for modeling validation that
     make this loader know what is config should to do pass to.
@@ -222,6 +198,23 @@ class YamlParser:
         extras: Optional[DictData] = None,
         obj: Optional[Union[object, str]] = None,
     ) -> None:
+        """Main constructure function.
+
+        Args:
+            name (str): A name of key of config data that read with YAML
+                Environment object.
+            path (Path): A config path object.
+            externals (DictData): An external config data that want to add to
+                loaded config data.
+            extras (DictDdata): An extra parameters that use to override core
+                config values.
+            obj (object | str): An object that want to validate from the `type`
+                key before keeping the config data.
+
+        Raises:
+            ValueError: If the data does not find on the config path with the
+                name parameter.
+        """
         self.path: Path = Path(dynamic("conf_path", f=path, extras=extras))
         self.externals: DictData = externals or {}
         self.extras: DictData = extras or {}
@@ -255,17 +248,19 @@ class YamlParser:
         """Find data with specific key and return the latest modify date data if
         this key exists multiple files.
 
-        :param name: (str) A name of data that want to find.
-        :param path: (Path) A config path object.
-        :param paths: (list[Path]) A list of config path object.
-        :param obj: (object | str) An object that want to validate matching
-            before return.
-        :param extras: (DictData)  An extra parameter that use to override core
-            config values.
-        :param ignore_filename: (str) An ignore filename. Default is
-            ``.confignore`` filename.
+        Args:
+            name (str): A name of data that want to find.
+            path (Path): A config path object.
+            paths (list[Path]): A list of config path object.
+            obj (object | str): An object that want to validate matching
+                before return.
+            extras (DictData):  An extra parameter that use to override core
+                config values.
+            ignore_filename (str): An ignore filename. Default is
+                ``.confignore`` filename.
 
-        :rtype: DictData
+        Returns:
+            DictData: A config data that was found on the searching paths.
         """
         path: Path = dynamic("conf_path", f=path, extras=extras)
         if not paths:
@@ -288,9 +283,12 @@ class YamlParser:
                     continue
 
                 if data := cls.filter_yaml(file, name=name):
+
+                    # NOTE: Start adding file metadata.
                     file_stat: os.stat_result = file.lstat()
                     data["created_at"] = file_stat.st_ctime
                     data["updated_at"] = file_stat.st_mtime
+
                     if not obj_type:
                         all_data.append((file_stat.st_mtime, data))
                     elif (t := data.get("type")) and t == obj_type:
@@ -324,11 +322,12 @@ class YamlParser:
             extras: (DictData) An extra parameter that use to override core
                 config values.
             ignore_filename: (str) An ignore filename. Default is
-            ``.confignore`` filename.
-            tags: (list[str])
-                A list of tag that want to filter.
+                ``.confignore`` filename.
+            tags (list[str]): A list of tag that want to filter.
 
-        :rtype: Iterator[tuple[str, DictData]]
+        Returns:
+            Iterator[tuple[str, DictData]]: An iterator of config data that was
+                found on the searching paths.
         """
         excluded: list[str] = excluded or []
         tags: list[str] = tags or []
@@ -364,7 +363,12 @@ class YamlParser:
                     ):
                         continue
 
-                    if (t := data.get("type")) and t == obj_type:
+                    if (
+                        # isinstance(data, dict) and
+                        (t := data.get("type"))
+                        and t == obj_type
+                    ):
+                        # NOTE: Start adding file metadata.
                         file_stat: os.stat_result = file.lstat()
                         data["created_at"] = file_stat.st_ctime
                         data["updated_at"] = file_stat.st_mtime
@@ -372,6 +376,7 @@ class YamlParser:
                             file.lstat().st_mtime,
                             data,
                         )
+
                         if key in all_data:
                             all_data[key].append(marking)
                         else:
@@ -405,15 +410,30 @@ class YamlParser:
     def filter_yaml(cls, file: Path, name: Optional[str] = None) -> DictData:
         """Read a YAML file context from an input file path and specific name.
 
-        :param file: (Path) A file path that want to extract YAML context.
-        :param name: (str) A key name that search on a YAML context.
+        Notes:
+            The data that will return from reading context will map with config
+            name if an input searching name does not pass to this function.
+
+                input: {"name": "foo", "type": "Some"}
+                output: {"foo": {"name": "foo", "type": "Some"}}
 
-        :rtype: DictData
+        Args:
+            file (Path): A file path that want to extract YAML context.
+            name (str): A key name that search on a YAML context.
+
+        Returns:
+            DictData: A data that read from this file if it is YAML format.
         """
         if any(file.suffix.endswith(s) for s in (".yml", ".yaml")):
             values: DictData = YamlFlResolve(file).read()
             if values is not None:
-                return values.get(name, {}) if name else values
+                if name:
+                    if "name" in values and values.get("name") == name:
+                        return values
+                    return (
+                        values[name] | {"name": name} if name in values else {}
+                    )
+                return {values["name"]: values} if "name" in values else values
         return {}
 
     @cached_property

{ddeutil_workflow-0.0.80 → ddeutil_workflow-0.0.82}/src/ddeutil/workflow/errors.py

@@ -135,11 +135,12 @@ class BaseError(Exception):
 
         Example:
             >>> error = BaseError("Something failed", refs="stage-1")
-            >>> # Simple format
+
+            Simple format
             >>> error.to_dict()
             >>> # Returns: {"name": "BaseError", "message": "Something failed"}
 
-            >>> # With reference mapping
+            With reference mapping
             >>> error.to_dict(with_refs=True)
             >>> # Returns: {"stage-1": {"name": "BaseError", "message": "Something failed"}}
             ```
@@ -165,6 +166,15 @@ class StageCancelError(StageError): ...
 class StageSkipError(StageError): ...
 
 
+class StageNestedError(StageError): ...
+
+
+class StageNestedCancelError(StageNestedError): ...
+
+
+class StageNestedSkipError(StageNestedError): ...
+
+
 class JobError(BaseError): ...