ddeutil-workflow 0.0.37__tar.gz → 0.0.39__tar.gz

{ddeutil_workflow-0.0.37 → ddeutil_workflow-0.0.39}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: ddeutil-workflow
-Version: 0.0.37
+Version: 0.0.39
 Summary: Lightweight workflow orchestration
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT
@@ -31,6 +31,10 @@ Provides-Extra: api
 Requires-Dist: fastapi<1.0.0,>=0.115.0; extra == "api"
 Requires-Dist: httpx; extra == "api"
 Requires-Dist: ujson; extra == "api"
+Provides-Extra: async
+Requires-Dist: aiofiles; extra == "async"
+Requires-Dist: aiohttp; extra == "async"
+Dynamic: license-file
 
 # Workflow Orchestration
 
@@ -126,7 +130,7 @@ flowchart LR
 > - [Google **Workflows**](https://cloud.google.com/workflows)
 > - [AWS **Step Functions**](https://aws.amazon.com/step-functions/)
 
-## :round_pushpin: Installation
+## 📦 Installation
 
 This project need `ddeutil` and `ddeutil-io` extension namespace packages.
 If you want to install this package with application add-ons, you should add
@@ -165,6 +169,8 @@ run-py-local:
       run-date: datetime
    jobs:
       getting-api-data:
+         runs-on:
+            type: local
          stages:
             - name: "Retrieve API Data"
               id: retrieve-api

{ddeutil_workflow-0.0.37 → ddeutil_workflow-0.0.39}/README.md

@@ -92,7 +92,7 @@ flowchart LR
 > - [Google **Workflows**](https://cloud.google.com/workflows)
 > - [AWS **Step Functions**](https://aws.amazon.com/step-functions/)
 
-## :round_pushpin: Installation
+## 📦 Installation
 
 This project need `ddeutil` and `ddeutil-io` extension namespace packages.
 If you want to install this package with application add-ons, you should add
@@ -131,6 +131,8 @@ run-py-local:
       run-date: datetime
    jobs:
       getting-api-data:
+         runs-on:
+            type: local
          stages:
             - name: "Retrieve API Data"
               id: retrieve-api

{ddeutil_workflow-0.0.37 → ddeutil_workflow-0.0.39}/pyproject.toml

@@ -40,6 +40,10 @@ api = [
     "httpx",
     "ujson",
 ]
+async = [
+    "aiofiles",
+    "aiohttp",
+]
 
 [project.urls]
 Homepage = "https://github.com/ddeutils/ddeutil-workflow/"
@@ -67,6 +71,7 @@ source = ["src.ddeutil.workflow"]
 omit = [
     "src/ddeutil/workflow/__about__.py",
     "src/ddeutil/workflow/__cron.py",
+    "src/ddeutil/workflow/context.py",
     "src/ddeutil/workflow/api/__init__.py",
     "src/ddeutil/workflow/api/api.py",
     "src/ddeutil/workflow/api/log.py",

ddeutil_workflow-0.0.39/src/ddeutil/workflow/__about__.py

@@ -0,0 +1 @@
+__version__: str = "0.0.39"

{ddeutil_workflow-0.0.37 → ddeutil_workflow-0.0.39}/src/ddeutil/workflow/__init__.py

@@ -39,6 +39,8 @@ from .job import (
     Job,
     RunsOn,
     Strategy,
+    local_execute,
+    local_execute_strategy,
 )
 from .logs import (
     TraceData,
@@ -69,6 +71,8 @@ from .stages import (
     BashStage,
     CallStage,
     EmptyStage,
+    ForEachStage,
+    ParallelStage,
     PyStage,
     Stage,
     TriggerStage,
@@ -89,7 +93,6 @@ from .templates import (
 from .utils import (
     batch,
     cross_product,
-    dash2underscore,
     delay,
     filter_func,
     gen_id,

{ddeutil_workflow-0.0.37 → ddeutil_workflow-0.0.39}/src/ddeutil/workflow/__types.py

@@ -61,8 +61,10 @@ class Re:
     #   Regular expression:
     #       - Version 1:
     #         \${{\s*(?P<caller>[a-zA-Z0-9_.\s'\"\[\]\(\)\-\{}]+?)\s*(?P<post_filters>(?:\|\s*(?:[a-zA-Z0-9_]{3,}[a-zA-Z0-9_.,-\\%\s'\"[\]()\{}]+)\s*)*)}}
+    #
     #       - Version 2: (2024-09-30):
     #         \${{\s*(?P<caller>(?P<caller_prefix>(?:[a-zA-Z_-]+\.)*)(?P<caller_last>[a-zA-Z0-9_\-.'\"(\)[\]{}]+))\s*(?P<post_filters>(?:\|\s*(?:[a-zA-Z0-9_]{3,}[a-zA-Z0-9_.,-\\%\s'\"[\]()\{}]+)\s*)*)}}
+    #
     #       - Version 3: (2024-10-05):
     #         \${{\s*(?P<caller>(?P<caller_prefix>(?:[a-zA-Z_-]+\??\.)*)(?P<caller_last>[a-zA-Z0-9_\-.'\"(\)[\]{}]+\??))\s*(?P<post_filters>(?:\|\s*(?:[a-zA-Z0-9_]{3,}[a-zA-Z0-9_.,-\\%\s'\"[\]()\{}]+)\s*)*)}}
     #

{ddeutil_workflow-0.0.37 → ddeutil_workflow-0.0.39}/src/ddeutil/workflow/api/routes/job.py

@@ -45,6 +45,7 @@ async def job_execute(
         run_id=result.run_id,
         parent_run_id=result.parent_run_id,
     )
+    context: DictData = {}
     try:
         job.set_outputs(
             job.execute(
@@ -52,7 +53,7 @@ async def job_execute(
                 run_id=rs.run_id,
                 parent_run_id=rs.parent_run_id,
             ).context,
-            to=params,
+            to=context,
         )
     except JobException as err:
         rs.trace.error(f"[WORKFLOW]: {err.__class__.__name__}: {err}")
@@ -70,4 +71,5 @@ async def job_execute(
             exclude_defaults=True,
         ),
         "params": params,
+        "context": context,
     }

{ddeutil_workflow-0.0.37 → ddeutil_workflow-0.0.39}/src/ddeutil/workflow/api/routes/logs.py

@@ -6,7 +6,7 @@
 """This route include audit and trace log paths."""
 from __future__ import annotations
 
-from fastapi import APIRouter
+from fastapi import APIRouter, Path, Query
 from fastapi import status as st
 from fastapi.responses import UJSONResponse
 
@@ -27,12 +27,17 @@ log_route = APIRouter(
     summary="Read all trace logs.",
     tags=["trace"],
 )
-async def get_traces():
+async def get_traces(
+    offset: int = Query(default=0, gt=0),
+    limit: int = Query(default=100, gt=0),
+):
     """Return all trace logs from the current trace log path that config with
     `WORKFLOW_LOG_PATH` environment variable name.
     """
     return {
-        "message": "Getting trace logs",
+        "message": (
+            f"Getting trace logs with offset: {offset} and limit: {limit}"
+        ),
         "traces": [
             trace.model_dump(
                 by_alias=True,
@@ -117,7 +122,10 @@ async def get_audit_with_workflow(workflow: str):
     summary="Read all audit logs with specific workflow name and release date.",
     tags=["audit"],
 )
-async def get_audit_with_workflow_release(workflow: str, release: str):
+async def get_audit_with_workflow_release(
+    workflow: str = Path(...),
+    release: str = Path(...),
+):
     """Return all audit logs with specific workflow name and release date from
     the current audit log path that config with `WORKFLOW_AUDIT_PATH`
     environment variable name.

{ddeutil_workflow-0.0.37 → ddeutil_workflow-0.0.39}/src/ddeutil/workflow/audit.py

@@ -20,7 +20,7 @@ from typing_extensions import Self
 
 from .__types import DictData, TupleStr
 from .conf import config
-from .logs import TraceLog, get_trace
+from .logs import TraceLog, get_dt_tznow, get_trace
 
 __all__: TupleStr = (
     "get_audit",
@@ -43,10 +43,12 @@ class BaseAudit(BaseModel, ABC):
         default_factory=dict,
         description="A context that receive from a workflow execution result.",
     )
-    parent_run_id: Optional[str] = Field(default=None)
-    run_id: str
-    update: datetime = Field(default_factory=datetime.now)
-    execution_time: float = Field(default=0)
+    parent_run_id: Optional[str] = Field(
+        default=None, description="A parent running ID."
+    )
+    run_id: str = Field(description="A running ID")
+    update: datetime = Field(default_factory=get_dt_tznow)
+    execution_time: float = Field(default=0, description="An execution time.")
 
     @model_validator(mode="after")
     def __model_action(self) -> Self:

{ddeutil_workflow-0.0.37 → ddeutil_workflow-0.0.39}/src/ddeutil/workflow/caller.py

@@ -26,6 +26,7 @@ T = TypeVar("T")
 P = ParamSpec("P")
 
 logger = logging.getLogger("ddeutil.workflow")
+logging.getLogger("asyncio").setLevel(logging.INFO)
 
 
 class TagFunc(Protocol):
@@ -47,9 +48,10 @@ def tag(
     """Tag decorator function that set function attributes, ``tag`` and ``name``
     for making registries variable.
 
-    :param: name: A tag name for make different use-case of a function.
-    :param: alias: A alias function name that keeping in registries. If this
-        value does not supply, it will use original function name from __name__.
+    :param: name: (str) A tag name for make different use-case of a function.
+    :param: alias: (str) A alias function name that keeping in registries.
+        If this value does not supply, it will use original function name
+        from `__name__` argument.
 
     :rtype: Callable[P, TagFunc]
     """
@@ -60,10 +62,13 @@ def tag(
 
         @wraps(func)
         def wrapped(*args: P.args, **kwargs: P.kwargs) -> TagFunc:
-            # NOTE: Able to do anything before calling the call function.
             return func(*args, **kwargs)
 
-        return wrapped
+        @wraps(func)
+        async def awrapped(*args: P.args, **kwargs: P.kwargs) -> TagFunc:
+            return await func(*args, **kwargs)
+
+        return awrapped if inspect.iscoroutinefunction(func) else wrapped
 
     return func_internal
 
@@ -74,7 +79,7 @@ Registry = dict[str, Callable[[], TagFunc]]
 def make_registry(submodule: str) -> dict[str, Registry]:
     """Return registries of all functions that able to called with task.
 
-    :param submodule: A module prefix that want to import registry.
+    :param submodule: (str) A module prefix that want to import registry.
 
     :rtype: dict[str, Registry]
     """
@@ -130,12 +135,7 @@ def extract_call(call: str) -> Callable[[], TagFunc]:
     """Extract Call function from string value to call partial function that
     does run it at runtime.
 
-    :raise NotImplementedError: When the searching call's function result does
-        not exist in the registry.
-    :raise NotImplementedError: When the searching call's tag result does not
-        exist in the registry with its function key.
-
-    :param call: A call value that able to match with Task regex.
+    :param call: (str) A call value that able to match with Task regex.
 
         The format of call value should contain 3 regular expression groups
     which match with the below config format:
@@ -148,6 +148,11 @@ def extract_call(call: str) -> Callable[[], TagFunc]:
         >>> extract_call("tasks/return-type-not-valid@raise")
         ...
 
+    :raise NotImplementedError: When the searching call's function result does
+        not exist in the registry.
+    :raise NotImplementedError: When the searching call's tag result does not
+        exist in the registry with its function key.
+
     :rtype: Callable[[], TagFunc]
     """
     if not (found := Re.RE_TASK_FMT.search(call)):

ddeutil_workflow-0.0.39/src/ddeutil/workflow/context.py

@@ -0,0 +1,61 @@
+from __future__ import annotations
+
+from typing import Optional, Union
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from .__types import DictData
+
+
+class ErrorContext(BaseModel):  # pragma: no cov
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    obj: Exception = Field(alias="class")
+    name: str = Field(description="A name of exception class.")
+    message: str = Field(description="A exception message.")
+
+
+class OutputContext(BaseModel):  # pragma: no cov
+    outputs: DictData = Field(default_factory=dict)
+    errors: Optional[ErrorContext] = Field(default=None)
+    skipped: bool = Field(default=False)
+
+    def is_exception(self) -> bool:
+        return self.errors is not None
+
+
+class StageContext(BaseModel):  # pragma: no cov
+    stages: dict[str, OutputContext]
+    errors: Optional[ErrorContext] = Field(default=None)
+
+    def is_exception(self) -> bool:
+        return self.errors is not None
+
+
+class MatrixContext(StageContext):  # pragma: no cov
+    matrix: DictData = Field(default_factory=dict)
+
+
+MatrixStageContext = dict[
+    str, Union[MatrixContext, StageContext]
+]  # pragma: no cov
+
+
+class StrategyContext(BaseModel):  # pragma: no cov
+    strategies: MatrixStageContext
+    errors: Optional[ErrorContext] = Field(default=None)
+
+    def is_exception(self) -> bool:
+        return self.errors is not None
+
+
+StrategyMatrixContext = Union[
+    StrategyContext, MatrixStageContext
+]  # pragma: no cov
+
+
+class JobContext(BaseModel):  # pragma: no cov
+    params: DictData = Field(description="A parameterize value")
+    jobs: dict[str, StrategyMatrixContext]
+    errors: Optional[ErrorContext] = Field(default=None)
+    skipped: bool = Field(default=False)

{ddeutil_workflow-0.0.37 → ddeutil_workflow-0.0.39}/src/ddeutil/workflow/exceptions.py

@@ -9,8 +9,21 @@ annotate for handle error only.
 """
 from __future__ import annotations
 
+from typing import Any
 
-class BaseWorkflowException(Exception): ...
+
+def to_dict(exception: Exception) -> dict[str, Any]:  # pragma: no cov
+    return {
+        "class": exception,
+        "name": exception.__class__.__name__,
+        "message": str(exception),
+    }
+
+
+class BaseWorkflowException(Exception):
+
+    def to_dict(self) -> dict[str, Any]:
+        return to_dict(self)
 
 
 class UtilException(BaseWorkflowException): ...