ddeutil-workflow 0.0.63__tar.gz → 0.0.64__tar.gz

{ddeutil_workflow-0.0.63 → ddeutil_workflow-0.0.64}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddeutil-workflow
-Version: 0.0.63
+Version: 0.0.64
 Summary: Lightweight workflow orchestration
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT
@@ -221,18 +221,18 @@ value (This config can override by extra parameters with `registry_caller` key).
 > engine will auto use the `model_validate` method before run your caller function.
 
 ```python
-from ddeutil.workflow import Result, WorkflowSecret, tag
+from ddeutil.workflow import Result, CallerSecret, tag
 from ddeutil.workflow.exceptions import StageException
 from pydantic import BaseModel
 
 class AwsCredential(BaseModel):
     path: str
     access_client_id: str
-    access_client_secret: WorkflowSecret
+    access_client_secret: CallerSecret
 
 class RestAuth(BaseModel):
     type: str
-    keys: WorkflowSecret
+    keys: CallerSecret
 
 @tag("requests", alias="get-api-with-oauth-to-s3")
 def get_api_with_oauth_to_s3(
@@ -265,45 +265,6 @@ result: Result = workflow.execute(
 )
 ```
 
-> [!NOTE]
-> So, this package provide the `Schedule` template for this action, and you can
-> pass the parameters dynamically for changing align with that running time by
-> the `release` prefix.
->
-> ```yaml
-> schedule-run-local-wf:
->
->    # Validate model that use to parsing exists for template file
->    type: Schedule
->    workflows:
->
->       # Map existing workflow that want to deploy with scheduler application.
->       # It allows you to pass release parameter that dynamic change depend on the
->       # current context of this scheduler application releasing that time.
->       - name: run-py-local
->         params:
->           source-extract: "USD-THB"
->           run-date: "${{ release.logical_date }}"
-> ```
->
-> The main method of the `Schedule` model that use to running is `pending`. If you
-> do not pass the `stop` date on this method, it will use config with
-> `WORKFLOW_APP_STOP_BOUNDARY_DELTA` key for generate this stop date.
->
-> ```python
-> from ddeutil.workflow import Schedule
->
-> (
->    Schedule
->    .from_conf("schedule-run-local-wf")
->    .pending(stop=None)
-> )
-> ```
-
-> [!WARNING]
-> The scheduler feature is the expensive feature of this project. You should
-> avoid to use it and find a scheduler tool instead.
-
 ## :cookie: Configuration
 
 The main configuration that use to dynamic changing this workflow engine for your
@@ -333,19 +294,6 @@ it will use default value and do not raise any error to you.
 | **TRACE_ENABLE_WRITE**       |    Log    | `false`                                                                                                                         |                                                                                                                    |
 | **AUDIT_PATH**               |    Log    | `./audits`                                                                                                                      |                                                                                                                    |
 | **AUDIT_ENABLE_WRITE**       |    Log    | `true`                                                                                                                          | A flag that enable logging object saving log to its destination.                                                   |
-| **MAX_PROCESS**              |    App    | `2`                                                                                                                             | The maximum process worker number that run in scheduler app module.                                                |
-| **MAX_SCHEDULE_PER_PROCESS** |    App    | `100`                                                                                                                           | A schedule per process that run parallel.                                                                          |
-| **STOP_BOUNDARY_DELTA**      |    App    | `'{"minutes": 5, "seconds": 20}'`                                                                                               | A time delta value that use to stop scheduler app in json string format.                                           |
-
-**API Application**:
-
-This config part use for the workflow application that build from the FastAPI
-only.
-
-| Environment                |  Component  | Default | Description                                                                        |
-|:---------------------------|:-----------:|---------|------------------------------------------------------------------------------------|
-| **ENABLE_ROUTE_WORKFLOW**  |     API     | `true`  | A flag that enable workflow route to manage execute manually and workflow logging. |
-| **ENABLE_ROUTE_SCHEDULE**  |     API     | `true`  | A flag that enable run scheduler.                                                  |
 
 ## :rocket: Deployment
 

{ddeutil_workflow-0.0.63 → ddeutil_workflow-0.0.64}/README.md

@@ -170,18 +170,18 @@ value (This config can override by extra parameters with `registry_caller` key).
 > engine will auto use the `model_validate` method before run your caller function.
 
 ```python
-from ddeutil.workflow import Result, WorkflowSecret, tag
+from ddeutil.workflow import Result, CallerSecret, tag
 from ddeutil.workflow.exceptions import StageException
 from pydantic import BaseModel
 
 class AwsCredential(BaseModel):
     path: str
     access_client_id: str
-    access_client_secret: WorkflowSecret
+    access_client_secret: CallerSecret
 
 class RestAuth(BaseModel):
     type: str
-    keys: WorkflowSecret
+    keys: CallerSecret
 
 @tag("requests", alias="get-api-with-oauth-to-s3")
 def get_api_with_oauth_to_s3(
@@ -214,45 +214,6 @@ result: Result = workflow.execute(
 )
 ```
 
-> [!NOTE]
-> So, this package provide the `Schedule` template for this action, and you can
-> pass the parameters dynamically for changing align with that running time by
-> the `release` prefix.
->
-> ```yaml
-> schedule-run-local-wf:
->
->    # Validate model that use to parsing exists for template file
->    type: Schedule
->    workflows:
->
->       # Map existing workflow that want to deploy with scheduler application.
->       # It allows you to pass release parameter that dynamic change depend on the
->       # current context of this scheduler application releasing that time.
->       - name: run-py-local
->         params:
->           source-extract: "USD-THB"
->           run-date: "${{ release.logical_date }}"
-> ```
->
-> The main method of the `Schedule` model that use to running is `pending`. If you
-> do not pass the `stop` date on this method, it will use config with
-> `WORKFLOW_APP_STOP_BOUNDARY_DELTA` key for generate this stop date.
->
-> ```python
-> from ddeutil.workflow import Schedule
->
-> (
->    Schedule
->    .from_conf("schedule-run-local-wf")
->    .pending(stop=None)
-> )
-> ```
-
-> [!WARNING]
-> The scheduler feature is the expensive feature of this project. You should
-> avoid to use it and find a scheduler tool instead.
-
 ## :cookie: Configuration
 
 The main configuration that use to dynamic changing this workflow engine for your
@@ -282,19 +243,6 @@ it will use default value and do not raise any error to you.
 | **TRACE_ENABLE_WRITE**       |    Log    | `false`                                                                                                                         |                                                                                                                    |
 | **AUDIT_PATH**               |    Log    | `./audits`                                                                                                                      |                                                                                                                    |
 | **AUDIT_ENABLE_WRITE**       |    Log    | `true`                                                                                                                          | A flag that enable logging object saving log to its destination.                                                   |
-| **MAX_PROCESS**              |    App    | `2`                                                                                                                             | The maximum process worker number that run in scheduler app module.                                                |
-| **MAX_SCHEDULE_PER_PROCESS** |    App    | `100`                                                                                                                           | A schedule per process that run parallel.                                                                          |
-| **STOP_BOUNDARY_DELTA**      |    App    | `'{"minutes": 5, "seconds": 20}'`                                                                                               | A time delta value that use to stop scheduler app in json string format.                                           |
-
-**API Application**:
-
-This config part use for the workflow application that build from the FastAPI
-only.
-
-| Environment                |  Component  | Default | Description                                                                        |
-|:---------------------------|:-----------:|---------|------------------------------------------------------------------------------------|
-| **ENABLE_ROUTE_WORKFLOW**  |     API     | `true`  | A flag that enable workflow route to manage execute manually and workflow logging. |
-| **ENABLE_ROUTE_SCHEDULE**  |     API     | `true`  | A flag that enable run scheduler.                                                  |
 
 ## :rocket: Deployment
 

ddeutil_workflow-0.0.64/src/ddeutil/workflow/__about__.py

@@ -0,0 +1 @@
+__version__: str = "0.0.64"

{ddeutil_workflow-0.0.63 → ddeutil_workflow-0.0.64}/src/ddeutil/workflow/__init__.py

@@ -33,13 +33,6 @@ from .result import (
     Status,
 )
 from .reusables import *
-from .scheduler import (
-    Schedule,
-    ScheduleWorkflow,
-    schedule_control,
-    schedule_runner,
-    schedule_task,
-)
 from .stages import *
 from .utils import *
 from .workflow import *

ddeutil_workflow-0.0.64/src/ddeutil/workflow/api/__init__.py

@@ -0,0 +1,91 @@
+# ------------------------------------------------------------------------------
+# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
+# Licensed under the MIT License. See LICENSE in the project root for
+# license information.
+# ------------------------------------------------------------------------------
+from __future__ import annotations
+
+import contextlib
+from collections.abc import AsyncIterator
+
+from dotenv import load_dotenv
+from fastapi import FastAPI, Request
+from fastapi import status as st
+from fastapi.encoders import jsonable_encoder
+from fastapi.exceptions import RequestValidationError
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.middleware.gzip import GZipMiddleware
+from fastapi.responses import UJSONResponse
+
+from ..__about__ import __version__
+from ..conf import api_config
+from ..logs import get_logger
+from .routes import job, log, workflow
+
+load_dotenv()
+logger = get_logger("uvicorn.error")
+
+
+@contextlib.asynccontextmanager
+async def lifespan(_: FastAPI) -> AsyncIterator[dict[str, list]]:
+    """Lifespan function for the FastAPI application."""
+    yield {}
+
+
+app = FastAPI(
+    titile="Workflow",
+    description=(
+        "This is a workflow FastAPI application that use to manage manual "
+        "execute, logging, and schedule workflow via RestAPI."
+    ),
+    version=__version__,
+    lifespan=lifespan,
+    default_response_class=UJSONResponse,
+)
+app.add_middleware(GZipMiddleware, minimum_size=1000)
+origins: list[str] = [
+    "http://localhost",
+    "http://localhost:88",
+    "http://localhost:80",
+]
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=origins,
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+
+@app.get(path="/", response_class=UJSONResponse)
+async def health():
+    """Index view that not return any template without json status."""
+    return {"message": "Workflow already start up with healthy status."}
+
+
+# NOTE Add the jobs and logs routes by default.
+app.include_router(job, prefix=api_config.prefix_path)
+app.include_router(log, prefix=api_config.prefix_path)
+app.include_router(workflow, prefix=api_config.prefix_path)
+
+
+@app.exception_handler(RequestValidationError)
+async def validation_exception_handler(
+    request: Request, exc: RequestValidationError
+):
+    _ = request
+    return UJSONResponse(
+        status_code=st.HTTP_422_UNPROCESSABLE_ENTITY,
+        content=jsonable_encoder({"detail": exc.errors(), "body": exc.body}),
+    )
+
+
+if __name__ == "__main__":
+    import uvicorn
+
+    uvicorn.run(
+        app,
+        host="0.0.0.0",
+        port=80,
+        log_level="DEBUG",
+    )

{ddeutil_workflow-0.0.63 → ddeutil_workflow-0.0.64}/src/ddeutil/workflow/api/routes/__init__.py

@@ -5,5 +5,4 @@
 # ------------------------------------------------------------------------------
 from .job import job_route as job
 from .logs import log_route as log
-from .schedules import schedule_route as schedule
 from .workflows import workflow_route as workflow

{ddeutil_workflow-0.0.63 → ddeutil_workflow-0.0.64}/src/ddeutil/workflow/api/routes/job.py

@@ -69,7 +69,6 @@ async def job_execute(
             by_alias=True,
             exclude_none=False,
             exclude_unset=True,
-            exclude_defaults=True,
         ),
         "params": params,
         "context": context,

{ddeutil_workflow-0.0.63 → ddeutil_workflow-0.0.64}/src/ddeutil/workflow/api/routes/logs.py

@@ -44,7 +44,6 @@ async def get_traces(
                 by_alias=True,
                 exclude_none=True,
                 exclude_unset=True,
-                exclude_defaults=True,
             )
             for trace in result.trace.find_traces()
         ],
@@ -73,7 +72,6 @@ async def get_trace_with_id(run_id: str):
                 by_alias=True,
                 exclude_none=True,
                 exclude_unset=True,
-                exclude_defaults=True,
             )
         ),
     }

{ddeutil_workflow-0.0.63 → ddeutil_workflow-0.0.64}/src/ddeutil/workflow/api/routes/workflows.py

@@ -56,7 +56,6 @@ async def get_workflow_by_name(name: str) -> DictData:
         by_alias=True,
         exclude_none=False,
         exclude_unset=True,
-        exclude_defaults=True,
     )
 
 
@@ -99,7 +98,6 @@ async def get_workflow_audits(name: str):
                     by_alias=True,
                     exclude_none=False,
                     exclude_unset=True,
-                    exclude_defaults=True,
                 )
                 for audit in get_audit().find_audits(name=name)
             ],
@@ -133,6 +131,5 @@ async def get_workflow_release_audit(name: str, release: str):
             by_alias=True,
             exclude_none=False,
             exclude_unset=True,
-            exclude_defaults=True,
         ),
     }

{ddeutil_workflow-0.0.63 → ddeutil_workflow-0.0.64}/src/ddeutil/workflow/conf.py

@@ -6,11 +6,9 @@
 from __future__ import annotations
 
 import copy
-import json
 import os
 from abc import ABC, abstractmethod
 from collections.abc import Iterator
-from datetime import timedelta
 from functools import cached_property
 from inspect import isclass
 from pathlib import Path
@@ -20,7 +18,7 @@ from zoneinfo import ZoneInfo
 from ddeutil.core import str2bool
 from ddeutil.io import YamlFlResolve, search_env_replace
 from ddeutil.io.paths import glob_files, is_ignored, read_ignore
-from pydantic import SecretStr
+from pydantic import SecretStr, TypeAdapter
 
 from .__types import DictData
 
@@ -163,28 +161,6 @@ class Config:  # pragma: no cov
     def max_queue_complete_hist(self) -> int:
         return int(env("CORE_MAX_QUEUE_COMPLETE_HIST", "16"))
 
-    # NOTE: App
-    @property
-    def max_schedule_process(self) -> int:
-        return int(env("APP_MAX_PROCESS", "2"))
-
-    @property
-    def max_schedule_per_process(self) -> int:
-        return int(env("APP_MAX_SCHEDULE_PER_PROCESS", "100"))
-
-    @property
-    def stop_boundary_delta(self) -> timedelta:
-        stop_boundary_delta_str: str = env(
-            "APP_STOP_BOUNDARY_DELTA", '{"minutes": 5, "seconds": 20}'
-        )
-        try:
-            return timedelta(**json.loads(stop_boundary_delta_str))
-        except Exception as err:
-            raise ValueError(
-                "Config `WORKFLOW_APP_STOP_BOUNDARY_DELTA` can not parsing to"
-                f"timedelta with {stop_boundary_delta_str}."
-            ) from err
-
 
 class APIConfig:
     """API Config object."""
@@ -193,14 +169,6 @@ class APIConfig:
     def prefix_path(self) -> str:
         return env("API_PREFIX_PATH", "/api/v1")
 
-    @property
-    def enable_route_workflow(self) -> bool:
-        return str2bool(env("API_ENABLE_ROUTE_WORKFLOW", "true"))
-
-    @property
-    def enable_route_schedule(self) -> bool:
-        return str2bool(env("API_ENABLE_ROUTE_SCHEDULE", "true"))
-
 
 class BaseLoad(ABC):  # pragma: no cov
     """Base Load object is the abstraction object for any Load object that
@@ -496,7 +464,7 @@ def pass_env(value: T) -> T:  # pragma: no cov
     return None if rs == "null" else rs
 
 
-class WorkflowSecret(SecretStr):  # pragma: no cov
+class CallerSecret(SecretStr):  # pragma: no cov
     """Workflow Secret String model."""
 
     def get_secret_value(self) -> str:
@@ -506,3 +474,7 @@ class WorkflowSecret(SecretStr):  # pragma: no cov
         :rtype: str
         """
         return pass_env(super().get_secret_value())
+
+
+# NOTE: Define the caller secret type for use it directly in the caller func.
+CallerSecretType = TypeAdapter(CallerSecret)

{ddeutil_workflow-0.0.63 → ddeutil_workflow-0.0.64}/src/ddeutil/workflow/exceptions.py

@@ -83,6 +83,3 @@ class WorkflowException(BaseWorkflowException): ...
 
 
 class ParamValueException(WorkflowException): ...
-
-
-class ScheduleException(BaseWorkflowException): ...

{ddeutil_workflow-0.0.63 → ddeutil_workflow-0.0.64}/src/ddeutil/workflow/reusables.py

@@ -15,6 +15,7 @@ from datetime import datetime
 from functools import wraps
 from importlib import import_module
 from typing import (
+    Annotated,
     Any,
     Callable,
     Literal,
@@ -32,7 +33,8 @@ except ImportError:
 
 from ddeutil.core import getdot, import_string, lazy
 from ddeutil.io import search_env_replace
-from pydantic import BaseModel, create_model
+from pydantic import BaseModel, ConfigDict, Field, create_model
+from pydantic.alias_generators import to_pascal
 from pydantic.dataclasses import dataclass
 
 from .__types import DictData, Re
@@ -121,7 +123,7 @@ def make_filter_registry(
             if not (
                 hasattr(func, "filter")
                 and str(getattr(func, "mark", "NOT SET")) == "filter"
-            ):
+            ):  # pragma: no cov
                 continue
 
             func: FilterFunc
@@ -635,12 +637,25 @@ def extract_call(
     return rgt[call.func][call.tag]
 
 
+class BaseCallerArgs(BaseModel):  # pragma: no cov
+    """Base Caller Args model."""
+
+    model_config = ConfigDict(
+        arbitrary_types_allowed=True,
+        use_enum_values=True,
+    )
+
+
 def create_model_from_caller(func: Callable) -> BaseModel:  # pragma: no cov
     """Create model from the caller function. This function will use for
     validate the caller function argument typed-hint that valid with the args
     field.
 
-    :param func: A caller function.
+    Reference:
+        - https://github.com/lmmx/pydantic-function-models
+        - https://docs.pydantic.dev/1.10/usage/models/#dynamic-model-creation
+
+    :param func: (Callable) A caller function.
 
     :rtype: BaseModel
     """
@@ -649,16 +664,34 @@ def create_model_from_caller(func: Callable) -> BaseModel:  # pragma: no cov
     fields: dict[str, Any] = {}
     for name in sig.parameters:
         param: inspect.Parameter = sig.parameters[name]
+
+        # NOTE: Skip all `*args` and `**kwargs` parameters.
         if param.kind in (
             inspect.Parameter.VAR_KEYWORD,
             inspect.Parameter.VAR_POSITIONAL,
         ):
             continue
+
+        if name.startswith("_"):
+            kwargs = {"serialization_alias": name}
+            rename: str = name.removeprefix("_")
+        else:
+            kwargs = {}
+            rename: str = name
+
         if param.default != inspect.Parameter.empty:
-            fields[name] = (type_hints[name], param.default)
+            fields[rename] = Annotated[
+                type_hints[name],
+                Field(default=param.default, **kwargs),
+            ]
         else:
-            fields[name] = (type_hints[name], ...)
+            fields[rename] = Annotated[
+                type_hints[name],
+                Field(..., **kwargs),
+            ]
 
     return create_model(
-        "".join(i.title() for i in func.__name__.split("_")), **fields
+        to_pascal(func.__name__),
+        __base__=BaseCallerArgs,
+        **fields,
     )

{ddeutil_workflow-0.0.63 → ddeutil_workflow-0.0.64}/src/ddeutil/workflow/stages.py

@@ -55,7 +55,7 @@ from textwrap import dedent
 from threading import Event
 from typing import Annotated, Any, Optional, TypeVar, Union, get_type_hints
 
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, ValidationError
 from pydantic.functional_validators import model_validator
 from typing_extensions import Self
 
@@ -63,7 +63,13 @@ from .__types import DictData, DictStr, StrOrInt, StrOrNone, TupleStr
 from .conf import dynamic, pass_env
 from .exceptions import StageException, to_dict
 from .result import CANCEL, FAILED, SUCCESS, WAIT, Result, Status
-from .reusables import TagFunc, extract_call, not_in_template, param2template
+from .reusables import (
+    TagFunc,
+    create_model_from_caller,
+    extract_call,
+    not_in_template,
+    param2template,
+)
 from .utils import (
     delay,
     dump_all,
@@ -1088,8 +1094,7 @@ class CallStage(BaseAsyncStage):
         if "result" not in sig.parameters and not has_keyword:
             args.pop("result")
 
-        args = self.parse_model_args(call_func, args, result)
-
+        args = self.validate_model_args(call_func, args, result)
         if inspect.iscoroutinefunction(call_func):
             loop = asyncio.get_event_loop()
             rs: DictData = loop.run_until_complete(
@@ -1177,7 +1182,7 @@ class CallStage(BaseAsyncStage):
         if "result" not in sig.parameters and not has_keyword:
             args.pop("result")
 
-        args = self.parse_model_args(call_func, args, result)
+        args = self.validate_model_args(call_func, args, result)
         if inspect.iscoroutinefunction(call_func):
             rs: DictOrModel = await call_func(
                 **param2template(args, params, extras=self.extras)
@@ -1200,13 +1205,12 @@ class CallStage(BaseAsyncStage):
         return result.catch(status=SUCCESS, context=dump_all(rs, by_alias=True))
 
     @staticmethod
-    def parse_model_args(
+    def validate_model_args(
         func: TagFunc,
         args: DictData,
         result: Result,
     ) -> DictData:
-        """Parse Pydantic model from any dict data before parsing to target
-        caller function.
+        """Validate an input arguments before passing to the caller function.
 
         :param func: A tag function that want to get typing.
         :param args: An arguments before passing to this tag function.
@@ -1216,7 +1220,25 @@ class CallStage(BaseAsyncStage):
         :rtype: DictData
         """
         try:
+            model_instance = create_model_from_caller(func).model_validate(args)
+            override = dict(model_instance)
+            args.update(override)
+
             type_hints: dict[str, Any] = get_type_hints(func)
+
+            for arg in type_hints:
+
+                if arg == "return":
+                    continue
+
+                if arg.removeprefix("_") in args:
+                    args[arg] = args.pop(arg.removeprefix("_"))
+
+            return args
+        except ValidationError as e:
+            raise StageException(
+                "Validate argument from the caller function raise invalid type."
+            ) from e
         except TypeError as e:
             result.trace.warning(
                 f"[STAGE]: Get type hint raise TypeError: {e}, so, it skip "
@@ -1224,26 +1246,6 @@ class CallStage(BaseAsyncStage):
             )
             return args
 
-        for arg in type_hints:
-
-            if arg == "return":
-                continue
-
-            if arg.removeprefix("_") in args:
-                args[arg] = args.pop(arg.removeprefix("_"))
-
-            t: Any = type_hints[arg]
-
-            # NOTE: Check Result argument was passed to this caller function.
-            #
-            # if is_dataclass(t) and t.__name__ == "Result" and arg not in args:
-            #     args[arg] = result
-
-            if issubclass(t, BaseModel) and arg in args:
-                args[arg] = t.model_validate(obj=args[arg])
-
-        return args
-
 
 class TriggerStage(BaseStage):
     """Trigger workflow executor stage that run an input trigger Workflow

{ddeutil_workflow-0.0.63 → ddeutil_workflow-0.0.64}/src/ddeutil/workflow/utils.py

@@ -293,11 +293,13 @@ def cut_id(run_id: str, *, num: int = 6) -> str:
 
 
 @overload
-def dump_all(value: BaseModel, by_alias: bool = False) -> DictData: ...
+def dump_all(
+    value: BaseModel, by_alias: bool = False
+) -> DictData: ...  # pragma: no cov
 
 
 @overload
-def dump_all(value: T, by_alias: bool = False) -> T: ...
+def dump_all(value: T, by_alias: bool = False) -> T: ...  # pragma: no cov
 
 
 def dump_all(