ddeutil-workflow 0.0.62__tar.gz → 0.0.64__tar.gz

{ddeutil_workflow-0.0.62 → ddeutil_workflow-0.0.64}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddeutil-workflow
-Version: 0.0.62
+Version: 0.0.64
 Summary: Lightweight workflow orchestration
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT
@@ -25,6 +25,7 @@ License-File: LICENSE
 Requires-Dist: ddeutil[checksum]>=0.4.8
 Requires-Dist: ddeutil-io[toml,yaml]>=0.2.13
 Requires-Dist: pydantic==2.11.4
+Requires-Dist: pydantic-extra-types==2.10.4
 Requires-Dist: python-dotenv==1.1.0
 Requires-Dist: schedule<2.0.0,==1.2.2
 Provides-Extra: all
@@ -215,19 +216,23 @@ registry-caller/
 This function will store as module that will import from `WORKFLOW_CORE_REGISTRY_CALLER`
 value (This config can override by extra parameters with `registry_caller` key).
 
+> [!NOTE]
+> You can use Pydantic Model as argument of your caller function. The core workflow
+> engine will auto use the `model_validate` method before run your caller function.
+
 ```python
-from ddeutil.workflow import Result, tag
+from ddeutil.workflow import Result, CallerSecret, tag
 from ddeutil.workflow.exceptions import StageException
-from pydantic import BaseModel, SecretStr
+from pydantic import BaseModel
 
 class AwsCredential(BaseModel):
     path: str
     access_client_id: str
-    access_client_secret: SecretStr
+    access_client_secret: CallerSecret
 
 class RestAuth(BaseModel):
     type: str
-    keys: SecretStr
+    keys: CallerSecret
 
 @tag("requests", alias="get-api-with-oauth-to-s3")
 def get_api_with_oauth_to_s3(
@@ -243,6 +248,7 @@ def get_api_with_oauth_to_s3(
     result.trace.info(f"... {method}: {url}")
     if method != "post":
        raise StageException(f"RestAPI does not support for {method} action.")
+    # NOTE: If you want to use secret, you can use `auth.keys.get_secret_value()`.
     return {"records": 1000}
 ```
 
@@ -259,45 +265,6 @@ result: Result = workflow.execute(
 )
 ```
 
-> [!NOTE]
-> So, this package provide the `Schedule` template for this action, and you can
-> pass the parameters dynamically for changing align with that running time by
-> the `release` prefix.
->
-> ```yaml
-> schedule-run-local-wf:
->
->    # Validate model that use to parsing exists for template file
->    type: Schedule
->    workflows:
->
->       # Map existing workflow that want to deploy with scheduler application.
->       # It allows you to pass release parameter that dynamic change depend on the
->       # current context of this scheduler application releasing that time.
->       - name: run-py-local
->         params:
->           source-extract: "USD-THB"
->           run-date: "${{ release.logical_date }}"
-> ```
->
-> The main method of the `Schedule` model that use to running is `pending`. If you
-> do not pass the `stop` date on this method, it will use config with
-> `WORKFLOW_APP_STOP_BOUNDARY_DELTA` key for generate this stop date.
->
-> ```python
-> from ddeutil.workflow import Schedule
->
-> (
->    Schedule
->    .from_conf("schedule-run-local-wf")
->    .pending(stop=None)
-> )
-> ```
-
-> [!WARNING]
-> The scheduler feature is the expensive feature of this project. You should
-> avoid to use it and find a scheduler tool instead.
-
 ## :cookie: Configuration
 
 The main configuration that use to dynamic changing this workflow engine for your
@@ -327,19 +294,6 @@ it will use default value and do not raise any error to you.
 | **TRACE_ENABLE_WRITE**       |    Log    | `false`                                                                                                                         |                                                                                                                    |
 | **AUDIT_PATH**               |    Log    | `./audits`                                                                                                                      |                                                                                                                    |
 | **AUDIT_ENABLE_WRITE**       |    Log    | `true`                                                                                                                          | A flag that enable logging object saving log to its destination.                                                   |
-| **MAX_PROCESS**              |    App    | `2`                                                                                                                             | The maximum process worker number that run in scheduler app module.                                                |
-| **MAX_SCHEDULE_PER_PROCESS** |    App    | `100`                                                                                                                           | A schedule per process that run parallel.                                                                          |
-| **STOP_BOUNDARY_DELTA**      |    App    | `'{"minutes": 5, "seconds": 20}'`                                                                                               | A time delta value that use to stop scheduler app in json string format.                                           |
-
-**API Application**:
-
-This config part use for the workflow application that build from the FastAPI
-only.
-
-| Environment                |  Component  | Default | Description                                                                        |
-|:---------------------------|:-----------:|---------|------------------------------------------------------------------------------------|
-| **ENABLE_ROUTE_WORKFLOW**  |     API     | `true`  | A flag that enable workflow route to manage execute manually and workflow logging. |
-| **ENABLE_ROUTE_SCHEDULE**  |     API     | `true`  | A flag that enable run scheduler.                                                  |
 
 ## :rocket: Deployment
 

{ddeutil_workflow-0.0.62 → ddeutil_workflow-0.0.64}/README.md

@@ -165,19 +165,23 @@ registry-caller/
 This function will store as module that will import from `WORKFLOW_CORE_REGISTRY_CALLER`
 value (This config can override by extra parameters with `registry_caller` key).
 
+> [!NOTE]
+> You can use Pydantic Model as argument of your caller function. The core workflow
+> engine will auto use the `model_validate` method before run your caller function.
+
 ```python
-from ddeutil.workflow import Result, tag
+from ddeutil.workflow import Result, CallerSecret, tag
 from ddeutil.workflow.exceptions import StageException
-from pydantic import BaseModel, SecretStr
+from pydantic import BaseModel
 
 class AwsCredential(BaseModel):
     path: str
     access_client_id: str
-    access_client_secret: SecretStr
+    access_client_secret: CallerSecret
 
 class RestAuth(BaseModel):
     type: str
-    keys: SecretStr
+    keys: CallerSecret
 
 @tag("requests", alias="get-api-with-oauth-to-s3")
 def get_api_with_oauth_to_s3(
@@ -193,6 +197,7 @@ def get_api_with_oauth_to_s3(
     result.trace.info(f"... {method}: {url}")
     if method != "post":
        raise StageException(f"RestAPI does not support for {method} action.")
+    # NOTE: If you want to use secret, you can use `auth.keys.get_secret_value()`.
     return {"records": 1000}
 ```
 
@@ -209,45 +214,6 @@ result: Result = workflow.execute(
 )
 ```
 
-> [!NOTE]
-> So, this package provide the `Schedule` template for this action, and you can
-> pass the parameters dynamically for changing align with that running time by
-> the `release` prefix.
->
-> ```yaml
-> schedule-run-local-wf:
->
->    # Validate model that use to parsing exists for template file
->    type: Schedule
->    workflows:
->
->       # Map existing workflow that want to deploy with scheduler application.
->       # It allows you to pass release parameter that dynamic change depend on the
->       # current context of this scheduler application releasing that time.
->       - name: run-py-local
->         params:
->           source-extract: "USD-THB"
->           run-date: "${{ release.logical_date }}"
-> ```
->
-> The main method of the `Schedule` model that use to running is `pending`. If you
-> do not pass the `stop` date on this method, it will use config with
-> `WORKFLOW_APP_STOP_BOUNDARY_DELTA` key for generate this stop date.
->
-> ```python
-> from ddeutil.workflow import Schedule
->
-> (
->    Schedule
->    .from_conf("schedule-run-local-wf")
->    .pending(stop=None)
-> )
-> ```
-
-> [!WARNING]
-> The scheduler feature is the expensive feature of this project. You should
-> avoid to use it and find a scheduler tool instead.
-
 ## :cookie: Configuration
 
 The main configuration that use to dynamic changing this workflow engine for your
@@ -277,19 +243,6 @@ it will use default value and do not raise any error to you.
 | **TRACE_ENABLE_WRITE**       |    Log    | `false`                                                                                                                         |                                                                                                                    |
 | **AUDIT_PATH**               |    Log    | `./audits`                                                                                                                      |                                                                                                                    |
 | **AUDIT_ENABLE_WRITE**       |    Log    | `true`                                                                                                                          | A flag that enable logging object saving log to its destination.                                                   |
-| **MAX_PROCESS**              |    App    | `2`                                                                                                                             | The maximum process worker number that run in scheduler app module.                                                |
-| **MAX_SCHEDULE_PER_PROCESS** |    App    | `100`                                                                                                                           | A schedule per process that run parallel.                                                                          |
-| **STOP_BOUNDARY_DELTA**      |    App    | `'{"minutes": 5, "seconds": 20}'`                                                                                               | A time delta value that use to stop scheduler app in json string format.                                           |
-
-**API Application**:
-
-This config part use for the workflow application that build from the FastAPI
-only.
-
-| Environment                |  Component  | Default | Description                                                                        |
-|:---------------------------|:-----------:|---------|------------------------------------------------------------------------------------|
-| **ENABLE_ROUTE_WORKFLOW**  |     API     | `true`  | A flag that enable workflow route to manage execute manually and workflow logging. |
-| **ENABLE_ROUTE_SCHEDULE**  |     API     | `true`  | A flag that enable run scheduler.                                                  |
 
 ## :rocket: Deployment
 

{ddeutil_workflow-0.0.62 → ddeutil_workflow-0.0.64}/pyproject.toml

@@ -28,6 +28,7 @@ dependencies = [
     "ddeutil[checksum]>=0.4.8",
     "ddeutil-io[yaml,toml]>=0.2.13",
     "pydantic==2.11.4",
+    "pydantic-extra-types==2.10.4",
     "python-dotenv==1.1.0",
     "schedule==1.2.2,<2.0.0",
 ]

ddeutil_workflow-0.0.64/src/ddeutil/workflow/__about__.py

@@ -0,0 +1 @@
+__version__: str = "0.0.64"

{ddeutil_workflow-0.0.62 → ddeutil_workflow-0.0.64}/src/ddeutil/workflow/__init__.py

@@ -5,12 +5,7 @@
 # ------------------------------------------------------------------------------
 from .__cron import CronJob, CronRunner
 from .__types import DictData, DictStr, Matrix, Re, TupleStr
-from .conf import (
-    Config,
-    FileLoad,
-    config,
-    env,
-)
+from .conf import *
 from .event import *
 from .exceptions import *
 from .job import *
@@ -37,31 +32,7 @@ from .result import (
     Result,
     Status,
 )
-from .reusables import (
-    FILTERS,
-    FilterFunc,
-    FilterRegistry,
-    ReturnTagFunc,
-    TagFunc,
-    custom_filter,
-    extract_call,
-    get_args_const,
-    has_template,
-    make_filter_registry,
-    make_registry,
-    map_post_filter,
-    not_in_template,
-    param2template,
-    str2template,
-    tag,
-)
-from .scheduler import (
-    Schedule,
-    ScheduleWorkflow,
-    schedule_control,
-    schedule_runner,
-    schedule_task,
-)
+from .reusables import *
 from .stages import *
 from .utils import *
 from .workflow import *

ddeutil_workflow-0.0.64/src/ddeutil/workflow/api/__init__.py

@@ -0,0 +1,91 @@
+# ------------------------------------------------------------------------------
+# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
+# Licensed under the MIT License. See LICENSE in the project root for
+# license information.
+# ------------------------------------------------------------------------------
+from __future__ import annotations
+
+import contextlib
+from collections.abc import AsyncIterator
+
+from dotenv import load_dotenv
+from fastapi import FastAPI, Request
+from fastapi import status as st
+from fastapi.encoders import jsonable_encoder
+from fastapi.exceptions import RequestValidationError
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.middleware.gzip import GZipMiddleware
+from fastapi.responses import UJSONResponse
+
+from ..__about__ import __version__
+from ..conf import api_config
+from ..logs import get_logger
+from .routes import job, log, workflow
+
+load_dotenv()
+logger = get_logger("uvicorn.error")
+
+
+@contextlib.asynccontextmanager
+async def lifespan(_: FastAPI) -> AsyncIterator[dict[str, list]]:
+    """Lifespan function for the FastAPI application."""
+    yield {}
+
+
+app = FastAPI(
+    titile="Workflow",
+    description=(
+        "This is a workflow FastAPI application that use to manage manual "
+        "execute, logging, and schedule workflow via RestAPI."
+    ),
+    version=__version__,
+    lifespan=lifespan,
+    default_response_class=UJSONResponse,
+)
+app.add_middleware(GZipMiddleware, minimum_size=1000)
+origins: list[str] = [
+    "http://localhost",
+    "http://localhost:88",
+    "http://localhost:80",
+]
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=origins,
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+
+@app.get(path="/", response_class=UJSONResponse)
+async def health():
+    """Index view that not return any template without json status."""
+    return {"message": "Workflow already start up with healthy status."}
+
+
+# NOTE Add the jobs and logs routes by default.
+app.include_router(job, prefix=api_config.prefix_path)
+app.include_router(log, prefix=api_config.prefix_path)
+app.include_router(workflow, prefix=api_config.prefix_path)
+
+
+@app.exception_handler(RequestValidationError)
+async def validation_exception_handler(
+    request: Request, exc: RequestValidationError
+):
+    _ = request
+    return UJSONResponse(
+        status_code=st.HTTP_422_UNPROCESSABLE_ENTITY,
+        content=jsonable_encoder({"detail": exc.errors(), "body": exc.body}),
+    )
+
+
+if __name__ == "__main__":
+    import uvicorn
+
+    uvicorn.run(
+        app,
+        host="0.0.0.0",
+        port=80,
+        log_level="DEBUG",
+    )

{ddeutil_workflow-0.0.62 → ddeutil_workflow-0.0.64}/src/ddeutil/workflow/api/routes/__init__.py

@@ -5,5 +5,4 @@
 # ------------------------------------------------------------------------------
 from .job import job_route as job
 from .logs import log_route as log
-from .schedules import schedule_route as schedule
 from .workflows import workflow_route as workflow

{ddeutil_workflow-0.0.62 → ddeutil_workflow-0.0.64}/src/ddeutil/workflow/api/routes/job.py

@@ -69,7 +69,6 @@ async def job_execute(
             by_alias=True,
             exclude_none=False,
             exclude_unset=True,
-            exclude_defaults=True,
         ),
         "params": params,
         "context": context,

{ddeutil_workflow-0.0.62 → ddeutil_workflow-0.0.64}/src/ddeutil/workflow/api/routes/logs.py

@@ -44,7 +44,6 @@ async def get_traces(
                 by_alias=True,
                 exclude_none=True,
                 exclude_unset=True,
-                exclude_defaults=True,
             )
             for trace in result.trace.find_traces()
         ],
@@ -73,7 +72,6 @@ async def get_trace_with_id(run_id: str):
                 by_alias=True,
                 exclude_none=True,
                 exclude_unset=True,
-                exclude_defaults=True,
             )
         ),
     }

{ddeutil_workflow-0.0.62 → ddeutil_workflow-0.0.64}/src/ddeutil/workflow/api/routes/workflows.py

@@ -56,7 +56,6 @@ async def get_workflow_by_name(name: str) -> DictData:
         by_alias=True,
         exclude_none=False,
         exclude_unset=True,
-        exclude_defaults=True,
     )
 
 
@@ -99,7 +98,6 @@ async def get_workflow_audits(name: str):
                     by_alias=True,
                     exclude_none=False,
                     exclude_unset=True,
-                    exclude_defaults=True,
                 )
                 for audit in get_audit().find_audits(name=name)
             ],
@@ -133,6 +131,5 @@ async def get_workflow_release_audit(name: str, release: str):
             by_alias=True,
             exclude_none=False,
             exclude_unset=True,
-            exclude_defaults=True,
         ),
     }

{ddeutil_workflow-0.0.62 → ddeutil_workflow-0.0.64}/src/ddeutil/workflow/conf.py

@@ -6,11 +6,9 @@
 from __future__ import annotations
 
 import copy
-import json
 import os
 from abc import ABC, abstractmethod
 from collections.abc import Iterator
-from datetime import timedelta
 from functools import cached_property
 from inspect import isclass
 from pathlib import Path
@@ -18,8 +16,9 @@ from typing import Final, Optional, Protocol, TypeVar, Union
 from zoneinfo import ZoneInfo
 
 from ddeutil.core import str2bool
-from ddeutil.io import YamlFlResolve
+from ddeutil.io import YamlFlResolve, search_env_replace
 from ddeutil.io.paths import glob_files, is_ignored, read_ignore
+from pydantic import SecretStr, TypeAdapter
 
 from .__types import DictData
 
@@ -162,28 +161,6 @@ class Config:  # pragma: no cov
     def max_queue_complete_hist(self) -> int:
         return int(env("CORE_MAX_QUEUE_COMPLETE_HIST", "16"))
 
-    # NOTE: App
-    @property
-    def max_schedule_process(self) -> int:
-        return int(env("APP_MAX_PROCESS", "2"))
-
-    @property
-    def max_schedule_per_process(self) -> int:
-        return int(env("APP_MAX_SCHEDULE_PER_PROCESS", "100"))
-
-    @property
-    def stop_boundary_delta(self) -> timedelta:
-        stop_boundary_delta_str: str = env(
-            "APP_STOP_BOUNDARY_DELTA", '{"minutes": 5, "seconds": 20}'
-        )
-        try:
-            return timedelta(**json.loads(stop_boundary_delta_str))
-        except Exception as err:
-            raise ValueError(
-                "Config `WORKFLOW_APP_STOP_BOUNDARY_DELTA` can not parsing to"
-                f"timedelta with {stop_boundary_delta_str}."
-            ) from err
-
 
 class APIConfig:
     """API Config object."""
@@ -192,14 +169,6 @@ class APIConfig:
     def prefix_path(self) -> str:
         return env("API_PREFIX_PATH", "/api/v1")
 
-    @property
-    def enable_route_workflow(self) -> bool:
-        return str2bool(env("API_ENABLE_ROUTE_WORKFLOW", "true"))
-
-    @property
-    def enable_route_schedule(self) -> bool:
-        return str2bool(env("API_ENABLE_ROUTE_SCHEDULE", "true"))
-
 
 class BaseLoad(ABC):  # pragma: no cov
     """Base Load object is the abstraction object for any Load object that
@@ -321,15 +290,16 @@ class FileLoad(BaseLoad):
         *,
         path: Optional[Path] = None,
         paths: Optional[list[Path]] = None,
-        excluded: list[str] | None = None,
+        excluded: Optional[list[str]] = None,
         extras: Optional[DictData] = None,
     ) -> Iterator[tuple[str, DictData]]:
         """Find all data that match with object type in config path. This class
         method can use include and exclude list of identity name for filter and
         adds-on.
 
-        :param obj: An object that want to validate matching before return.
-        :param path: A config path object.
+        :param obj: (object) An object that want to validate matching before
+            return.
+        :param path: (Path) A config path object.
         :param paths: (list[Path]) A list of config path object.
         :param excluded: An included list of data key that want to filter from
             data.
@@ -474,3 +444,37 @@ class Loader(Protocol):  # pragma: no cov
     def finds(
         cls, obj: object, *args, **kwargs
     ) -> Iterator[tuple[str, DictData]]: ...
+
+
+def pass_env(value: T) -> T:  # pragma: no cov
+    """Passing environment variable to an input value.
+
+    :param value: (Any) A value that want to pass env var searching.
+
+    :rtype: Any
+    """
+    if isinstance(value, dict):
+        return {k: pass_env(value[k]) for k in value}
+    elif isinstance(value, (list, tuple, set)):
+        return type(value)([pass_env(i) for i in value])
+    if not isinstance(value, str):
+        return value
+
+    rs: str = search_env_replace(value)
+    return None if rs == "null" else rs
+
+
+class CallerSecret(SecretStr):  # pragma: no cov
+    """Workflow Secret String model."""
+
+    def get_secret_value(self) -> str:
+        """Override get_secret_value by adding pass_env before return the
+        real-value.
+
+        :rtype: str
+        """
+        return pass_env(super().get_secret_value())
+
+
+# NOTE: Define the caller secret type for use it directly in the caller func.
+CallerSecretType = TypeAdapter(CallerSecret)

{ddeutil_workflow-0.0.62 → ddeutil_workflow-0.0.64}/src/ddeutil/workflow/event.py

@@ -17,6 +17,7 @@ from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
 from pydantic import BaseModel, ConfigDict, Field, ValidationInfo
 from pydantic.functional_serializers import field_serializer
 from pydantic.functional_validators import field_validator, model_validator
+from pydantic_extra_types.timezone_name import TimeZoneName
 from typing_extensions import Self
 
 from .__cron import WEEKDAYS, CronJob, CronJobYear, CronRunner, Options
@@ -92,7 +93,7 @@ class Crontab(BaseModel):
         ),
     ]
     tz: Annotated[
-        str,
+        TimeZoneName,
         Field(
             description="A timezone string value",
             alias="timezone",

{ddeutil_workflow-0.0.62 → ddeutil_workflow-0.0.64}/src/ddeutil/workflow/exceptions.py

@@ -83,6 +83,3 @@ class WorkflowException(BaseWorkflowException): ...
 
 
 class ParamValueException(WorkflowException): ...
-
-
-class ScheduleException(BaseWorkflowException): ...