ddeutil-workflow 0.0.62__py3-none-any.whl → 0.0.64__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1 +1 @@
-__version__: str = "0.0.62"
+__version__: str = "0.0.64"

ddeutil/workflow/__init__.py

@@ -5,12 +5,7 @@
 # ------------------------------------------------------------------------------
 from .__cron import CronJob, CronRunner
 from .__types import DictData, DictStr, Matrix, Re, TupleStr
-from .conf import (
-    Config,
-    FileLoad,
-    config,
-    env,
-)
+from .conf import *
 from .event import *
 from .exceptions import *
 from .job import *
@@ -37,31 +32,7 @@ from .result import (
     Result,
     Status,
 )
-from .reusables import (
-    FILTERS,
-    FilterFunc,
-    FilterRegistry,
-    ReturnTagFunc,
-    TagFunc,
-    custom_filter,
-    extract_call,
-    get_args_const,
-    has_template,
-    make_filter_registry,
-    make_registry,
-    map_post_filter,
-    not_in_template,
-    param2template,
-    str2template,
-    tag,
-)
-from .scheduler import (
-    Schedule,
-    ScheduleWorkflow,
-    schedule_control,
-    schedule_runner,
-    schedule_task,
-)
+from .reusables import *
 from .stages import *
 from .utils import *
 from .workflow import *

ddeutil/workflow/api/__init__.py

@@ -7,8 +7,6 @@ from __future__ import annotations
 
 import contextlib
 from collections.abc import AsyncIterator
-from datetime import datetime, timedelta
-from typing import TypedDict
 
 from dotenv import load_dotenv
 from fastapi import FastAPI, Request
@@ -20,49 +18,18 @@ from fastapi.middleware.gzip import GZipMiddleware
 from fastapi.responses import UJSONResponse
 
 from ..__about__ import __version__
-from ..conf import api_config, config
+from ..conf import api_config
 from ..logs import get_logger
-from ..scheduler import ReleaseThread, ReleaseThreads
-from ..workflow import ReleaseQueue, WorkflowTask
-from .routes import job, log
-from .utils import repeat_at
+from .routes import job, log, workflow
 
 load_dotenv()
 logger = get_logger("uvicorn.error")
 
 
-class State(TypedDict):
-    """TypeDict for State of FastAPI application."""
-
-    scheduler: list[str]
-    workflow_threads: ReleaseThreads
-    workflow_tasks: list[WorkflowTask]
-    workflow_queue: dict[str, ReleaseQueue]
-
-
 @contextlib.asynccontextmanager
-async def lifespan(a: FastAPI) -> AsyncIterator[State]:
+async def lifespan(_: FastAPI) -> AsyncIterator[dict[str, list]]:
     """Lifespan function for the FastAPI application."""
-    a.state.scheduler = []
-    a.state.workflow_threads = {}
-    a.state.workflow_tasks = []
-    a.state.workflow_queue = {}
-
-    yield {
-        # NOTE: Scheduler value should be contained a key of workflow and
-        #   list of datetime of queue and running.
-        #
-        #   ... {
-        #   ...     '<workflow-name>': (
-        #   ...         [<running-datetime>, ...], [<queue-datetime>, ...]
-        #   ...     )
-        #   ... }
-        #
-        "scheduler": a.state.scheduler,
-        "workflow_queue": a.state.workflow_queue,
-        "workflow_threads": a.state.workflow_threads,
-        "workflow_tasks": a.state.workflow_tasks,
-    }
+    yield {}
 
 
 app = FastAPI(
@@ -99,53 +66,7 @@ async def health():
 # NOTE Add the jobs and logs routes by default.
 app.include_router(job, prefix=api_config.prefix_path)
 app.include_router(log, prefix=api_config.prefix_path)
-
-
-# NOTE: Enable the workflows route.
-if api_config.enable_route_workflow:
-    from .routes import workflow
-
-    app.include_router(workflow, prefix=api_config.prefix_path)
-
-
-# NOTE: Enable the schedules route.
-if api_config.enable_route_schedule:
-    from ..logs import get_audit
-    from ..scheduler import schedule_task
-    from .routes import schedule
-
-    app.include_router(schedule, prefix=api_config.prefix_path)
-
-    @schedule.on_event("startup")
-    @repeat_at(cron="* * * * *", delay=2)
-    def scheduler_listener():
-        """Schedule broker every minute at 02 second."""
-        logger.debug(
-            f"[SCHEDULER]: Start listening schedule from queue "
-            f"{app.state.scheduler}"
-        )
-        if app.state.workflow_tasks:
-            schedule_task(
-                app.state.workflow_tasks,
-                stop=datetime.now(config.tz) + timedelta(minutes=1),
-                queue=app.state.workflow_queue,
-                threads=app.state.workflow_threads,
-                audit=get_audit(),
-            )
-
-    @schedule.on_event("startup")
-    @repeat_at(cron="*/5 * * * *", delay=10)
-    def monitoring():
-        """Monitoring workflow thread that running in the background."""
-        logger.debug("[MONITOR]: Start monitoring threading.")
-        snapshot_threads: list[str] = list(app.state.workflow_threads.keys())
-        for t_name in snapshot_threads:
-
-            thread_release: ReleaseThread = app.state.workflow_threads[t_name]
-
-            # NOTE: remove the thread that running success.
-            if not thread_release["thread"].is_alive():
-                app.state.workflow_threads.pop(t_name)
+app.include_router(workflow, prefix=api_config.prefix_path)
 
 
 @app.exception_handler(RequestValidationError)

ddeutil/workflow/api/routes/__init__.py

@@ -5,5 +5,4 @@
 # ------------------------------------------------------------------------------
 from .job import job_route as job
 from .logs import log_route as log
-from .schedules import schedule_route as schedule
 from .workflows import workflow_route as workflow

ddeutil/workflow/api/routes/job.py

@@ -69,7 +69,6 @@ async def job_execute(
             by_alias=True,
             exclude_none=False,
             exclude_unset=True,
-            exclude_defaults=True,
         ),
         "params": params,
         "context": context,

ddeutil/workflow/api/routes/logs.py

@@ -44,7 +44,6 @@ async def get_traces(
                 by_alias=True,
                 exclude_none=True,
                 exclude_unset=True,
-                exclude_defaults=True,
             )
             for trace in result.trace.find_traces()
         ],
@@ -73,7 +72,6 @@ async def get_trace_with_id(run_id: str):
                 by_alias=True,
                 exclude_none=True,
                 exclude_unset=True,
-                exclude_defaults=True,
             )
         ),
     }

ddeutil/workflow/api/routes/workflows.py

@@ -56,7 +56,6 @@ async def get_workflow_by_name(name: str) -> DictData:
         by_alias=True,
         exclude_none=False,
         exclude_unset=True,
-        exclude_defaults=True,
     )
 
 
@@ -99,7 +98,6 @@ async def get_workflow_audits(name: str):
                     by_alias=True,
                     exclude_none=False,
                     exclude_unset=True,
-                    exclude_defaults=True,
                 )
                 for audit in get_audit().find_audits(name=name)
             ],
@@ -133,6 +131,5 @@ async def get_workflow_release_audit(name: str, release: str):
             by_alias=True,
             exclude_none=False,
             exclude_unset=True,
-            exclude_defaults=True,
         ),
     }

ddeutil/workflow/conf.py

@@ -6,11 +6,9 @@
 from __future__ import annotations
 
 import copy
-import json
 import os
 from abc import ABC, abstractmethod
 from collections.abc import Iterator
-from datetime import timedelta
 from functools import cached_property
 from inspect import isclass
 from pathlib import Path
@@ -18,8 +16,9 @@ from typing import Final, Optional, Protocol, TypeVar, Union
 from zoneinfo import ZoneInfo
 
 from ddeutil.core import str2bool
-from ddeutil.io import YamlFlResolve
+from ddeutil.io import YamlFlResolve, search_env_replace
 from ddeutil.io.paths import glob_files, is_ignored, read_ignore
+from pydantic import SecretStr, TypeAdapter
 
 from .__types import DictData
 
@@ -162,28 +161,6 @@ class Config:  # pragma: no cov
     def max_queue_complete_hist(self) -> int:
         return int(env("CORE_MAX_QUEUE_COMPLETE_HIST", "16"))
 
-    # NOTE: App
-    @property
-    def max_schedule_process(self) -> int:
-        return int(env("APP_MAX_PROCESS", "2"))
-
-    @property
-    def max_schedule_per_process(self) -> int:
-        return int(env("APP_MAX_SCHEDULE_PER_PROCESS", "100"))
-
-    @property
-    def stop_boundary_delta(self) -> timedelta:
-        stop_boundary_delta_str: str = env(
-            "APP_STOP_BOUNDARY_DELTA", '{"minutes": 5, "seconds": 20}'
-        )
-        try:
-            return timedelta(**json.loads(stop_boundary_delta_str))
-        except Exception as err:
-            raise ValueError(
-                "Config `WORKFLOW_APP_STOP_BOUNDARY_DELTA` can not parsing to"
-                f"timedelta with {stop_boundary_delta_str}."
-            ) from err
-
 
 class APIConfig:
     """API Config object."""
@@ -192,14 +169,6 @@ class APIConfig:
     def prefix_path(self) -> str:
         return env("API_PREFIX_PATH", "/api/v1")
 
-    @property
-    def enable_route_workflow(self) -> bool:
-        return str2bool(env("API_ENABLE_ROUTE_WORKFLOW", "true"))
-
-    @property
-    def enable_route_schedule(self) -> bool:
-        return str2bool(env("API_ENABLE_ROUTE_SCHEDULE", "true"))
-
 
 class BaseLoad(ABC):  # pragma: no cov
     """Base Load object is the abstraction object for any Load object that
@@ -321,15 +290,16 @@ class FileLoad(BaseLoad):
         *,
         path: Optional[Path] = None,
         paths: Optional[list[Path]] = None,
-        excluded: list[str] | None = None,
+        excluded: Optional[list[str]] = None,
         extras: Optional[DictData] = None,
     ) -> Iterator[tuple[str, DictData]]:
         """Find all data that match with object type in config path. This class
         method can use include and exclude list of identity name for filter and
         adds-on.
 
-        :param obj: An object that want to validate matching before return.
-        :param path: A config path object.
+        :param obj: (object) An object that want to validate matching before
+            return.
+        :param path: (Path) A config path object.
         :param paths: (list[Path]) A list of config path object.
         :param excluded: An included list of data key that want to filter from
             data.
@@ -474,3 +444,37 @@ class Loader(Protocol):  # pragma: no cov
     def finds(
         cls, obj: object, *args, **kwargs
     ) -> Iterator[tuple[str, DictData]]: ...
+
+
+def pass_env(value: T) -> T:  # pragma: no cov
+    """Passing environment variable to an input value.
+
+    :param value: (Any) A value that want to pass env var searching.
+
+    :rtype: Any
+    """
+    if isinstance(value, dict):
+        return {k: pass_env(value[k]) for k in value}
+    elif isinstance(value, (list, tuple, set)):
+        return type(value)([pass_env(i) for i in value])
+    if not isinstance(value, str):
+        return value
+
+    rs: str = search_env_replace(value)
+    return None if rs == "null" else rs
+
+
+class CallerSecret(SecretStr):  # pragma: no cov
+    """Workflow Secret String model."""
+
+    def get_secret_value(self) -> str:
+        """Override get_secret_value by adding pass_env before return the
+        real-value.
+
+        :rtype: str
+        """
+        return pass_env(super().get_secret_value())
+
+
+# NOTE: Define the caller secret type for use it directly in the caller func.
+CallerSecretType = TypeAdapter(CallerSecret)

ddeutil/workflow/event.py

@@ -17,6 +17,7 @@ from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
 from pydantic import BaseModel, ConfigDict, Field, ValidationInfo
 from pydantic.functional_serializers import field_serializer
 from pydantic.functional_validators import field_validator, model_validator
+from pydantic_extra_types.timezone_name import TimeZoneName
 from typing_extensions import Self
 
 from .__cron import WEEKDAYS, CronJob, CronJobYear, CronRunner, Options
@@ -92,7 +93,7 @@ class Crontab(BaseModel):
         ),
     ]
     tz: Annotated[
-        str,
+        TimeZoneName,
         Field(
             description="A timezone string value",
             alias="timezone",

ddeutil/workflow/exceptions.py

@@ -83,6 +83,3 @@ class WorkflowException(BaseWorkflowException): ...
 
 
 class ParamValueException(WorkflowException): ...
-
-
-class ScheduleException(BaseWorkflowException): ...

ddeutil/workflow/reusables.py

@@ -14,7 +14,17 @@ from ast import Call, Constant, Expr, Module, Name, parse
 from datetime import datetime
 from functools import wraps
 from importlib import import_module
-from typing import Any, Callable, Literal, Optional, Protocol, TypeVar, Union
+from typing import (
+    Annotated,
+    Any,
+    Callable,
+    Literal,
+    Optional,
+    Protocol,
+    TypeVar,
+    Union,
+    get_type_hints,
+)
 
 try:
     from typing import ParamSpec
@@ -23,6 +33,8 @@ except ImportError:
 
 from ddeutil.core import getdot, import_string, lazy
 from ddeutil.io import search_env_replace
+from pydantic import BaseModel, ConfigDict, Field, create_model
+from pydantic.alias_generators import to_pascal
 from pydantic.dataclasses import dataclass
 
 from .__types import DictData, Re
@@ -111,7 +123,7 @@ def make_filter_registry(
             if not (
                 hasattr(func, "filter")
                 and str(getattr(func, "mark", "NOT SET")) == "filter"
-            ):
+            ):  # pragma: no cov
                 continue
 
             func: FilterFunc
@@ -231,6 +243,7 @@ def not_in_template(value: Any, *, not_in: str = "matrix.") -> bool:
 
     :param value: A value that want to find parameter template prefix.
     :param not_in: The not-in string that use in the `.startswith` function.
+        (Default is `matrix.`)
 
     :rtype: bool
     """
@@ -279,7 +292,7 @@ def str2template(
     :param value: (str) A string value that want to map with params.
     :param params: (DictData) A parameter value that getting with matched
         regular expression.
-    :param filters: A mapping of filter registry.
+    :param filters: (dict[str, FilterRegistry]) A mapping of filter registry.
     :param registers: (Optional[list[str]]) Override list of register.
 
     :rtype: str
@@ -304,12 +317,14 @@ def str2template(
         ]
 
         # NOTE: from validate step, it guarantees that caller exists in params.
+        #   I recommend to avoid logging params context on this case because it
+        #   can include secret value.
         try:
             getter: Any = getdot(caller, params)
-        except ValueError as err:
+        except ValueError:
             raise UtilException(
                 f"Parameters does not get dot with caller: {caller!r}."
-            ) from err
+            ) from None
 
         # NOTE:
         #   If type of getter caller is not string type, and it does not use to
@@ -341,10 +356,11 @@ def param2template(
     """Pass param to template string that can search by ``RE_CALLER`` regular
     expression.
 
-    :param value: A value that want to map with params
-    :param params: A parameter value that getting with matched regular
-        expression.
-    :param filters: A filter mapping for mapping with `map_post_filter` func.
+    :param value: (Any) A value that want to map with params.
+    :param params: (DictData) A parameter value that getting with matched
+        regular expression.
+    :param filters: (dict[str, FilterRegistry]) A filter mapping for mapping
+        with `map_post_filter` func.
     :param extras: (Optional[list[str]]) An Override extras.
 
     :rtype: Any
@@ -376,8 +392,8 @@ def datetime_format(value: datetime, fmt: str = "%Y-%m-%d %H:%M:%S") -> str:
 
     Examples:
 
-        > ${{ start-date | fmt('%Y%m%d') }}
-        > ${{ start-date | fmt }}
+        >>> "${{ start-date | fmt('%Y%m%d') }}"
+        >>> "${{ start-date | fmt }}"
 
     :param value: (datetime) A datetime value that want to format to string
         value.
@@ -399,7 +415,7 @@ def coalesce(value: Optional[T], default: Any) -> T:
 
     Examples:
 
-        > ${{ value | coalesce("foo") }}
+        >>> "${{ value | coalesce('foo') }}"
 
     :param value: A value that want to check nullable.
     :param default: A default value that use to returned value if an input
@@ -412,7 +428,14 @@ def coalesce(value: Optional[T], default: Any) -> T:
 def get_item(
     value: DictData, key: Union[str, int], default: Optional[Any] = None
 ) -> Any:
-    """Get a value with an input specific key."""
+    """Get a value with an input specific key.
+
+    Examples:
+
+        >>> "${{ value | getitem('key') }}"
+        >>> "${{ value | getitem('key', 'default') }}"
+
+    """
     if not isinstance(value, dict):
         raise UtilException(
             f"The value that pass to `getitem` filter should be `dict` not "
@@ -422,7 +445,14 @@ def get_item(
 
 
 @custom_filter("getindex")  # pragma: no cov
-def get_index(value: list[Any], index: int):
+def get_index(value: list[Any], index: int) -> Any:
+    """Get a value with an input specific index.
+
+    Examples:
+
+        >>> "${{ value | getindex(1) }}"
+
+    """
     if not isinstance(value, list):
         raise UtilException(
             f"The value that pass to `getindex` filter should be `list` not "
@@ -605,3 +635,63 @@ def extract_call(
             f"`REGISTER.{call.path}.registries.{call.func}`"
         )
     return rgt[call.func][call.tag]
+
+
+class BaseCallerArgs(BaseModel):  # pragma: no cov
+    """Base Caller Args model."""
+
+    model_config = ConfigDict(
+        arbitrary_types_allowed=True,
+        use_enum_values=True,
+    )
+
+
+def create_model_from_caller(func: Callable) -> BaseModel:  # pragma: no cov
+    """Create model from the caller function. This function will use for
+    validate the caller function argument typed-hint that valid with the args
+    field.
+
+    Reference:
+        - https://github.com/lmmx/pydantic-function-models
+        - https://docs.pydantic.dev/1.10/usage/models/#dynamic-model-creation
+
+    :param func: (Callable) A caller function.
+
+    :rtype: BaseModel
+    """
+    sig: inspect.Signature = inspect.signature(func)
+    type_hints: dict[str, Any] = get_type_hints(func)
+    fields: dict[str, Any] = {}
+    for name in sig.parameters:
+        param: inspect.Parameter = sig.parameters[name]
+
+        # NOTE: Skip all `*args` and `**kwargs` parameters.
+        if param.kind in (
+            inspect.Parameter.VAR_KEYWORD,
+            inspect.Parameter.VAR_POSITIONAL,
+        ):
+            continue
+
+        if name.startswith("_"):
+            kwargs = {"serialization_alias": name}
+            rename: str = name.removeprefix("_")
+        else:
+            kwargs = {}
+            rename: str = name
+
+        if param.default != inspect.Parameter.empty:
+            fields[rename] = Annotated[
+                type_hints[name],
+                Field(default=param.default, **kwargs),
+            ]
+        else:
+            fields[rename] = Annotated[
+                type_hints[name],
+                Field(..., **kwargs),
+            ]
+
+    return create_model(
+        to_pascal(func.__name__),
+        __base__=BaseCallerArgs,
+        **fields,
+    )