ddeutil-workflow 0.0.62__py3-none-any.whl → 0.0.64__py3-none-any.whl

ddeutil/workflow/stages.py

@@ -55,15 +55,21 @@ from textwrap import dedent
 from threading import Event
 from typing import Annotated, Any, Optional, TypeVar, Union, get_type_hints
 
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, ValidationError
 from pydantic.functional_validators import model_validator
 from typing_extensions import Self
 
 from .__types import DictData, DictStr, StrOrInt, StrOrNone, TupleStr
-from .conf import dynamic
+from .conf import dynamic, pass_env
 from .exceptions import StageException, to_dict
 from .result import CANCEL, FAILED, SUCCESS, WAIT, Result, Status
-from .reusables import TagFunc, extract_call, not_in_template, param2template
+from .reusables import (
+    TagFunc,
+    create_model_from_caller,
+    extract_call,
+    not_in_template,
+    param2template,
+)
 from .utils import (
     delay,
     dump_all,
@@ -626,10 +632,10 @@ class BashStage(BaseAsyncStage):
             await f.write(f"#!/bin/{f_shebang}\n\n")
 
             # NOTE: add setting environment variable before bash skip statement.
-            await f.writelines([f"{k}='{env[k]}';\n" for k in env])
+            await f.writelines(pass_env([f"{k}='{env[k]}';\n" for k in env]))
 
             # NOTE: make sure that shell script file does not have `\r` char.
-            await f.write("\n" + bash.replace("\r\n", "\n"))
+            await f.write("\n" + pass_env(bash.replace("\r\n", "\n")))
 
         # NOTE: Make this .sh file able to executable.
         make_exec(f"./{f_name}")
@@ -662,10 +668,10 @@ class BashStage(BaseAsyncStage):
             f.write(f"#!/bin/{f_shebang}\n\n")
 
             # NOTE: add setting environment variable before bash skip statement.
-            f.writelines([f"{k}='{env[k]}';\n" for k in env])
+            f.writelines(pass_env([f"{k}='{env[k]}';\n" for k in env]))
 
             # NOTE: make sure that shell script file does not have `\r` char.
-            f.write("\n" + bash.replace("\r\n", "\n"))
+            f.write("\n" + pass_env(bash.replace("\r\n", "\n")))
 
         # NOTE: Make this .sh file able to executable.
         make_exec(f"./{f_name}")
@@ -895,7 +901,9 @@ class PyStage(BaseAsyncStage):
         # WARNING: The exec build-in function is very dangerous. So, it
         #   should use the re module to validate exec-string before running.
         exec(
-            param2template(dedent(self.run), params, extras=self.extras),
+            pass_env(
+                param2template(dedent(self.run), params, extras=self.extras)
+            ),
             gb,
             lc,
         )
@@ -1060,12 +1068,12 @@ class CallStage(BaseAsyncStage):
         args: DictData = {"result": result} | param2template(
             self.args, params, extras=self.extras
         )
-        ips = inspect.signature(call_func)
+        sig = inspect.signature(call_func)
         necessary_params: list[str] = []
         has_keyword: bool = False
-        for k in ips.parameters:
+        for k in sig.parameters:
             if (
-                v := ips.parameters[k]
+                v := sig.parameters[k]
             ).default == Parameter.empty and v.kind not in (
                 Parameter.VAR_KEYWORD,
                 Parameter.VAR_POSITIONAL,
@@ -1083,11 +1091,10 @@ class CallStage(BaseAsyncStage):
                 f"does not set to args, {list(args.keys())}."
             )
 
-        if "result" not in ips.parameters and not has_keyword:
+        if "result" not in sig.parameters and not has_keyword:
             args.pop("result")
 
-        args = self.parse_model_args(call_func, args, result)
-
+        args = self.validate_model_args(call_func, args, result)
         if inspect.iscoroutinefunction(call_func):
             loop = asyncio.get_event_loop()
             rs: DictData = loop.run_until_complete(
@@ -1149,12 +1156,12 @@ class CallStage(BaseAsyncStage):
         args: DictData = {"result": result} | param2template(
             self.args, params, extras=self.extras
         )
-        ips = inspect.signature(call_func)
+        sig = inspect.signature(call_func)
         necessary_params: list[str] = []
         has_keyword: bool = False
-        for k in ips.parameters:
+        for k in sig.parameters:
             if (
-                v := ips.parameters[k]
+                v := sig.parameters[k]
             ).default == Parameter.empty and v.kind not in (
                 Parameter.VAR_KEYWORD,
                 Parameter.VAR_POSITIONAL,
@@ -1172,10 +1179,10 @@ class CallStage(BaseAsyncStage):
                 f"does not set to args, {list(args.keys())}."
             )
 
-        if "result" not in ips.parameters and not has_keyword:
+        if "result" not in sig.parameters and not has_keyword:
             args.pop("result")
 
-        args = self.parse_model_args(call_func, args, result)
+        args = self.validate_model_args(call_func, args, result)
         if inspect.iscoroutinefunction(call_func):
             rs: DictOrModel = await call_func(
                 **param2template(args, params, extras=self.extras)
@@ -1198,13 +1205,12 @@ class CallStage(BaseAsyncStage):
         return result.catch(status=SUCCESS, context=dump_all(rs, by_alias=True))
 
     @staticmethod
-    def parse_model_args(
+    def validate_model_args(
         func: TagFunc,
         args: DictData,
         result: Result,
     ) -> DictData:
-        """Parse Pydantic model from any dict data before parsing to target
-        caller function.
+        """Validate an input arguments before passing to the caller function.
 
         :param func: A tag function that want to get typing.
         :param args: An arguments before passing to this tag function.
@@ -1214,7 +1220,25 @@ class CallStage(BaseAsyncStage):
         :rtype: DictData
         """
         try:
+            model_instance = create_model_from_caller(func).model_validate(args)
+            override = dict(model_instance)
+            args.update(override)
+
             type_hints: dict[str, Any] = get_type_hints(func)
+
+            for arg in type_hints:
+
+                if arg == "return":
+                    continue
+
+                if arg.removeprefix("_") in args:
+                    args[arg] = args.pop(arg.removeprefix("_"))
+
+            return args
+        except ValidationError as e:
+            raise StageException(
+                "Validate argument from the caller function raise invalid type."
+            ) from e
         except TypeError as e:
             result.trace.warning(
                 f"[STAGE]: Get type hint raise TypeError: {e}, so, it skip "
@@ -1222,26 +1246,6 @@ class CallStage(BaseAsyncStage):
             )
             return args
 
-        for arg in type_hints:
-
-            if arg == "return":
-                continue
-
-            if arg.removeprefix("_") in args:
-                args[arg] = args.pop(arg.removeprefix("_"))
-
-            t: Any = type_hints[arg]
-
-            # NOTE: Check Result argument was passed to this caller function.
-            #
-            # if is_dataclass(t) and t.__name__ == "Result" and arg not in args:
-            #     args[arg] = result
-
-            if issubclass(t, BaseModel) and arg in args:
-                args[arg] = t.model_validate(obj=args[arg])
-
-        return args
-
 
 class TriggerStage(BaseStage):
     """Trigger workflow executor stage that run an input trigger Workflow
@@ -1295,7 +1299,7 @@ class TriggerStage(BaseStage):
         _trigger: str = param2template(self.trigger, params, extras=self.extras)
         result.trace.info(f"[STAGE]: Execute Trigger-Stage: {_trigger!r}")
         rs: Result = Workflow.from_conf(
-            name=_trigger,
+            name=pass_env(_trigger),
             extras=self.extras | {"stage_raise_error": True},
         ).execute(
             params=param2template(self.params, params, extras=self.extras),
@@ -2417,9 +2421,11 @@ class DockerStage(BaseStage):  # pragma: no cov
         )
 
         resp = client.api.pull(
-            repository=f"{self.image}",
-            tag=self.tag,
-            auth_config=param2template(self.auth, params, extras=self.extras),
+            repository=pass_env(self.image),
+            tag=pass_env(self.tag),
+            auth_config=pass_env(
+                param2template(self.auth, params, extras=self.extras)
+            ),
             stream=True,
             decode=True,
         )
@@ -2438,10 +2444,10 @@ class DockerStage(BaseStage):  # pragma: no cov
 
         unique_image_name: str = f"{self.image}_{datetime.now():%Y%m%d%H%M%S%f}"
         container = client.containers.run(
-            image=f"{self.image}:{self.tag}",
+            image=pass_env(f"{self.image}:{self.tag}"),
             name=unique_image_name,
-            environment=self.env,
-            volumes=(
+            environment=pass_env(self.env),
+            volumes=pass_env(
                 {
                     Path.cwd()
                     / f".docker.{result.run_id}.logs": {
@@ -2549,8 +2555,10 @@ class VirtualPyStage(PyStage):  # pragma: no cov
         f_name: str = f"{run_id}.py"
         with open(f"./{f_name}", mode="w", newline="\n") as f:
             # NOTE: Create variable mapping that write before running statement.
-            vars_str: str = "\n ".join(
-                f"{var} = {value!r}" for var, value in values.items()
+            vars_str: str = pass_env(
+                "\n ".join(
+                    f"{var} = {value!r}" for var, value in values.items()
+                )
             )
 
             # NOTE: `uv` supports PEP 723 — inline TOML metadata.
@@ -2568,7 +2576,7 @@ class VirtualPyStage(PyStage):  # pragma: no cov
             )
 
             # NOTE: make sure that py script file does not have `\r` char.
-            f.write("\n" + py.replace("\r\n", "\n"))
+            f.write("\n" + pass_env(py.replace("\r\n", "\n")))
 
         # NOTE: Make this .py file able to executable.
         make_exec(f"./{f_name}")

ddeutil/workflow/utils.py

@@ -293,11 +293,13 @@ def cut_id(run_id: str, *, num: int = 6) -> str:
 
 
 @overload
-def dump_all(value: BaseModel, by_alias: bool = False) -> DictData: ...
+def dump_all(
+    value: BaseModel, by_alias: bool = False
+) -> DictData: ...  # pragma: no cov
 
 
 @overload
-def dump_all(value: T, by_alias: bool = False) -> T: ...
+def dump_all(value: T, by_alias: bool = False) -> T: ...  # pragma: no cov
 
 
 def dump_all(

{ddeutil_workflow-0.0.62.dist-info → ddeutil_workflow-0.0.64.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddeutil-workflow
-Version: 0.0.62
+Version: 0.0.64
 Summary: Lightweight workflow orchestration
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT
@@ -25,6 +25,7 @@ License-File: LICENSE
 Requires-Dist: ddeutil[checksum]>=0.4.8
 Requires-Dist: ddeutil-io[toml,yaml]>=0.2.13
 Requires-Dist: pydantic==2.11.4
+Requires-Dist: pydantic-extra-types==2.10.4
 Requires-Dist: python-dotenv==1.1.0
 Requires-Dist: schedule<2.0.0,==1.2.2
 Provides-Extra: all
@@ -215,19 +216,23 @@ registry-caller/
 This function will store as module that will import from `WORKFLOW_CORE_REGISTRY_CALLER`
 value (This config can override by extra parameters with `registry_caller` key).
 
+> [!NOTE]
+> You can use Pydantic Model as argument of your caller function. The core workflow
+> engine will auto use the `model_validate` method before run your caller function.
+
 ```python
-from ddeutil.workflow import Result, tag
+from ddeutil.workflow import Result, CallerSecret, tag
 from ddeutil.workflow.exceptions import StageException
-from pydantic import BaseModel, SecretStr
+from pydantic import BaseModel
 
 class AwsCredential(BaseModel):
     path: str
     access_client_id: str
-    access_client_secret: SecretStr
+    access_client_secret: CallerSecret
 
 class RestAuth(BaseModel):
     type: str
-    keys: SecretStr
+    keys: CallerSecret
 
 @tag("requests", alias="get-api-with-oauth-to-s3")
 def get_api_with_oauth_to_s3(
@@ -243,6 +248,7 @@ def get_api_with_oauth_to_s3(
     result.trace.info(f"... {method}: {url}")
     if method != "post":
        raise StageException(f"RestAPI does not support for {method} action.")
+    # NOTE: If you want to use secret, you can use `auth.keys.get_secret_value()`.
     return {"records": 1000}
 ```
 
@@ -259,45 +265,6 @@ result: Result = workflow.execute(
 )
 ```
 
-> [!NOTE]
-> So, this package provide the `Schedule` template for this action, and you can
-> pass the parameters dynamically for changing align with that running time by
-> the `release` prefix.
->
-> ```yaml
-> schedule-run-local-wf:
->
->    # Validate model that use to parsing exists for template file
->    type: Schedule
->    workflows:
->
->       # Map existing workflow that want to deploy with scheduler application.
->       # It allows you to pass release parameter that dynamic change depend on the
->       # current context of this scheduler application releasing that time.
->       - name: run-py-local
->         params:
->           source-extract: "USD-THB"
->           run-date: "${{ release.logical_date }}"
-> ```
->
-> The main method of the `Schedule` model that use to running is `pending`. If you
-> do not pass the `stop` date on this method, it will use config with
-> `WORKFLOW_APP_STOP_BOUNDARY_DELTA` key for generate this stop date.
->
-> ```python
-> from ddeutil.workflow import Schedule
->
-> (
->    Schedule
->    .from_conf("schedule-run-local-wf")
->    .pending(stop=None)
-> )
-> ```
-
-> [!WARNING]
-> The scheduler feature is the expensive feature of this project. You should
-> avoid to use it and find a scheduler tool instead.
-
 ## :cookie: Configuration
 
 The main configuration that use to dynamic changing this workflow engine for your
@@ -327,19 +294,6 @@ it will use default value and do not raise any error to you.
 | **TRACE_ENABLE_WRITE**       |    Log    | `false`                                                                                                                         |                                                                                                                    |
 | **AUDIT_PATH**               |    Log    | `./audits`                                                                                                                      |                                                                                                                    |
 | **AUDIT_ENABLE_WRITE**       |    Log    | `true`                                                                                                                          | A flag that enable logging object saving log to its destination.                                                   |
-| **MAX_PROCESS**              |    App    | `2`                                                                                                                             | The maximum process worker number that run in scheduler app module.                                                |
-| **MAX_SCHEDULE_PER_PROCESS** |    App    | `100`                                                                                                                           | A schedule per process that run parallel.                                                                          |
-| **STOP_BOUNDARY_DELTA**      |    App    | `'{"minutes": 5, "seconds": 20}'`                                                                                               | A time delta value that use to stop scheduler app in json string format.                                           |
-
-**API Application**:
-
-This config part use for the workflow application that build from the FastAPI
-only.
-
-| Environment                |  Component  | Default | Description                                                                        |
-|:---------------------------|:-----------:|---------|------------------------------------------------------------------------------------|
-| **ENABLE_ROUTE_WORKFLOW**  |     API     | `true`  | A flag that enable workflow route to manage execute manually and workflow logging. |
-| **ENABLE_ROUTE_SCHEDULE**  |     API     | `true`  | A flag that enable run scheduler.                                                  |
 
 ## :rocket: Deployment
 

ddeutil_workflow-0.0.64.dist-info/RECORD

@@ -0,0 +1,28 @@
+ddeutil/workflow/__about__.py,sha256=u0hgLvitHp32P7aLr3vGCiFbiGUkcbr4qFyevZfSorQ,28
+ddeutil/workflow/__cron.py,sha256=BOKQcreiex0SAigrK1gnLxpvOeF3aca_rQwyz9Kfve4,28751
+ddeutil/workflow/__init__.py,sha256=g-NTBGwIJaXmvuWoIRuUE65nqsIuXEM2CGWlLSszVmM,905
+ddeutil/workflow/__main__.py,sha256=x-sYedl4T8p6054aySk-EQX6vhytvPR0HvaBNYxMzp0,364
+ddeutil/workflow/__types.py,sha256=uNfoRbVmNK5O37UUMVnqcmoghD9oMS1q9fXC0APnjSI,4584
+ddeutil/workflow/conf.py,sha256=trcsW2_jVeSFT_f58H_s99n2h8oRLIurRIF1ZrEjPYg,14927
+ddeutil/workflow/event.py,sha256=S2eJAZZx_V5TuQ0l417hFVCtjWXnfNPZBgSCICzxQ48,11041
+ddeutil/workflow/exceptions.py,sha256=_USvPsqPDkjmZycfjahpWHFhE-5fA6KRoc6ii0jRsVA,2401
+ddeutil/workflow/job.py,sha256=Php1b3n6c-jddel8PTSa61kAW22QBTetzoLVR4XXM4E,35240
+ddeutil/workflow/logs.py,sha256=iVtyl8i69y7t07tAuWkihc54WlkHCcBy_Ur0WtzJ_lM,31367
+ddeutil/workflow/params.py,sha256=1u8gXs1ZyMq-2eD9H8L7Yjfu5t7b_OzjA0fJvhxdYWY,12505
+ddeutil/workflow/result.py,sha256=4M9VCcveI8Yz6ZrnI-67SZlry-Z8G7e0hziy1k-pklk,5906
+ddeutil/workflow/reusables.py,sha256=PP1K264a4YeFpEFsXVLicAFSjHdDXnPVAGmASXJpVmc,21706
+ddeutil/workflow/stages.py,sha256=kS4g7N7kg31kEoveI6cZ9bzk_V0hm87lzyZpGU4-fss,93525
+ddeutil/workflow/utils.py,sha256=1i82dWXCnbpEXhYtztNz6IPy7uOyNDY2BZcXJCAfzno,9611
+ddeutil/workflow/workflow.py,sha256=8Z_h8OtNHkaGf8MJixTHNeXsyA4mBlYtHDqj0oEVFBs,44858
+ddeutil/workflow/api/__init__.py,sha256=0UIilYwW29RL6HrCRHACSWvnATJVLSJzXiCMny0bHQk,2627
+ddeutil/workflow/api/logs.py,sha256=NMTnOnsBrDB5129329xF2myLdrb-z9k1MQrmrP7qXJw,1818
+ddeutil/workflow/api/routes/__init__.py,sha256=jC1pM7q4_eo45IyO3hQbbe6RnL9B8ibRq_K6aCMP6Ag,434
+ddeutil/workflow/api/routes/job.py,sha256=PWDbIvvoPKbG7JQHwM80e6YIB7nnsGeZaqNSOTQE99k,2155
+ddeutil/workflow/api/routes/logs.py,sha256=QJH8IF102897WLfCJ29-1g15wl29M9Yq6omroZfbahs,5305
+ddeutil/workflow/api/routes/workflows.py,sha256=Gmg3e-K5rfi95pbRtWI_aIr5C089sIde_vefZVvh3U0,4420
+ddeutil_workflow-0.0.64.dist-info/licenses/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
+ddeutil_workflow-0.0.64.dist-info/METADATA,sha256=ytJpKiesulHXeHt1JaSuaeOsp8o6D7b37r8bMuwnXCg,16957
+ddeutil_workflow-0.0.64.dist-info/WHEEL,sha256=DnLRTWE75wApRYVsjgc6wsVswC54sMSJhAEd4xhDpBk,91
+ddeutil_workflow-0.0.64.dist-info/entry_points.txt,sha256=qDTpPSauL0ciO6T4iSVt8bJeYrVEkkoEEw_RlGx6Kgk,63
+ddeutil_workflow-0.0.64.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
+ddeutil_workflow-0.0.64.dist-info/RECORD,,

ddeutil/workflow/api/routes/schedules.py

@@ -1,141 +0,0 @@
-# ------------------------------------------------------------------------------
-# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
-# Licensed under the MIT License. See LICENSE in the project root for
-# license information.
-# ------------------------------------------------------------------------------
-from __future__ import annotations
-
-import copy
-from datetime import datetime, timedelta
-
-from fastapi import APIRouter, HTTPException, Request
-from fastapi import status as st
-from fastapi.responses import UJSONResponse
-
-from ...conf import config
-from ...logs import get_logger
-from ...scheduler import Schedule
-
-logger = get_logger("uvicorn.error")
-schedule_route = APIRouter(
-    prefix="/schedules",
-    tags=["schedules"],
-    default_response_class=UJSONResponse,
-)
-
-
-@schedule_route.get(path="/{name}", status_code=st.HTTP_200_OK)
-async def get_schedules(name: str):
-    """Get schedule object."""
-    try:
-        schedule: Schedule = Schedule.from_conf(name=name, extras={})
-    except ValueError:
-        raise HTTPException(
-            status_code=st.HTTP_404_NOT_FOUND,
-            detail=f"Schedule name: {name!r} does not found in /conf path",
-        ) from None
-    return schedule.model_dump(
-        by_alias=True,
-        exclude_none=True,
-        exclude_unset=True,
-        exclude_defaults=True,
-    )
-
-
-@schedule_route.get(path="/deploy/", status_code=st.HTTP_200_OK)
-async def get_deploy_schedulers(request: Request):
-    snapshot = copy.deepcopy(request.state.scheduler)
-    return {"schedule": snapshot}
-
-
-@schedule_route.get(path="/deploy/{name}", status_code=st.HTTP_200_OK)
-async def get_deploy_scheduler(request: Request, name: str):
-    if name in request.state.scheduler:
-        schedule = Schedule.from_conf(name)
-        getter: list[dict[str, dict[str, list[datetime]]]] = []
-        for workflow in schedule.workflows:
-            getter.append(
-                {
-                    workflow.name: {
-                        "queue": copy.deepcopy(
-                            request.state.workflow_queue[workflow.name]
-                        ),
-                        "running": copy.deepcopy(
-                            request.state.workflow_running[workflow.name]
-                        ),
-                    }
-                }
-            )
-        return {
-            "message": f"Getting {name!r} to schedule listener.",
-            "scheduler": getter,
-        }
-    raise HTTPException(
-        status_code=st.HTTP_404_NOT_FOUND,
-        detail=f"Does not found {name!r} in schedule listener",
-    )
-
-
-@schedule_route.post(path="/deploy/{name}", status_code=st.HTTP_202_ACCEPTED)
-async def add_deploy_scheduler(request: Request, name: str):
-    """Adding schedule name to application state store."""
-    if name in request.state.scheduler:
-        raise HTTPException(
-            status_code=st.HTTP_302_FOUND,
-            detail=f"This schedule {name!r} already exists in scheduler list.",
-        )
-
-    request.state.scheduler.append(name)
-
-    start_date: datetime = datetime.now(tz=config.tz)
-    start_date_waiting: datetime = (start_date + timedelta(minutes=1)).replace(
-        second=0, microsecond=0
-    )
-
-    # NOTE: Create a pair of workflow and on from schedule model.
-    try:
-        schedule: Schedule = Schedule.from_conf(name)
-    except ValueError as err:
-        request.state.scheduler.remove(name)
-        logger.exception(err)
-        raise HTTPException(
-            status_code=st.HTTP_404_NOT_FOUND,
-            detail=str(err),
-        ) from None
-
-    request.state.workflow_tasks.extend(
-        schedule.tasks(
-            start_date_waiting,
-            queue=request.state.workflow_queue,
-        ),
-    )
-    return {
-        "message": f"Adding {name!r} to schedule listener.",
-        "start_date": start_date_waiting,
-    }
-
-
-@schedule_route.delete(path="/deploy/{name}", status_code=st.HTTP_202_ACCEPTED)
-async def del_deploy_scheduler(request: Request, name: str):
-    """Delete workflow task on the schedule listener."""
-    if name in request.state.scheduler:
-
-        # NOTE: Remove current schedule name from the state.
-        request.state.scheduler.remove(name)
-
-        schedule: Schedule = Schedule.from_conf(name)
-
-        for task in schedule.tasks(datetime.now(tz=config.tz), queue={}):
-            if task in request.state.workflow_tasks:
-                request.state.workflow_tasks.remove(task)
-
-        for workflow in schedule.workflows:
-            if workflow.alias in request.state.workflow_queue:
-                request.state.workflow_queue.pop(workflow.alias)
-
-        return {"message": f"Deleted schedule {name!r} in listener."}
-
-    raise HTTPException(
-        status_code=st.HTTP_404_NOT_FOUND,
-        detail=f"Does not found schedule {name!r} in listener",
-    )