ddeutil-workflow 0.0.40__py3-none-any.whl → 0.0.41__py3-none-any.whl

ddeutil/workflow/stages.py

@@ -3,6 +3,7 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
+# [x] Use config
 """Stage Model that use for getting stage data template from the Job Model.
 The stage handle the minimize task that run in some thread (same thread at
 its job owner) that mean it is the lowest executor of a workflow that can
@@ -41,6 +42,7 @@ from inspect import Parameter
 from pathlib import Path
 from subprocess import CompletedProcess
 from textwrap import dedent
+from threading import Event
 from typing import Annotated, Optional, Union
 
 from pydantic import BaseModel, Field
@@ -48,11 +50,10 @@ from pydantic.functional_validators import model_validator
 from typing_extensions import Self
 
 from .__types import DictData, DictStr, TupleStr
-from .caller import TagFunc, extract_call
 from .conf import config
 from .exceptions import StageException, to_dict
 from .result import Result, Status
-from .templates import not_in_template, param2template
+from .reusables import TagFunc, extract_call, not_in_template, param2template
 from .utils import (
     gen_id,
     make_exec,
@@ -93,6 +94,10 @@ class BaseStage(BaseModel, ABC):
         description="A stage condition statement to allow stage executable.",
         alias="if",
     )
+    extras: DictData = Field(
+        default_factory=dict,
+        description="An extra override values.",
+    )
 
     @property
     def iden(self) -> str:
@@ -126,7 +131,11 @@ class BaseStage(BaseModel, ABC):
 
     @abstractmethod
     def execute(
-        self, params: DictData, *, result: Result | None = None
+        self,
+        params: DictData,
+        *,
+        result: Result | None = None,
+        event: Event | None = None,
     ) -> Result:
         """Execute abstraction method that action something by sub-model class.
         This is important method that make this class is able to be the stage.
@@ -135,11 +144,22 @@ class BaseStage(BaseModel, ABC):
             execution.
         :param result: (Result) A result object for keeping context and status
             data.
+        :param event: (Event) An event manager that use to track parent execute
+            was not force stopped.
 
         :rtype: Result
         """
         raise NotImplementedError("Stage should implement `execute` method.")
 
+    async def axecute(
+        self,
+        params: DictData,
+        *,
+        result: Result | None = None,
+        event: Event | None,
+    ) -> Result:  # pragma: no cov
+        ...
+
     def handler_execute(
         self,
         params: DictData,
@@ -149,6 +169,7 @@ class BaseStage(BaseModel, ABC):
         result: Result | None = None,
         raise_error: bool = False,
         to: DictData | None = None,
+        event: Event | None = None,
     ) -> Result:
         """Handler stage execution result from the stage `execute` method.
 
@@ -183,6 +204,7 @@ class BaseStage(BaseModel, ABC):
         :param raise_error: (bool) A flag that all this method raise error
         :param to: (DictData) A target object for auto set the return output
             after execution.
+        :param event: (Event) An event manager that pass to the stage execution.
 
         :rtype: Result
         """
@@ -194,7 +216,7 @@ class BaseStage(BaseModel, ABC):
         )
 
         try:
-            rs: Result = self.execute(params, result=result)
+            rs: Result = self.execute(params, result=result, event=event)
             if to is not None:
                 return self.set_outputs(rs.context, to=to)
             return rs
@@ -322,7 +344,11 @@ class EmptyStage(BaseStage):
     )
 
     def execute(
-        self, params: DictData, *, result: Result | None = None
+        self,
+        params: DictData,
+        *,
+        result: Result | None = None,
+        event: Event | None = None,
     ) -> Result:
         """Execution method for the Empty stage that do only logging out to
         stdout. This method does not use the `handler_result` decorator because
@@ -335,6 +361,8 @@ class EmptyStage(BaseStage):
             But this stage does not pass any output.
         :param result: (Result) A result object for keeping context and status
             data.
+        :param event: (Event) An event manager that use to track parent execute
+            was not force stopped.
 
         :rtype: Result
         """
@@ -354,9 +382,25 @@ class EmptyStage(BaseStage):
         return result.catch(status=Status.SUCCESS)
 
     # TODO: Draft async execute method for the perf improvement.
-    async def aexecute(
-        self, params: DictData, *, result: Result | None = None
+    async def axecute(
+        self,
+        params: DictData,
+        *,
+        result: Result | None = None,
+        event: Event | None,
     ) -> Result:  # pragma: no cov
+        """Async execution method for this Empty stage that only logging out to
+        stdout.
+
+        :param params: (DictData) A context data that want to add output result.
+            But this stage does not pass any output.
+        :param result: (Result) A result object for keeping context and status
+            data.
+        :param event: (Event) An event manager that use to track parent execute
+            was not force stopped.
+
+        :rtype: Result
+        """
         if result is None:  # pragma: no cov
             result: Result = Result(
                 run_id=gen_id(self.name + (self.id or ""), unique=True)
@@ -367,7 +411,11 @@ class EmptyStage(BaseStage):
             f"( {param2template(self.echo, params=params) or '...'} )"
         )
 
-        await asyncio.sleep(1)
+        if self.sleep > 0:
+            if self.sleep > 5:
+                result.trace.info(f"[STAGE]: ... sleep ({self.sleep} seconds)")
+            await asyncio.sleep(self.sleep)
+
         return result.catch(status=Status.SUCCESS)
 
 
@@ -438,7 +486,11 @@ class BashStage(BaseStage):
         Path(f"./{f_name}").unlink()
 
     def execute(
-        self, params: DictData, *, result: Result | None = None
+        self,
+        params: DictData,
+        *,
+        result: Result | None = None,
+        event: Event | None = None,
     ) -> Result:
         """Execute the Bash statement with the Python build-in ``subprocess``
         package.
@@ -446,6 +498,8 @@ class BashStage(BaseStage):
         :param params: A parameter data that want to use in this execution.
         :param result: (Result) A result object for keeping context and status
             data.
+        :param event: (Event) An event manager that use to track parent execute
+            was not force stopped.
 
         :rtype: Result
         """
@@ -555,7 +609,11 @@ class PyStage(BaseStage):
         return to
 
     def execute(
-        self, params: DictData, *, result: Result | None = None
+        self,
+        params: DictData,
+        *,
+        result: Result | None = None,
+        event: Event | None = None,
     ) -> Result:
         """Execute the Python statement that pass all globals and input params
         to globals argument on ``exec`` build-in function.
@@ -563,6 +621,8 @@ class PyStage(BaseStage):
         :param params: A parameter that want to pass before run any statement.
         :param result: (Result) A result object for keeping context and status
             data.
+        :param event: (Event) An event manager that use to track parent execute
+            was not force stopped.
 
         :rtype: Result
         """
@@ -629,7 +689,11 @@ class CallStage(BaseStage):
     )
 
     def execute(
-        self, params: DictData, *, result: Result | None = None
+        self,
+        params: DictData,
+        *,
+        result: Result | None = None,
+        event: Event | None = None,
     ) -> Result:
         """Execute the Call function that already in the call registry.
 
@@ -638,11 +702,12 @@ class CallStage(BaseStage):
         :raise TypeError: When the return type of call function does not be
             dict type.
 
-        :param params: A parameter that want to pass before run any statement.
-        :type params: DictData
+        :param params: (DictData) A parameter that want to pass before run any
+            statement.
         :param result: (Result) A result object for keeping context and status
             data.
-        :type: str | None
+        :param event: (Event) An event manager that use to track parent execute
+            was not force stopped.
 
         :rtype: Result
         """
@@ -728,7 +793,11 @@ class TriggerStage(BaseStage):
     )
 
     def execute(
-        self, params: DictData, *, result: Result | None = None
+        self,
+        params: DictData,
+        *,
+        result: Result | None = None,
+        event: Event | None = None,
     ) -> Result:
         """Trigger another workflow execution. It will wait the trigger
         workflow running complete before catching its result.
@@ -736,6 +805,8 @@ class TriggerStage(BaseStage):
         :param params: A parameter data that want to use in this execution.
         :param result: (Result) A result object for keeping context and status
             data.
+        :param event: (Event) An event manager that use to track parent execute
+            was not force stopped.
 
         :rtype: Result
         """
@@ -838,7 +909,11 @@ class ParallelStage(BaseStage):  # pragma: no cov
         return context
 
     def execute(
-        self, params: DictData, *, result: Result | None = None
+        self,
+        params: DictData,
+        *,
+        result: Result | None = None,
+        event: Event | None = None,
     ) -> Result:
         """Execute the stages that parallel each branch via multi-threading mode
         or async mode by changing `async_mode` flag.
@@ -846,6 +921,8 @@ class ParallelStage(BaseStage):  # pragma: no cov
         :param params: A parameter that want to pass before run any statement.
         :param result: (Result) A result object for keeping context and status
             data.
+        :param event: (Event) An event manager that use to track parent execute
+            was not force stopped.
 
         :rtype: Result
         """
@@ -910,19 +987,34 @@ class ForEachStage(BaseStage):
         ),
     )
     stages: list[Stage] = Field(
+        default_factory=list,
         description=(
             "A list of stage that will run with each item in the foreach field."
         ),
     )
+    concurrent: int = Field(
+        default=1,
+        gt=0,
+        description=(
+            "A concurrent value allow to run each item at the same time. It "
+            "will be sequential mode if this value equal 1."
+        ),
+    )
 
     def execute(
-        self, params: DictData, *, result: Result | None = None
+        self,
+        params: DictData,
+        *,
+        result: Result | None = None,
+        event: Event | None = None,
     ) -> Result:
         """Execute the stages that pass each item form the foreach field.
 
         :param params: A parameter that want to pass before run any statement.
         :param result: (Result) A result object for keeping context and status
             data.
+        :param event: (Event) An event manager that use to track parent execute
+            was not force stopped.
 
         :rtype: Result
         """
@@ -933,6 +1025,7 @@ class ForEachStage(BaseStage):
 
         rs: DictData = {"items": self.foreach, "foreach": {}}
         status = Status.SUCCESS
+        # TODO: Implement concurrent more than 1.
         for item in self.foreach:
             result.trace.debug(f"[STAGE]: Execute foreach item: {item!r}")
             params["item"] = item
@@ -969,6 +1062,33 @@ class ForEachStage(BaseStage):
         return result.catch(status=status, context=rs)
 
 
+# TODO: Not implement this stages yet
+class UntilStage(BaseStage):  # pragma: no cov
+    """Until execution stage."""
+
+    until: str = Field(description="A until condition.")
+    item: Union[str, int, bool] = Field(description="An initial value.")
+    stages: list[Stage] = Field(
+        default_factory=list,
+        description=(
+            "A list of stage that will run with each item until condition "
+            "correct."
+        ),
+    )
+    max_until_not_change: int = Field(
+        default=3,
+        description="The maximum value of loop if condition not change.",
+    )
+
+    def execute(
+        self,
+        params: DictData,
+        *,
+        result: Result | None = None,
+        event: Event | None = None,
+    ) -> Result: ...
+
+
 # TODO: Not implement this stages yet
 class IfStage(BaseStage):  # pragma: no cov
     """If execution stage.
@@ -1008,19 +1128,45 @@ class IfStage(BaseStage):  # pragma: no cov
     match: list[dict[str, Union[str, Stage]]]
 
     def execute(
-        self, params: DictData, *, result: Result | None = None
+        self,
+        params: DictData,
+        *,
+        result: Result | None = None,
+        event: Event | None = None,
     ) -> Result: ...
 
 
 class RaiseStage(BaseStage):  # pragma: no cov
+    """Raise error stage that raise StageException that use a message field for
+    making error message before raise.
+
+    Data Validate:
+        >>> stage = {
+        ...     "name": "Raise stage",
+        ...     "raise": "raise this stage",
+        ... }
+
+    """
+
     message: str = Field(
         description="An error message that want to raise",
         alias="raise",
     )
 
     def execute(
-        self, params: DictData, *, result: Result | None = None
+        self,
+        params: DictData,
+        *,
+        result: Result | None = None,
+        event: Event | None = None,
     ) -> Result:
+        """Raise the stage."""
+        if result is None:  # pragma: no cov
+            result: Result = Result(
+                run_id=gen_id(self.name + (self.id or ""), unique=True)
+            )
+
+        result.trace.error(f"[STAGE]: ... raise ( {self.message} )")
         raise StageException(self.message)
 
 
@@ -1031,7 +1177,11 @@ class HookStage(BaseStage):  # pragma: no cov
     callback: str
 
     def execute(
-        self, params: DictData, *, result: Result | None = None
+        self,
+        params: DictData,
+        *,
+        result: Result | None = None,
+        event: Event | None = None,
     ) -> Result: ...
 
 
@@ -1039,13 +1189,19 @@ class HookStage(BaseStage):  # pragma: no cov
 class DockerStage(BaseStage):  # pragma: no cov
     """Docker container stage execution."""
 
-    image: str
+    image: str = Field(
+        description="A Docker image url with tag that want to run.",
+    )
     env: DictData = Field(default_factory=dict)
     volume: DictData = Field(default_factory=dict)
     auth: DictData = Field(default_factory=dict)
 
     def execute(
-        self, params: DictData, *, result: Result | None = None
+        self,
+        params: DictData,
+        *,
+        result: Result | None = None,
+        event: Event | None = None,
     ) -> Result: ...
 
 
@@ -1059,7 +1215,11 @@ class VirtualPyStage(PyStage):  # pragma: no cov
     def create_py_file(self, py: str, run_id: str | None): ...
 
     def execute(
-        self, params: DictData, *, result: Result | None = None
+        self,
+        params: DictData,
+        *,
+        result: Result | None = None,
+        event: Event | None = None,
     ) -> Result:
         return super().execute(params, result=result)
 
@@ -1068,7 +1228,11 @@ class VirtualPyStage(PyStage):  # pragma: no cov
 class SensorStage(BaseStage):  # pragma: no cov
 
     def execute(
-        self, params: DictData, *, result: Result | None = None
+        self,
+        params: DictData,
+        *,
+        result: Result | None = None,
+        event: Event | None = None,
     ) -> Result: ...
 
 

ddeutil/workflow/utils.py

@@ -3,6 +3,8 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
+# [ ] Use config
+"""Utility function model."""
 from __future__ import annotations
 
 import stat
@@ -154,6 +156,15 @@ def gen_id(
     ).hexdigest()
 
 
+def default_gen_id() -> str:
+    """Return running ID which use for making default ID for the Result model if
+    a run_id field initializes at the first time.
+
+    :rtype: str
+    """
+    return gen_id("manual", unique=True)
+
+
 def make_exec(path: str | Path) -> None:
     """Change mode of file to be executable file.
 

ddeutil/workflow/workflow.py

@@ -31,14 +31,14 @@ from typing_extensions import Self
 
 from .__cron import CronJob, CronRunner
 from .__types import DictData, TupleStr
-from .audit import Audit, get_audit
 from .conf import Loader, SimLoad, config, get_logger
 from .cron import On
 from .exceptions import JobException, WorkflowException
 from .job import Job, TriggerState
+from .logs import Audit, get_audit
 from .params import Param
 from .result import Result, Status
-from .templates import has_template, param2template
+from .reusables import has_template, param2template
 from .utils import (
     gen_id,
     get_dt_now,
@@ -271,6 +271,10 @@ class Workflow(BaseModel):
         default_factory=dict,
         description="A mapping of job ID and job model that already loaded.",
     )
+    extras: DictData = Field(
+        default_factory=dict,
+        description="An extra override values.",
+    )
 
     @classmethod
     def from_loader(
@@ -297,9 +301,10 @@ class Workflow(BaseModel):
             raise ValueError(f"Type {loader.type} does not match with {cls}")
 
         loader_data: DictData = copy.deepcopy(loader.data)
-
-        # NOTE: Add name to loader data
         loader_data["name"] = name.replace(" ", "_")
+        if externals:  # pragma: no cov
+            loader_data["extras"] = externals
+
         cls.__bypass_on__(
             loader_data, path=loader.conf_path, externals=externals
         )
@@ -333,9 +338,10 @@ class Workflow(BaseModel):
             raise ValueError(f"Type {loader.type} does not match with {cls}")
 
         loader_data: DictData = copy.deepcopy(loader.data)
-
-        # NOTE: Add name to loader data
         loader_data["name"] = name.replace(" ", "_")
+        if externals:  # pragma: no cov
+            loader_data["extras"] = externals
+
         cls.__bypass_on__(loader_data, path=path, externals=externals)
         return cls.model_validate(obj=loader_data)
 
@@ -446,7 +452,6 @@ class Workflow(BaseModel):
                     f"{self.name!r}."
                 )
 
-            # NOTE: update a job id with its job id from workflow template
             self.jobs[job].id = job
 
         # VALIDATE: Validate workflow name should not dynamic with params
@@ -476,7 +481,10 @@ class Workflow(BaseModel):
                 f"A Job {name!r} does not exists in this workflow, "
                 f"{self.name!r}"
             )
-        return self.jobs[name]
+        job: Job = self.jobs[name]
+        if self.extras:
+            job.extras = self.extras
+        return job
 
     def parameterize(self, params: DictData) -> DictData:
         """Prepare a passing parameters before use it in execution process.

{ddeutil_workflow-0.0.40.dist-info → ddeutil_workflow-0.0.41.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddeutil-workflow
-Version: 0.0.40
+Version: 0.0.41
 Summary: Lightweight workflow orchestration
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT
@@ -27,6 +27,12 @@ Requires-Dist: ddeutil-io[toml,yaml]>=0.2.10
 Requires-Dist: pydantic==2.11.1
 Requires-Dist: python-dotenv==1.1.0
 Requires-Dist: schedule<2.0.0,==1.2.2
+Provides-Extra: all
+Requires-Dist: fastapi<1.0.0,>=0.115.0; extra == "all"
+Requires-Dist: httpx; extra == "all"
+Requires-Dist: ujson; extra == "all"
+Requires-Dist: aiofiles; extra == "all"
+Requires-Dist: aiohttp; extra == "all"
 Provides-Extra: api
 Requires-Dist: fastapi<1.0.0,>=0.115.0; extra == "api"
 Requires-Dist: httpx; extra == "api"

ddeutil_workflow-0.0.41.dist-info/RECORD

@@ -0,0 +1,31 @@
+ddeutil/workflow/__about__.py,sha256=chYiprva6VKQg6XFgrwBoMIWSlSshsb-yyHtFoiipvc,28
+ddeutil/workflow/__cron.py,sha256=h8rLeIUAAEB2SdZ4Jhch7LU1Yl3bbJ-iNNJ3tQ0eYVM,28095
+ddeutil/workflow/__init__.py,sha256=cYWwG2utpsYvdwqvkFSRWi_Q6gylDgNQBcIWcF5NFs4,1861
+ddeutil/workflow/__types.py,sha256=8jBdbfb3aZSetjz0mvNrpGHwwxJff7mK8_4v41cLqlc,4316
+ddeutil/workflow/conf.py,sha256=wQXL5bfEUx8DbS6cEIRtsDjvQZzw7sGPvs-g5r2zOeM,12095
+ddeutil/workflow/context.py,sha256=vsk4JQL7t3KsnKPfshw3O7YrPFo2h4rnnNd3B-G9Kj4,1700
+ddeutil/workflow/cron.py,sha256=j8EeoHst70toRfnD_frix41vrI-eLYVJkZ9yeJtpfnI,8871
+ddeutil/workflow/exceptions.py,sha256=fO37f9p7lOjIJgVOpKE_1X44yJTwBepyukZV9a7NNm4,1241
+ddeutil/workflow/job.py,sha256=TAtOFOFWHGqXlc6QWvtum1KpfWFDs5AfNbYMtwvrIE8,30102
+ddeutil/workflow/logs.py,sha256=nbyoUONqSZ4QpowgMs962m-Qs-UDfcyMHs4bxIAT470,18782
+ddeutil/workflow/params.py,sha256=Mv-D2DY5inm1ug0lsgCPDkO5wT_AUhc5XEF5jxgDx6U,8036
+ddeutil/workflow/result.py,sha256=ynZB0g_vEEXn24034J-hatjNWDBmRAj38S8SqGRM-8I,4029
+ddeutil/workflow/reusables.py,sha256=Rw7qS2cQM4SBxuIyqiyJN5yZNBfObISXWRIgDUMC2fY,17449
+ddeutil/workflow/scheduler.py,sha256=Y4VuVDIHz64l3IN-7tKP76qrMKShpgXXjX64mKjSWLo,27651
+ddeutil/workflow/stages.py,sha256=Kp3VLhKMMlgQyMLEYWJfBdR8DoCD0nuzSY0-r-P8eS8,41574
+ddeutil/workflow/utils.py,sha256=XwkxOpPaHbvjCKCGA3kVriEYabKyZ_P6pTbkYPnK704,7380
+ddeutil/workflow/workflow.py,sha256=s2z_QMTmfsRjiw25CuKFQIqY9smCbKjFwbBwCZkikfc,49615
+ddeutil/workflow/api/__init__.py,sha256=F53NMBWtb9IKaDWkPU5KvybGGfKAcbehgn6TLBwHuuM,21
+ddeutil/workflow/api/api.py,sha256=C9f6w11zE1SHz8lwjRxVOwrO90pCMr9REj2WLQsO0lI,5190
+ddeutil/workflow/api/log.py,sha256=NMTnOnsBrDB5129329xF2myLdrb-z9k1MQrmrP7qXJw,1818
+ddeutil/workflow/api/repeat.py,sha256=cycd1-91j-4v6uY1SkrZHd9l95e-YgVC4UCSNNFuGJ8,5277
+ddeutil/workflow/api/routes/__init__.py,sha256=qoGtOMyVgQ5nTUc8J8wH27A8isaxl3IFCX8qoyibeCY,484
+ddeutil/workflow/api/routes/job.py,sha256=YVta083i8vU8-o4WdKFwDpfdC9vN1dZ6goZSmNlQXHA,1954
+ddeutil/workflow/api/routes/logs.py,sha256=TeRDrEelbKS2Hu_EovgLh0bOdmSv9mfnrIZsrE7uPD4,5353
+ddeutil/workflow/api/routes/schedules.py,sha256=uWYDOwlV8w56hKQmfkQFwdZ6t2gZSJeCdBIzMmJenAQ,4824
+ddeutil/workflow/api/routes/workflows.py,sha256=TiGlwosHucccVmGv_SE9mvs3_BR2c6QZWQhw--aAv5w,4537
+ddeutil_workflow-0.0.41.dist-info/licenses/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
+ddeutil_workflow-0.0.41.dist-info/METADATA,sha256=ACsWRIhGZuaRo_d400dlAFsSRlNcha8Vy0PzVMkE3NU,19729
+ddeutil_workflow-0.0.41.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+ddeutil_workflow-0.0.41.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
+ddeutil_workflow-0.0.41.dist-info/RECORD,,