ddeutil-workflow 0.0.66__py3-none-any.whl → 0.0.67__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1 +1 @@
-__version__: str = "0.0.66"
+__version__: str = "0.0.67"

ddeutil/workflow/cli.py

@@ -48,6 +48,7 @@ def api(
     host: Annotated[str, typer.Option(help="A host url.")] = "0.0.0.0",
     port: Annotated[int, typer.Option(help="A port url.")] = 80,
     debug: Annotated[bool, typer.Option(help="A debug mode flag")] = True,
+    worker: Annotated[int, typer.Option(help="A worker number")] = None,
 ):
     """
     Provision API application from the FastAPI.
@@ -59,6 +60,7 @@ def api(
         port=port,
         log_config=uvicorn.config.LOGGING_CONFIG | LOGGING_CONFIG,
         log_level=("DEBUG" if debug else "INFO"),
+        workers=worker,
     )
 
 

ddeutil/workflow/errors.py

@@ -90,9 +90,6 @@ class ResultError(UtilError): ...
 class StageError(BaseError): ...
 
 
-class StageRetryError(StageError): ...
-
-
 class StageCancelError(StageError): ...
 
 

ddeutil/workflow/result.py

@@ -11,7 +11,7 @@ from __future__ import annotations
 
 from dataclasses import field
 from datetime import datetime
-from enum import IntEnum, auto
+from enum import Enum
 from typing import Optional, Union
 
 from pydantic import ConfigDict
@@ -36,16 +36,16 @@ from .logs import TraceModel, get_dt_tznow, get_trace
 from .utils import default_gen_id, gen_id, get_dt_now
 
 
-class Status(IntEnum):
+class Status(str, Enum):
     """Status Int Enum object that use for tracking execution status to the
     Result dataclass object.
     """
 
-    SUCCESS = auto()
-    FAILED = auto()
-    WAIT = auto()
-    SKIP = auto()
-    CANCEL = auto()
+    SUCCESS = "SUCCESS"
+    FAILED = "FAILED"
+    WAIT = "WAIT"
+    SKIP = "SKIP"
+    CANCEL = "CANCEL"
 
     @property
     def emoji(self) -> str:  # pragma: no cov
@@ -67,6 +67,9 @@ class Status(IntEnum):
     def __str__(self) -> str:
         return self.name
 
+    def is_result(self) -> bool:
+        return self in ResultStatuses
+
 
 SUCCESS = Status.SUCCESS
 FAILED = Status.FAILED
@@ -74,6 +77,8 @@ WAIT = Status.WAIT
 SKIP = Status.SKIP
 CANCEL = Status.CANCEL
 
+ResultStatuses: list[Status] = [SUCCESS, FAILED, CANCEL, SKIP]
+
 
 def validate_statuses(statuses: list[Status]) -> Status:
     """Validate the final status from list of Status object.

ddeutil/workflow/stages.py

@@ -62,10 +62,9 @@ from pydantic import BaseModel, Field, ValidationError
 from pydantic.functional_validators import field_validator, model_validator
 from typing_extensions import Self
 
-from . import StageCancelError, StageRetryError
 from .__types import DictData, DictStr, StrOrInt, StrOrNone, TupleStr
 from .conf import dynamic, pass_env
-from .errors import StageError, StageSkipError, to_dict
+from .errors import StageCancelError, StageError, StageSkipError, to_dict
 from .result import (
     CANCEL,
     FAILED,
@@ -252,16 +251,20 @@ class BaseStage(BaseModel, ABC):
                 f"[STAGE]: Handler {to_train(self.__class__.__name__)}: "
                 f"{self.name!r}."
             )
+
+            # NOTE: Show the description of this stage before execution.
             if self.desc:
                 result.trace.debug(f"[STAGE]: Description:||{self.desc}||")
 
+            # VALIDATE: Checking stage condition before execution.
             if self.is_skipped(params):
                 raise StageSkipError(
                     f"Skip because condition {self.condition} was valid."
                 )
+
             # NOTE: Start call wrapped execution method that will use custom
             #   execution before the real execution from inherit stage model.
-            result_caught: Result = self.__execute(
+            result_caught: Result = self._execute(
                 params, result=result, event=event
             )
             if result_caught.status == WAIT:
@@ -296,7 +299,7 @@ class BaseStage(BaseModel, ABC):
             )
             return result.catch(status=FAILED, context={"errors": to_dict(e)})
 
-    def __execute(
+    def _execute(
         self, params: DictData, result: Result, event: Optional[Event]
     ) -> Result:
         """Wrapped the execute method before returning to handler execution.
@@ -514,11 +517,14 @@ class BaseAsyncStage(BaseStage, ABC):
                 f"[STAGE]: Handler {to_train(self.__class__.__name__)}: "
                 f"{self.name!r}."
             )
+
+            # NOTE: Show the description of this stage before execution.
             if self.desc:
                 await result.trace.adebug(
                     f"[STAGE]: Description:||{self.desc}||"
                 )
 
+            # VALIDATE: Checking stage condition before execution.
             if self.is_skipped(params=params):
                 raise StageSkipError(
                     f"Skip because condition {self.condition} was valid."
@@ -526,7 +532,7 @@ class BaseAsyncStage(BaseStage, ABC):
 
             # NOTE: Start call wrapped execution method that will use custom
             #   execution before the real execution from inherit stage model.
-            result_caught: Result = await self.__axecute(
+            result_caught: Result = await self._axecute(
                 params, result=result, event=event
             )
             if result_caught.status == WAIT:
@@ -561,7 +567,7 @@ class BaseAsyncStage(BaseStage, ABC):
             )
             return result.catch(status=FAILED, context={"errors": to_dict(e)})
 
-    async def __axecute(
+    async def _axecute(
         self, params: DictData, result: Result, event: Optional[Event]
     ) -> Result:
         """Wrapped the axecute method before returning to handler axecute.
@@ -591,7 +597,7 @@ class BaseRetryStage(BaseAsyncStage, ABC):  # pragma: no cov
         description="Retry number if stage execution get the error.",
     )
 
-    def __execute(
+    def _execute(
         self,
         params: DictData,
         result: Result,
@@ -610,15 +616,50 @@ class BaseRetryStage(BaseAsyncStage, ABC):  # pragma: no cov
         :rtype: Result
         """
         current_retry: int = 0
-        with current_retry < (self.retry + 1):
+        exception: Exception
+
+        # NOTE: First execution for not pass to retry step if it passes.
+        try:
+            result.catch(status=WAIT)
+            return self.execute(
+                params | {"retry": current_retry},
+                result=result,
+                event=event,
+            )
+        except Exception as e:
+            current_retry += 1
+            exception = e
+
+        if self.retry == 0:
+            raise exception
+
+        result.trace.warning(
+            f"[STAGE]: Retry count: {current_retry} ... "
+            f"( {exception.__class__.__name__} )"
+        )
+
+        while current_retry < (self.retry + 1):
             try:
                 result.catch(status=WAIT, context={"retry": current_retry})
-                return self.execute(params, result=result, event=event)
-            except StageRetryError:
+                return self.execute(
+                    params | {"retry": current_retry},
+                    result=result,
+                    event=event,
+                )
+            except Exception as e:
                 current_retry += 1
-        raise StageError(f"Reach the maximum of retry number: {self.retry}.")
+                result.trace.warning(
+                    f"[STAGE]: Retry count: {current_retry} ... "
+                    f"( {e.__class__.__name__} )"
+                )
+                exception = e
+
+        result.trace.error(
+            f"[STAGE]: Reach the maximum of retry number: {self.retry}."
+        )
+        raise exception
 
-    async def __axecute(
+    async def _axecute(
         self,
         params: DictData,
         result: Result,
@@ -637,13 +678,48 @@ class BaseRetryStage(BaseAsyncStage, ABC):  # pragma: no cov
         :rtype: Result
         """
         current_retry: int = 0
-        with current_retry < (self.retry + 1):
+        exception: Exception
+
+        # NOTE: First execution for not pass to retry step if it passes.
+        try:
+            result.catch(status=WAIT)
+            return await self.axecute(
+                params | {"retry": current_retry},
+                result=result,
+                event=event,
+            )
+        except Exception as e:
+            current_retry += 1
+            exception = e
+
+        if self.retry == 0:
+            raise exception
+
+        await result.trace.awarning(
+            f"[STAGE]: Retry count: {current_retry} ... "
+            f"( {exception.__class__.__name__} )"
+        )
+
+        while current_retry < (self.retry + 1):
             try:
                 result.catch(status=WAIT, context={"retry": current_retry})
-                return await self.axecute(params, result=result, event=event)
-            except StageRetryError:
+                return await self.axecute(
+                    params | {"retry": current_retry},
+                    result=result,
+                    event=event,
+                )
+            except Exception as e:
                 current_retry += 1
-        raise StageError(f"Reach the maximum of retry number: {self.retry}.")
+                await result.trace.awarning(
+                    f"[STAGE]: Retry count: {current_retry} ... "
+                    f"( {e.__class__.__name__} )"
+                )
+                exception = e
+
+        await result.trace.aerror(
+            f"[STAGE]: Reach the maximum of retry number: {self.retry}."
+        )
+        raise exception
 
 
 class EmptyStage(BaseAsyncStage):
@@ -765,7 +841,7 @@ class EmptyStage(BaseAsyncStage):
         return result.catch(status=SUCCESS)
 
 
-class BashStage(BaseAsyncStage):
+class BashStage(BaseRetryStage):
     """Bash stage executor that execute bash script on the current OS.
     If your current OS is Windows, it will run on the bash from the current WSL.
     It will use `bash` for Windows OS and use `sh` for Linux OS.
@@ -911,9 +987,8 @@ class BashStage(BaseAsyncStage):
             )
         if rs.returncode > 0:
             e: str = rs.stderr.removesuffix("\n")
-            raise StageError(
-                f"Subprocess: {e}\n---( statement )---\n```bash\n{bash}\n```"
-            )
+            e_bash: str = bash.replace("\n", "\n\t")
+            raise StageError(f"Subprocess: {e}\n\t```bash\n\t{e_bash}\n\t```")
         return result.catch(
             status=SUCCESS,
             context={
@@ -964,9 +1039,8 @@ class BashStage(BaseAsyncStage):
 
         if rs.returncode > 0:
             e: str = rs.stderr.removesuffix("\n")
-            raise StageError(
-                f"Subprocess: {e}\n---( statement )---\n```bash\n{bash}\n```"
-            )
+            e_bash: str = bash.replace("\n", "\n\t")
+            raise StageError(f"Subprocess: {e}\n\t```bash\n\t{e_bash}\n\t```")
         return result.catch(
             status=SUCCESS,
             context={
@@ -977,7 +1051,7 @@ class BashStage(BaseAsyncStage):
         )
 
 
-class PyStage(BaseAsyncStage):
+class PyStage(BaseRetryStage):
     """Python stage that running the Python statement with the current globals
     and passing an input additional variables via `exec` built-in function.
 
@@ -1164,7 +1238,7 @@ class PyStage(BaseAsyncStage):
         )
 
 
-class CallStage(BaseAsyncStage):
+class CallStage(BaseRetryStage):
     """Call stage executor that call the Python function from registry with tag
     decorator function in `reusables` module and run it with input arguments.
 
@@ -1433,7 +1507,7 @@ class CallStage(BaseAsyncStage):
             return args
 
 
-class BaseNestedStage(BaseStage, ABC):
+class BaseNestedStage(BaseRetryStage, ABC):
     """Base Nested Stage model. This model is use for checking the child stage
     is the nested stage or not.
     """
@@ -1467,6 +1541,17 @@ class BaseNestedStage(BaseStage, ABC):
         else:
             context["errors"] = error.to_dict(with_refs=True)
 
+    async def axecute(
+        self,
+        params: DictData,
+        *,
+        result: Optional[Result] = None,
+        event: Optional[Event] = None,
+    ) -> Result:
+        raise NotImplementedError(
+            "The nested-stage does not implement the `axecute` method yet."
+        )
+
 
 class TriggerStage(BaseNestedStage):
     """Trigger workflow executor stage that run an input trigger Workflow

{ddeutil_workflow-0.0.66.dist-info → ddeutil_workflow-0.0.67.dist-info}/METADATA

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: ddeutil-workflow
-Version: 0.0.66
-Summary: Lightweight workflow orchestration
+Version: 0.0.67
+Summary: Lightweight workflow orchestration with YAML template
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT
 Project-URL: Homepage, https://github.com/ddeutils/ddeutil-workflow/
@@ -35,18 +35,9 @@ Requires-Dist: httpx; extra == "all"
 Requires-Dist: ujson; extra == "all"
 Requires-Dist: aiofiles; extra == "all"
 Requires-Dist: aiohttp; extra == "all"
-Provides-Extra: api
-Requires-Dist: fastapi<1.0.0,>=0.115.0; extra == "api"
-Requires-Dist: uvicorn; extra == "api"
-Requires-Dist: httpx; extra == "api"
-Requires-Dist: ujson; extra == "api"
-Provides-Extra: async
-Requires-Dist: aiofiles; extra == "async"
-Requires-Dist: aiohttp; extra == "async"
+Requires-Dist: requests==2.32.3; extra == "all"
 Provides-Extra: docker
 Requires-Dist: docker==7.1.0; extra == "docker"
-Provides-Extra: self-hosted
-Requires-Dist: requests==2.32.3; extra == "self-hosted"
 Dynamic: license-file
 
 # Workflow Orchestration
@@ -142,10 +133,10 @@ the base deps.
 If you want to install this package with application add-ons, you should add
 `app` in installation;
 
-| Use-case       | Install Optional         |       Support       |
-|----------------|--------------------------|:-------------------:|
-| Python         | `ddeutil-workflow`       | :heavy_check_mark:  |
-| FastAPI Server | `ddeutil-workflow[api]`  | :heavy_check_mark:  |
+| Use-case       | Install Optional        | Support |
+|----------------|-------------------------|:-------:|
+| Python         | `ddeutil-workflow`      |    ✅    |
+| FastAPI Server | `ddeutil-workflow[all]` |    ✅    |
 
 ## 🎯 Usage
 
@@ -300,40 +291,27 @@ it will use default value and do not raise any error to you.
 ## :rocket: Deployment
 
 This package able to run as an application service for receive manual trigger
-from any node via RestAPI or use to be Scheduler background application
-like crontab job but via Python API or FastAPI app.
+from any node via RestAPI with the FastAPI package.
 
 ### API Server
 
 This server use FastAPI package to be the base application.
 
 ```shell
-(.venv) $ uvicorn ddeutil.workflow.api:app \
-  --host 127.0.0.1 \
-  --port 80 \
-  --no-access-log
+(.venv) $ workflow-cli api --host 127.0.0.1 --port 80
 ```
 
 > [!NOTE]
 > If this package already deploy, it is able to use multiprocess;
-> `uvicorn ddeutil.workflow.api:app --host 127.0.0.1 --port 80 --workers 4`
-
-### Local Schedule
-
-> [!WARNING]
-> This CLI does not implement yet.
-
-```shell
-(.venv) $ ddeutil-workflow schedule
-```
+> `$ workflow-cli api --host 127.0.0.1 --port 80 --workers 4`
 
 ### Docker Container
 
 Build a Docker container from this package.
 
 ```shell
-$ docker build -t ddeutil-workflow:latest -f .container/Dockerfile .
-$ docker run -i ddeutil-workflow:latest ddeutil-workflow
+$ docker pull ghcr.io/ddeutils/ddeutil-workflow:latest
+$ docker run --rm ghcr.io/ddeutils/ddeutil-workflow:latest ddeutil-worker
 ```
 
 ## :speech_balloon: Contribute

{ddeutil_workflow-0.0.66.dist-info → ddeutil_workflow-0.0.67.dist-info}/RECORD

@@ -1,18 +1,18 @@
-ddeutil/workflow/__about__.py,sha256=nkpvuhIACvuey4B8paloyuDA6U0uRDZqZ83SUGdlIt4,28
+ddeutil/workflow/__about__.py,sha256=JZ9Er-4hkPGd0SSb_wI8VFJvPCjm8q09g7oG_MshBMo,28
 ddeutil/workflow/__cron.py,sha256=BOKQcreiex0SAigrK1gnLxpvOeF3aca_rQwyz9Kfve4,28751
 ddeutil/workflow/__init__.py,sha256=JfFZlPRDgR2J0rb0SRejt1OSrOrD3GGv9Um14z8MMfs,901
 ddeutil/workflow/__main__.py,sha256=Qd-f8z2Q2vpiEP2x6PBFsJrpACWDVxFKQk820MhFmHo,59
 ddeutil/workflow/__types.py,sha256=uNfoRbVmNK5O37UUMVnqcmoghD9oMS1q9fXC0APnjSI,4584
-ddeutil/workflow/cli.py,sha256=__ydzTs_e9C1tHqhnyp__0iKelZJRUwZn8VsyVYw5yo,1458
+ddeutil/workflow/cli.py,sha256=YtfNfozYRvQyohhYVcZ2_8o_IBXOpmok531eYw0DScM,1555
 ddeutil/workflow/conf.py,sha256=w1WDWZDCvRVDSz2HnJxeqySzpYWSubJZjTVjXO9imK0,14669
-ddeutil/workflow/errors.py,sha256=vh_AbLegPbb61TvfwHmjSz0t2SH09RQuGHCKmoyNjn0,2934
+ddeutil/workflow/errors.py,sha256=4DaKnyUm8RrUyQA5qakgW0ycSQLO7j-owyoh79LWQ5c,2893
 ddeutil/workflow/event.py,sha256=S2eJAZZx_V5TuQ0l417hFVCtjWXnfNPZBgSCICzxQ48,11041
 ddeutil/workflow/job.py,sha256=qcbKSOa39256nfJHL0vKJsHrelcRujX5KET2IEGS8dw,38995
 ddeutil/workflow/logs.py,sha256=4rL8TsRJsYVqyPfLjFW5bSoWtRwUgwmaRONu7nnVxQ8,31374
 ddeutil/workflow/params.py,sha256=Pco3DyjptC5Jkx53dhLL9xlIQdJvNAZs4FLzMUfXpbQ,12402
-ddeutil/workflow/result.py,sha256=CzSjK9EQtSS0a1z8b6oBW4gg8t0uVbZ_Ppx4Ckvcork,7786
+ddeutil/workflow/result.py,sha256=GU84psZFiJ4LRf_HXgz-R98YN4lOUkER0VR7x9DDdOU,7922
 ddeutil/workflow/reusables.py,sha256=jPrOCbxagqRvRFGXJzIyDa1wKV5AZ4crZyJ10cldQP0,21620
-ddeutil/workflow/stages.py,sha256=s6Jc4A6ApXT40JMrnVf7n7yHG_mmjfHJIOU_niMW5hI,102555
+ddeutil/workflow/stages.py,sha256=xsJactN-Qk5Yg7ooXfoq-JVdlduIAdXXJUzCKFJuWGA,105093
 ddeutil/workflow/utils.py,sha256=slhBbsBNl0yaSk9EOiCK6UL-o7smgHVsLT7svRqAWXU,10436
 ddeutil/workflow/workflow.py,sha256=AcSGqsH1N4LqWhYIcCPy9CoV_AGlXUrBgjpl-gniv6g,28267
 ddeutil/workflow/api/__init__.py,sha256=0UIilYwW29RL6HrCRHACSWvnATJVLSJzXiCMny0bHQk,2627
@@ -21,9 +21,9 @@ ddeutil/workflow/api/routes/__init__.py,sha256=jC1pM7q4_eo45IyO3hQbbe6RnL9B8ibRq
 ddeutil/workflow/api/routes/job.py,sha256=32TkNm7QY9gt6fxIqEPjDqPgc8XqDiMPjUb7disSrCw,2143
 ddeutil/workflow/api/routes/logs.py,sha256=QJH8IF102897WLfCJ29-1g15wl29M9Yq6omroZfbahs,5305
 ddeutil/workflow/api/routes/workflows.py,sha256=Gmg3e-K5rfi95pbRtWI_aIr5C089sIde_vefZVvh3U0,4420
-ddeutil_workflow-0.0.66.dist-info/licenses/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
-ddeutil_workflow-0.0.66.dist-info/METADATA,sha256=LhCFhlwk3N5wbnylmi7zi3sAJqwxkRVlrRThlNB4Cj0,16676
-ddeutil_workflow-0.0.66.dist-info/WHEEL,sha256=Nw36Djuh_5VDukK0H78QzOX-_FQEo6V37m3nkm96gtU,91
-ddeutil_workflow-0.0.66.dist-info/entry_points.txt,sha256=qDTpPSauL0ciO6T4iSVt8bJeYrVEkkoEEw_RlGx6Kgk,63
-ddeutil_workflow-0.0.66.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
-ddeutil_workflow-0.0.66.dist-info/RECORD,,
+ddeutil_workflow-0.0.67.dist-info/licenses/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
+ddeutil_workflow-0.0.67.dist-info/METADATA,sha256=w9iP1ofTfKIdirH9WSZf5rMOA4MrqMKM5jJk1hFO3oU,16072
+ddeutil_workflow-0.0.67.dist-info/WHEEL,sha256=Nw36Djuh_5VDukK0H78QzOX-_FQEo6V37m3nkm96gtU,91
+ddeutil_workflow-0.0.67.dist-info/entry_points.txt,sha256=qDTpPSauL0ciO6T4iSVt8bJeYrVEkkoEEw_RlGx6Kgk,63
+ddeutil_workflow-0.0.67.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
+ddeutil_workflow-0.0.67.dist-info/RECORD,,