ddeutil-workflow 0.0.14__py3-none-any.whl → 0.0.15__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1 +1 @@
-__version__: str = "0.0.14"
+__version__: str = "0.0.15"

ddeutil/workflow/__types.py

@@ -6,14 +6,19 @@
 from __future__ import annotations
 
 import re
+from collections.abc import Iterator
+from dataclasses import dataclass
 from re import (
     IGNORECASE,
     MULTILINE,
     UNICODE,
     VERBOSE,
+    Match,
     Pattern,
 )
-from typing import Any, Union
+from typing import Any, Optional, Union
+
+from typing_extensions import Self
 
 TupleStr = tuple[str, ...]
 DictData = dict[str, Any]
@@ -23,6 +28,27 @@ MatrixInclude = list[dict[str, Union[str, int]]]
 MatrixExclude = list[dict[str, Union[str, int]]]
 
 
+@dataclass(frozen=True)
+class CallerRe:
+    """Caller dataclass that catching result from the matching regex with the
+    Re.RE_CALLER value.
+    """
+
+    full: str
+    caller: str
+    caller_prefix: Optional[str]
+    caller_last: str
+    post_filters: str
+
+    @classmethod
+    def from_regex(cls, match: Match[str]) -> Self:
+        """Class construct from matching result.
+
+        :rtype: Self
+        """
+        return cls(full=match.group(0), **match.groupdict())
+
+
 class Re:
     """Regular expression config for this package."""
 
@@ -31,10 +57,10 @@ class Re:
     #       - Version 1:
     #         \${{\s*(?P<caller>[a-zA-Z0-9_.\s'\"\[\]\(\)\-\{}]+?)\s*(?P<post_filters>(?:\|\s*(?:[a-zA-Z0-9_]{3,}[a-zA-Z0-9_.,-\\%\s'\"[\]()\{}]+)\s*)*)}}
     #       - Version 2 (2024-09-30):
-    #         \${{\s*(?P<caller>(?P<caller_prefix>[a-zA-Z_-]+\.)*(?P<caller_last>[a-zA-Z0-9_\-.'\"(\)[\]{}]+))\s*(?P<post_filters>(?:\|\s*(?:[a-zA-Z0-9_]{3,}[a-zA-Z0-9_.,-\\%\s'\"[\]()\{}]+)\s*)*)}}
+    #         \${{\s*(?P<caller>(?P<caller_prefix>(?:[a-zA-Z_-]+\.)*)(?P<caller_last>[a-zA-Z0-9_\-.'\"(\)[\]{}]+))\s*(?P<post_filters>(?:\|\s*(?:[a-zA-Z0-9_]{3,}[a-zA-Z0-9_.,-\\%\s'\"[\]()\{}]+)\s*)*)}}
     #
     #   Examples:
-    #       - ${{ params.asat_dt }}
+    #       - ${{ params.data_dt }}
     #       - ${{ params.source.table }}
     #
     __re_caller: str = r"""
@@ -42,16 +68,17 @@ class Re:
         {{
             \s*
             (?P<caller>
-                (?P<caller_prefix>[a-zA-Z_-]+\.)*
+                (?P<caller_prefix>(?:[a-zA-Z_-]+\.)*)
                 (?P<caller_last>[a-zA-Z0-9_\-.'\"(\)[\]{}]+)
             )
             \s*
             (?P<post_filters>
                 (?:
-                    \|
-                    \s*
-                    (?:[a-zA-Z0-9_]{3,}[a-zA-Z0-9_.,-\\%\s'\"[\]()\{}]*)
-                    \s*
+                    \|\s*
+                    (?:
+                        [a-zA-Z0-9_]{3,}
+                        [a-zA-Z0-9_.,-\\%\s'\"[\]()\{}]*
+                    )\s*
                 )*
             )
         }}
@@ -62,7 +89,8 @@ class Re:
 
     # NOTE:
     #   Regular expression:
-    #   ^(?P<path>[^/@]+)/(?P<func>[^@]+)@(?P<tag>.+)$
+    #       - Version 1:
+    #         ^(?P<path>[^/@]+)/(?P<func>[^@]+)@(?P<tag>.+)$
     #
     #   Examples:
     #       - tasks/function@dummy
@@ -78,3 +106,8 @@ class Re:
     RE_TASK_FMT: Pattern = re.compile(
         __re_task_fmt, MULTILINE | IGNORECASE | UNICODE | VERBOSE
     )
+
+    @classmethod
+    def finditer_caller(cls, value) -> Iterator[CallerRe]:
+        for found in cls.RE_CALLER.finditer(value):
+            yield CallerRe.from_regex(found)

ddeutil/workflow/conf.py

@@ -16,6 +16,10 @@ env = os.getenv
 
 
 class Config:
+    """Config object for keeping application configuration on current session
+    without changing when if the application still running.
+    """
+
     # NOTE: Core
     tz: ZoneInfo = ZoneInfo(env("WORKFLOW_CORE_TIMEZONE", "UTC"))
 

ddeutil/workflow/job.py

@@ -320,13 +320,15 @@ class Job(BaseModel):
         and want to set on the `to` like;
 
             ... (i)   output: {'strategy01': bar, 'strategy02': bar}
-            ... (ii)  to: {}
+            ... (ii)  to: {'jobs'}
 
         The result of the `to` variable will be;
 
             ... (iii) to: {
-                            'strategies': {
-                                'strategy01': bar, 'strategy02': bar
+                            'jobs': {
+                                'strategies': {
+                                    'strategy01': bar, 'strategy02': bar
+                                }
                             }
                         }
 
@@ -339,7 +341,7 @@ class Job(BaseModel):
                 "This job do not set the ID before setting output."
             )
 
-        to[self.id] = (
+        to["jobs"][self.id] = (
             {"strategies": output}
             if self.strategy.is_set()
             # NOTE:

ddeutil/workflow/scheduler.py

@@ -513,7 +513,6 @@ class Workflow(BaseModel):
                 f"workflow."
             )
 
-        context: DictData = {}
         logger.info(f"({self.run_id}) [WORKFLOW]: Start execute: {job_id!r}")
 
         # IMPORTANT:
@@ -523,7 +522,7 @@ class Workflow(BaseModel):
             job: Job = self.jobs[job_id].get_running_id(self.run_id)
             job.set_outputs(
                 job.execute(params=params).context,
-                to=context,
+                to=params,
             )
         except JobException as err:
             logger.error(
@@ -536,7 +535,7 @@ class Workflow(BaseModel):
             else:
                 raise NotImplementedError() from None
 
-        return Result(status=0, context=context)
+        return Result(status=0, context=params)
 
     def execute(
         self,
@@ -587,8 +586,14 @@ class Workflow(BaseModel):
         for job_id in self.jobs:
             jq.put(job_id)
 
-        # NOTE: Create result context that will pass this context to any
-        #   execution dependency.
+        # NOTE: Create data context that will pass to any job executions
+        #   on this workflow.
+        #
+        #   {
+        #       'params': <input-params>,
+        #       'jobs': {},
+        #   }
+        #
         context: DictData = self.parameterize(params)
         status: int = 0
         try:
@@ -657,15 +662,23 @@ class Workflow(BaseModel):
                 job: Job = self.jobs[job_id]
 
                 if any(need not in context["jobs"] for need in job.needs):
+                    job_queue.task_done()
                     job_queue.put(job_id)
                     time.sleep(0.25)
                     continue
 
+                # NOTE: Start workflow job execution with deep copy context data
+                #   before release.
+                #
+                #   {
+                #       'params': <input-params>,
+                #       'jobs': {},
+                #   }
                 futures.append(
                     executor.submit(
                         self.execute_job,
                         job_id,
-                        params=copy.deepcopy(context),
+                        params=context,
                     ),
                 )
 
@@ -677,14 +690,13 @@ class Workflow(BaseModel):
 
             for future in as_completed(futures, timeout=1800):
                 if err := future.exception():
-                    logger.error(f"{err}")
+                    logger.error(f"({self.run_id}) [CORE]: {err}")
                     raise WorkflowException(f"{err}")
                 try:
-                    # NOTE: Update job result to workflow result.
-                    context["jobs"].update(future.result(timeout=60).context)
+                    future.result(timeout=60)
                 except TimeoutError as err:
                     raise WorkflowException(
-                        "Get result from future was timeout"
+                        "Timeout when getting result from future"
                     ) from err
 
         if not_time_out_flag:
@@ -731,18 +743,21 @@ class Workflow(BaseModel):
             job_id: str = job_queue.get()
             job: Job = self.jobs[job_id]
 
-            # NOTE:
+            # NOTE: Waiting dependency job run successful before release.
             if any(need not in context["jobs"] for need in job.needs):
+                job_queue.task_done()
                 job_queue.put(job_id)
-                time.sleep(0.25)
+                time.sleep(0.05)
                 continue
 
-            # NOTE: Start workflow job execution.
-            job_rs = self.execute_job(
-                job_id=job_id,
-                params=copy.deepcopy(context),
-            )
-            context["jobs"].update(job_rs.context)
+            # NOTE: Start workflow job execution with deep copy context data
+            #   before release.
+            #
+            #   {
+            #       'params': <input-params>,
+            #       'jobs': {},
+            #   }
+            self.execute_job(job_id=job_id, params=context)
 
             # NOTE: Mark this job queue done.
             job_queue.task_done()

ddeutil/workflow/stage.py

@@ -24,6 +24,7 @@ import contextlib
 import inspect
 import subprocess
 import sys
+import time
 import uuid
 from abc import ABC, abstractmethod
 from collections.abc import Iterator
@@ -60,6 +61,8 @@ from .utils import (
 )
 
 P = ParamSpec("P")
+ReturnResult = Callable[P, Result]
+DecoratorResult = Callable[[ReturnResult], ReturnResult]
 logger = get_logger("ddeutil.workflow")
 
 
@@ -77,7 +80,7 @@ __all__: TupleStr = (
 )
 
 
-def handler_result(message: str | None = None) -> Callable[P, Result]:
+def handler_result(message: str | None = None) -> DecoratorResult:
     """Decorator function for handler result from the stage execution. This
     function should to use with execution method only.
 
@@ -103,7 +106,7 @@ def handler_result(message: str | None = None) -> Callable[P, Result]:
     #
     message: str = message or ""
 
-    def decorator(func: Callable[P, Result]) -> Callable[P, Result]:
+    def decorator(func: ReturnResult) -> ReturnResult:
 
         @wraps(func)
         def wrapped(self: Stage, *args, **kwargs):
@@ -293,6 +296,10 @@ class EmptyStage(BaseStage):
         default=None,
         description="A string statement that want to logging",
     )
+    sleep: float = Field(
+        default=0,
+        description="A second value to sleep before finish execution",
+    )
 
     def execute(self, params: DictData) -> Result:
         """Execution method for the Empty stage that do only logging out to
@@ -310,6 +317,8 @@ class EmptyStage(BaseStage):
             f"({self.run_id}) [STAGE]: Empty-Execute: {self.name!r}: "
             f"( {param2template(self.echo, params=params) or '...'} )"
         )
+        if self.sleep > 0:
+            time.sleep(self.sleep)
         return Result(status=0, context={})
 
 

ddeutil/workflow/utils.py

@@ -206,6 +206,12 @@ class SimLoad:
         """Find all data that match with object type in config path. This class
         method can use include and exclude list of identity name for filter and
         adds-on.
+
+        :param obj:
+        :param params:
+        :param include:
+        :param exclude:
+        :rtype: Iterator[tuple[str, DictData]]
         """
         exclude: list[str] = exclude or []
         for file in PathSearch(params.engine.paths.conf).files:
@@ -317,10 +323,14 @@ class TagFunc(Protocol):
     name: str
     tag: str
 
-    def __call__(self, *args, **kwargs): ...
+    def __call__(self, *args, **kwargs): ...  # pragma: no cove
+
+
+ReturnTagFunc = Callable[P, TagFunc]
+DecoratorTagFunc = Callable[[Callable[[...], Any]], ReturnTagFunc]
 
 
-def tag(name: str, alias: str | None = None) -> Callable[P, TagFunc]:
+def tag(name: str, alias: str | None = None) -> DecoratorTagFunc:
     """Tag decorator function that set function attributes, ``tag`` and ``name``
     for making registries variable.
 
@@ -330,7 +340,7 @@ def tag(name: str, alias: str | None = None) -> Callable[P, TagFunc]:
     :rtype: Callable[P, TagFunc]
     """
 
-    def func_internal(func: Callable[[...], Any]) -> TagFunc:
+    def func_internal(func: Callable[[...], Any]) -> ReturnTagFunc:
         func.tag = name
         func.name = alias or func.__name__.replace("_", "-")
 
@@ -398,6 +408,10 @@ class BaseParam(BaseModel, ABC):
 
     @field_serializer("type")
     def __serializer_type(self, value: str) -> str:
+        """Serialize the value of the type field.
+
+        :rtype: str
+        """
         return value
 
 
@@ -793,8 +807,8 @@ def not_in_template(value: Any, *, not_in: str = "matrix.") -> bool:
     elif not isinstance(value, str):
         return False
     return any(
-        (not found.group("caller").strip().startswith(not_in))
-        for found in Re.RE_CALLER.finditer(value.strip())
+        (not found.caller.strip().startswith(not_in))
+        for found in Re.finditer_caller(value.strip())
     )
 
 
@@ -835,18 +849,16 @@ def str2template(
 
     # NOTE: remove space before and after this string value.
     value: str = value.strip()
-    for found in Re.RE_CALLER.finditer(value):
+    for found in Re.finditer_caller(value):
         # NOTE:
         #   Get caller and filter values that setting inside;
         #
         #   ... ``${{ <caller-value> [ | <filter-value>] ... }}``
         #
-        caller: str = found.group("caller")
+        caller: str = found.caller
         pfilter: list[str] = [
             i.strip()
-            for i in (
-                found.group("post_filters").strip().removeprefix("|").split("|")
-            )
+            for i in (found.post_filters.strip().removeprefix("|").split("|"))
             if i != ""
         ]
         if not hasdot(caller, params):
@@ -859,7 +871,7 @@ def str2template(
         #   If type of getter caller is not string type and it does not use to
         #   concat other string value, it will return origin value from the
         #   ``getdot`` function.
-        if value.replace(found.group(0), "", 1) == "":
+        if value.replace(found.full, "", 1) == "":
             return map_post_filter(getter, pfilter, filters=filters)
 
         # NOTE: map post-filter function.
@@ -867,7 +879,7 @@ def str2template(
         if not isinstance(getter, str):
             getter: str = str(getter)
 
-        value: str = value.replace(found.group(0), getter, 1)
+        value: str = value.replace(found.full, getter, 1)
 
     return search_env_replace(value)
 

{ddeutil_workflow-0.0.14.dist-info → ddeutil_workflow-0.0.15.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ddeutil-workflow
-Version: 0.0.14
+Version: 0.0.15
 Summary: Lightweight workflow orchestration with less dependencies
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT

ddeutil_workflow-0.0.15.dist-info/RECORD

@@ -0,0 +1,22 @@
+ddeutil/workflow/__about__.py,sha256=w_vBOopUg1crMbDyfdE0LgsxsncnhGYp0D39LSSnSVI,28
+ddeutil/workflow/__init__.py,sha256=-DIy8SGFsD7_wqp-V-K8v8jTxacmqrcyj_SFx1WS6qg,687
+ddeutil/workflow/__types.py,sha256=WWugALcayRiP0IQO-eBWK767_XxK7KGlY7SuVgyaJnk,3196
+ddeutil/workflow/api.py,sha256=cwju_qhY6m0kLtaoa77QLglC9tl7RjjZ4UnJYV3SlQQ,4810
+ddeutil/workflow/cli.py,sha256=Ikcq526WeIl-737-v55T0PwAZ2pNiZFxlN0Y-DjhDbQ,3374
+ddeutil/workflow/conf.py,sha256=D0g7rHXilpGwOD36QwVd9I5kEwqsAUA0Z3tAINS2Pws,1287
+ddeutil/workflow/cron.py,sha256=naWefHc3EnVo41Yf1zQeXOzF27YlTlnfj0XnQ6_HO-U,25514
+ddeutil/workflow/exceptions.py,sha256=Uf1-Tn8rAzj0aiVHSqo4fBqO80W0za7UFZgKv24E-tg,706
+ddeutil/workflow/job.py,sha256=9H_2C0ikD5y6jLVdIBj8de4CdSpS632XOfqYVhM4bHI,21582
+ddeutil/workflow/log.py,sha256=Ev-Szi0KC_MmbFY4g4BWv6tUSmcLKWKZ03ZInmYPmgU,6490
+ddeutil/workflow/on.py,sha256=vsZG19mNoztDSB_ObD_4ZWPKgHYpBDJMWw97ZiTavNE,7237
+ddeutil/workflow/repeat.py,sha256=e3dekPTlMlxCCizfBYsZ8dD8Juy4rtfqDZJU3Iky2oA,5011
+ddeutil/workflow/route.py,sha256=ABEk-WlVo9XGFc7zCPbckX33URCNH7woQFU1keX_8PQ,6970
+ddeutil/workflow/scheduler.py,sha256=12Dd5CVphOVKjUwoiB8dCHt4WpYRPG3dSOt-pR6NNxc,46167
+ddeutil/workflow/stage.py,sha256=Avz1Mbb8WAP6kFn0bnN0p14-EnQ_AzdKr435JRxjkao,23844
+ddeutil/workflow/utils.py,sha256=XUD5hoygAyxi4xo1spTacoDNGKN2TlRob_o8qfCj4Pc,30993
+ddeutil_workflow-0.0.15.dist-info/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
+ddeutil_workflow-0.0.15.dist-info/METADATA,sha256=6JvY9y-cT3WnirRva45NS582Iz7ZuXJZpsiCtN57OoA,11653
+ddeutil_workflow-0.0.15.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
+ddeutil_workflow-0.0.15.dist-info/entry_points.txt,sha256=0BVOgO3LdUdXVZ-CiHHDKxzEk2c8J30jEwHeKn2YCWI,62
+ddeutil_workflow-0.0.15.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
+ddeutil_workflow-0.0.15.dist-info/RECORD,,

ddeutil_workflow-0.0.14.dist-info/RECORD

@@ -1,22 +0,0 @@
-ddeutil/workflow/__about__.py,sha256=Xas_M3BaGwGfZOJTvMWUlmlW09aGC-Apst-NtkPddY4,28
-ddeutil/workflow/__init__.py,sha256=-DIy8SGFsD7_wqp-V-K8v8jTxacmqrcyj_SFx1WS6qg,687
-ddeutil/workflow/__types.py,sha256=aBbytylSPIe_cip2KIyqLN2eUloMOJdkayqKWCBrwhk,2353
-ddeutil/workflow/api.py,sha256=cwju_qhY6m0kLtaoa77QLglC9tl7RjjZ4UnJYV3SlQQ,4810
-ddeutil/workflow/cli.py,sha256=Ikcq526WeIl-737-v55T0PwAZ2pNiZFxlN0Y-DjhDbQ,3374
-ddeutil/workflow/conf.py,sha256=j19G7rDxQRGgSRQW3pxIYtK5lB3fZv0eG_CAoqoIhPw,1140
-ddeutil/workflow/cron.py,sha256=naWefHc3EnVo41Yf1zQeXOzF27YlTlnfj0XnQ6_HO-U,25514
-ddeutil/workflow/exceptions.py,sha256=Uf1-Tn8rAzj0aiVHSqo4fBqO80W0za7UFZgKv24E-tg,706
-ddeutil/workflow/job.py,sha256=zEefiEAxyC34NvbNVpKexTVU1E_031446308zGMdcmE,21488
-ddeutil/workflow/log.py,sha256=Ev-Szi0KC_MmbFY4g4BWv6tUSmcLKWKZ03ZInmYPmgU,6490
-ddeutil/workflow/on.py,sha256=vsZG19mNoztDSB_ObD_4ZWPKgHYpBDJMWw97ZiTavNE,7237
-ddeutil/workflow/repeat.py,sha256=e3dekPTlMlxCCizfBYsZ8dD8Juy4rtfqDZJU3Iky2oA,5011
-ddeutil/workflow/route.py,sha256=ABEk-WlVo9XGFc7zCPbckX33URCNH7woQFU1keX_8PQ,6970
-ddeutil/workflow/scheduler.py,sha256=CcUFichnvPbQzSEk_ikNgFwZimTObGHfXxHChuysAo4,45706
-ddeutil/workflow/stage.py,sha256=Tt5QQrO_dN8MO9gPtiziOqVrd64UTJZwbgifWeXBCIA,23574
-ddeutil/workflow/utils.py,sha256=epJMTsA4BPQa0gECgcWJ38IENlejpnF3OTBNc0eaqYE,30715
-ddeutil_workflow-0.0.14.dist-info/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
-ddeutil_workflow-0.0.14.dist-info/METADATA,sha256=sf9kiPoGazaRxqymTjaxZyH47yNyGj0RElc-NLmUa4w,11653
-ddeutil_workflow-0.0.14.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
-ddeutil_workflow-0.0.14.dist-info/entry_points.txt,sha256=0BVOgO3LdUdXVZ-CiHHDKxzEk2c8J30jEwHeKn2YCWI,62
-ddeutil_workflow-0.0.14.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
-ddeutil_workflow-0.0.14.dist-info/RECORD,,