ddeutil-workflow 0.0.13__py3-none-any.whl → 0.0.15__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1 +1 @@
-__version__: str = "0.0.13"
+__version__: str = "0.0.15"

ddeutil/workflow/__init__.py

@@ -12,7 +12,10 @@ from .exceptions import (
 )
 from .job import Job, Strategy
 from .on import On, interval2crontab
-from .scheduler import Workflow
+from .scheduler import (
+    Schedule,
+    Workflow,
+)
 from .stage import Stage, handler_result
 from .utils import (
     Param,

ddeutil/workflow/__types.py

@@ -6,14 +6,19 @@
 from __future__ import annotations
 
 import re
+from collections.abc import Iterator
+from dataclasses import dataclass
 from re import (
     IGNORECASE,
     MULTILINE,
     UNICODE,
     VERBOSE,
+    Match,
     Pattern,
 )
-from typing import Any, Union
+from typing import Any, Optional, Union
+
+from typing_extensions import Self
 
 TupleStr = tuple[str, ...]
 DictData = dict[str, Any]
@@ -23,23 +28,57 @@ MatrixInclude = list[dict[str, Union[str, int]]]
 MatrixExclude = list[dict[str, Union[str, int]]]
 
 
+@dataclass(frozen=True)
+class CallerRe:
+    """Caller dataclass that catching result from the matching regex with the
+    Re.RE_CALLER value.
+    """
+
+    full: str
+    caller: str
+    caller_prefix: Optional[str]
+    caller_last: str
+    post_filters: str
+
+    @classmethod
+    def from_regex(cls, match: Match[str]) -> Self:
+        """Class construct from matching result.
+
+        :rtype: Self
+        """
+        return cls(full=match.group(0), **match.groupdict())
+
+
 class Re:
-    """Regular expression config."""
+    """Regular expression config for this package."""
 
-    # NOTE: Search caller
-    # \${{\s*(?P<caller>[a-zA-Z0-9_.\s'\"\[\]\(\)\-\{}]+?)\s*(?P<post_filters>(?:\|\s*(?:[a-zA-Z0-9_]{3,}[a-zA-Z0-9_.,-\\%\s'\"[\]()\{}]+)\s*)*)}}
+    # NOTE:
+    #   Regular expression:
+    #       - Version 1:
+    #         \${{\s*(?P<caller>[a-zA-Z0-9_.\s'\"\[\]\(\)\-\{}]+?)\s*(?P<post_filters>(?:\|\s*(?:[a-zA-Z0-9_]{3,}[a-zA-Z0-9_.,-\\%\s'\"[\]()\{}]+)\s*)*)}}
+    #       - Version 2 (2024-09-30):
+    #         \${{\s*(?P<caller>(?P<caller_prefix>(?:[a-zA-Z_-]+\.)*)(?P<caller_last>[a-zA-Z0-9_\-.'\"(\)[\]{}]+))\s*(?P<post_filters>(?:\|\s*(?:[a-zA-Z0-9_]{3,}[a-zA-Z0-9_.,-\\%\s'\"[\]()\{}]+)\s*)*)}}
+    #
+    #   Examples:
+    #       - ${{ params.data_dt }}
+    #       - ${{ params.source.table }}
+    #
     __re_caller: str = r"""
         \$
         {{
             \s*
             (?P<caller>
-                [a-zA-Z0-9_.\s'\"\[\]\(\)\-\{}]+?
-            )\s*
+                (?P<caller_prefix>(?:[a-zA-Z_-]+\.)*)
+                (?P<caller_last>[a-zA-Z0-9_\-.'\"(\)[\]{}]+)
+            )
+            \s*
             (?P<post_filters>
                 (?:
                     \|\s*
-                    (?:[a-zA-Z0-9_]{3,}[a-zA-Z0-9_.,-\\%\s'\"[\]()\{}]*)
-                    \s*
+                    (?:
+                        [a-zA-Z0-9_]{3,}
+                        [a-zA-Z0-9_.,-\\%\s'\"[\]()\{}]*
+                    )\s*
                 )*
             )
         }}
@@ -48,8 +87,13 @@ class Re:
         __re_caller, MULTILINE | IGNORECASE | UNICODE | VERBOSE
     )
 
-    # NOTE: Search task
-    # ^(?P<path>[^/@]+)/(?P<func>[^@]+)@(?P<tag>.+)$
+    # NOTE:
+    #   Regular expression:
+    #       - Version 1:
+    #         ^(?P<path>[^/@]+)/(?P<func>[^@]+)@(?P<tag>.+)$
+    #
+    #   Examples:
+    #       - tasks/function@dummy
     __re_task_fmt: str = r"""
         ^
             (?P<path>[^/@]+)
@@ -62,3 +106,8 @@ class Re:
     RE_TASK_FMT: Pattern = re.compile(
         __re_task_fmt, MULTILINE | IGNORECASE | UNICODE | VERBOSE
     )
+
+    @classmethod
+    def finditer_caller(cls, value) -> Iterator[CallerRe]:
+        for found in cls.RE_CALLER.finditer(value):
+            yield CallerRe.from_regex(found)

ddeutil/workflow/api.py

@@ -25,7 +25,7 @@ from pydantic import BaseModel
 from .__about__ import __version__
 from .log import get_logger
 from .repeat import repeat_at, repeat_every
-from .scheduler import WorkflowTask
+from .scheduler import WorkflowTaskData
 
 load_dotenv()
 logger = get_logger("ddeutil.workflow")
@@ -36,7 +36,7 @@ class State(TypedDict):
     upper_result: dict[str, str]
     scheduler: list[str]
     workflow_threads: dict[str, Thread]
-    workflow_tasks: list[WorkflowTask]
+    workflow_tasks: list[WorkflowTaskData]
     workflow_queue: dict[str, list[datetime]]
     workflow_running: dict[str, list[datetime]]
 

ddeutil/workflow/conf.py

@@ -0,0 +1,45 @@
+# ------------------------------------------------------------------------------
+# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
+# Licensed under the MIT License. See LICENSE in the project root for
+# license information.
+# ------------------------------------------------------------------------------
+from __future__ import annotations
+
+import os
+from zoneinfo import ZoneInfo
+
+from ddeutil.core import str2bool
+from dotenv import load_dotenv
+
+load_dotenv()
+env = os.getenv
+
+
+class Config:
+    """Config object for keeping application configuration on current session
+    without changing when if the application still running.
+    """
+
+    # NOTE: Core
+    tz: ZoneInfo = ZoneInfo(env("WORKFLOW_CORE_TIMEZONE", "UTC"))
+
+    # NOTE: Stage
+    stage_raise_error: bool = str2bool(
+        env("WORKFLOW_CORE_STAGE_RAISE_ERROR", "true")
+    )
+    stage_default_id: bool = str2bool(
+        env("WORKFLOW_CORE_STAGE_DEFAULT_ID", "false")
+    )
+
+    # NOTE: Workflow
+    max_job_parallel: int = int(env("WORKFLOW_CORE_MAX_JOB_PARALLEL", "2"))
+
+    def __init__(self):
+        if self.max_job_parallel < 0:
+            raise ValueError(
+                f"MAX_JOB_PARALLEL should more than 0 but got "
+                f"{self.max_job_parallel}."
+            )
+
+
+config = Config()

ddeutil/workflow/cron.py

@@ -14,7 +14,7 @@ from typing import ClassVar, Optional, Union
 from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
 
 from ddeutil.core import (
-    is_int,
+    checker,
     isinstance_check,
     must_split,
 )
@@ -38,16 +38,21 @@ class CronYearLimit(Exception): ...
 
 
 def str2cron(value: str) -> str:
-    """Convert Special String to Crontab.
-
-    @reboot 	Run once, at system startup
-    @yearly 	Run once every year, "0 0 1 1 *"
-    @annually   (same as @yearly)
-    @monthly    Run once every month, "0 0 1 * *"
-    @weekly 	Run once every week, "0 0 * * 0"
-    @daily  	Run once each day, "0 0 * * *"
-    @midnight   (same as @daily)
-    @hourly 	Run once an hour, "0 * * * *"
+    """Convert Special String with the @ prefix to Crontab value.
+
+    :param value: A string value that want to convert to cron value.
+    :rtype: str
+
+    Table:
+
+        @reboot 	Run once, at system startup
+        @yearly 	Run once every year, "0 0 1 1 *"
+        @annually   (same as @yearly)
+        @monthly    Run once every month, "0 0 1 * *"
+        @weekly 	Run once every week, "0 0 * * 0"
+        @daily  	Run once each day, "0 0 * * *"
+        @midnight   (same as @daily)
+        @hourly 	Run once an hour, "0 * * * *"
     """
     mapping_spacial_str = {
         "@reboot": "",
@@ -321,7 +326,9 @@ class CronPart:
                 self._parse_range(value_range)
             )
 
-            if (value_step and not is_int(value_step)) or value_step == "":
+            if (
+                value_step and not checker.is_int(value_step)
+            ) or value_step == "":
                 raise ValueError(
                     f"Invalid interval step value {value_step!r} for "
                     f"{self.unit.name!r}"