ddeutil-workflow 0.0.14__tar.gz → 0.0.16__tar.gz

{ddeutil_workflow-0.0.14 → ddeutil_workflow-0.0.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ddeutil-workflow
-Version: 0.0.14
+Version: 0.0.16
 Summary: Lightweight workflow orchestration with less dependencies
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT
@@ -22,8 +22,8 @@ Classifier: Programming Language :: Python :: 3.13
 Requires-Python: >=3.9.13
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: ddeutil>=0.4.0
-Requires-Dist: ddeutil-io>=0.1.13
+Requires-Dist: ddeutil>=0.4.3
+Requires-Dist: ddeutil-io[yaml]>=0.2.3
 Requires-Dist: python-dotenv==1.0.1
 Requires-Dist: typer<1.0.0,==0.12.5
 Requires-Dist: schedule<2.0.0,==1.2.2
@@ -179,29 +179,31 @@ The main configuration that use to dynamic changing with your propose of this
 application. If any configuration values do not set yet, it will use default value
 and do not raise any error to you.
 
-| Environment                         | Component | Default                          | Description                                                                |
-|-------------------------------------|-----------|----------------------------------|----------------------------------------------------------------------------|
-| `WORKFLOW_ROOT_PATH`                | Core      | .                                | The root path of the workflow application                                  |
-| `WORKFLOW_CORE_REGISTRY`            | Core      | src.ddeutil.workflow,tests.utils | List of importable string for the hook stage                               |
-| `WORKFLOW_CORE_REGISTRY_FILTER`     | Core      | ddeutil.workflow.utils           | List of importable string for the filter template                          |
-| `WORKFLOW_CORE_PATH_CONF`           | Core      | conf                             | The config path that keep all template `.yaml` files                       |
-| `WORKFLOW_CORE_TIMEZONE`            | Core      | Asia/Bangkok                     | A Timezone string value that will pass to `ZoneInfo` object                |
-| `WORKFLOW_CORE_STAGE_DEFAULT_ID`    | Core      | true                             | A flag that enable default stage ID that use for catch an execution output |
-| `WORKFLOW_CORE_STAGE_RAISE_ERROR`   | Core      | true                             | A flag that all stage raise StageException from stage execution            |
-| `WORKFLOW_CORE_MAX_NUM_POKING`      | Core      | 4                                |                                                                            |
-| `WORKFLOW_CORE_MAX_JOB_PARALLEL`    | Core      | 2                                | The maximum job number that able to run parallel in workflow executor      |
-| `WORKFLOW_LOG_DEBUG_MODE`           | Log       | true                             | A flag that enable logging with debug level mode                           |
-| `WORKFLOW_LOG_ENABLE_WRITE`         | Log       | true                             | A flag that enable logging object saving log to its destination            |
-| `WORKFLOW_APP_PROCESS_WORKER`       | Schedule  | 2                                | The maximum process worker number that run in scheduler app module         |
-| `WORKFLOW_APP_SCHEDULE_PER_PROCESS` | Schedule  | 100                              | A schedule per process that run parallel                                   |
-| `WORKFLOW_APP_STOP_BOUNDARY_DELTA`  | Schedule  | '{"minutes": 5, "seconds": 20}'  | A time delta value that use to stop scheduler app in json string format    |
+| Environment                             | Component | Default                          | Description                                                                                                        | Remark |
+|:----------------------------------------|-----------|----------------------------------|--------------------------------------------------------------------------------------------------------------------|--------|
+| `WORKFLOW_ROOT_PATH`                    | Core      | .                                | The root path of the workflow application                                                                          |        |
+| `WORKFLOW_CORE_REGISTRY`                | Core      | src.ddeutil.workflow,tests.utils | List of importable string for the hook stage                                                                       |        |
+| `WORKFLOW_CORE_REGISTRY_FILTER`         | Core      | ddeutil.workflow.utils           | List of importable string for the filter template                                                                  |        |
+| `WORKFLOW_CORE_PATH_CONF`               | Core      | conf                             | The config path that keep all template `.yaml` files                                                               |        |
+| `WORKFLOW_CORE_TIMEZONE`                | Core      | Asia/Bangkok                     | A Timezone string value that will pass to `ZoneInfo` object                                                        |        |
+| `WORKFLOW_CORE_STAGE_DEFAULT_ID`        | Core      | true                             | A flag that enable default stage ID that use for catch an execution output                                         |        |
+| `WORKFLOW_CORE_STAGE_RAISE_ERROR`       | Core      | false                            | A flag that all stage raise StageException from stage execution                                                    |        |
+| `WORKFLOW_CORE_JOB_DEFAULT_ID`          | Core      | false                            | A flag that enable default job ID that use for catch an execution output. The ID that use will be sequence number. |        |
+| `WORKFLOW_CORE_MAX_NUM_POKING`          | Core      | 4                                |                                                                                                                    |        |
+| `WORKFLOW_CORE_MAX_JOB_PARALLEL`        | Core      | 2                                | The maximum job number that able to run parallel in workflow executor                                              |        |
+| `WORKFLOW_CORE_WORKFLOW_ID_SIMPLE_MODE` | Core      | true                             |                                                                                                                    |        |
+| `WORKFLOW_LOG_DEBUG_MODE`               | Log       | true                             | A flag that enable logging with debug level mode                                                                   |        |
+| `WORKFLOW_LOG_ENABLE_WRITE`             | Log       | true                             | A flag that enable logging object saving log to its destination                                                    |        |
+| `WORKFLOW_APP_MAX_PROCESS`              | Schedule  | 2                                | The maximum process worker number that run in scheduler app module                                                 |        |
+| `WORKFLOW_APP_MAX_SCHEDULE_PER_PROCESS` | Schedule  | 100                              | A schedule per process that run parallel                                                                           |        |
+| `WORKFLOW_APP_STOP_BOUNDARY_DELTA`      | Schedule  | '{"minutes": 5, "seconds": 20}'  | A time delta value that use to stop scheduler app in json string format                                            |        |
 
 **API Application**:
 
-| Environment                          | Component | Default | Description                                                                       |
-|--------------------------------------|-----------|---------|-----------------------------------------------------------------------------------|
-| `WORKFLOW_API_ENABLE_ROUTE_WORKFLOW` | API       | true    | A flag that enable workflow route to manage execute manually and workflow logging |
-| `WORKFLOW_API_ENABLE_ROUTE_SCHEDULE` | API       | true    | A flag that enable run scheduler                                                  |
+| Environment                           | Component | Default | Description                                                                       | Remark |
+|:--------------------------------------|-----------|---------|-----------------------------------------------------------------------------------|--------|
+| `WORKFLOW_API_ENABLE_ROUTE_WORKFLOW`  | API       | true    | A flag that enable workflow route to manage execute manually and workflow logging |        |
+| `WORKFLOW_API_ENABLE_ROUTE_SCHEDULE`  | API       | true    | A flag that enable run scheduler                                                  |        |
 
 ## :rocket: Deployment
 
@@ -224,3 +226,17 @@ like crontab job but via Python API.
 > [!NOTE]
 > If this package already deploy, it able to use
 > `uvicorn ddeutil.workflow.api:app --host 127.0.0.1 --port 80 --workers 4`
+
+### Docker Container
+
+Create Docker image;
+
+```shell
+$ docker build -t ddeutil-workflow:latest -f .container/Dockerfile .
+```
+
+Run the above Docker image;
+
+```shell
+$ docker run -i ddeutil-workflow:latest
+```

{ddeutil_workflow-0.0.14 → ddeutil_workflow-0.0.16}/README.md

@@ -147,29 +147,31 @@ The main configuration that use to dynamic changing with your propose of this
 application. If any configuration values do not set yet, it will use default value
 and do not raise any error to you.
 
-| Environment                         | Component | Default                          | Description                                                                |
-|-------------------------------------|-----------|----------------------------------|----------------------------------------------------------------------------|
-| `WORKFLOW_ROOT_PATH`                | Core      | .                                | The root path of the workflow application                                  |
-| `WORKFLOW_CORE_REGISTRY`            | Core      | src.ddeutil.workflow,tests.utils | List of importable string for the hook stage                               |
-| `WORKFLOW_CORE_REGISTRY_FILTER`     | Core      | ddeutil.workflow.utils           | List of importable string for the filter template                          |
-| `WORKFLOW_CORE_PATH_CONF`           | Core      | conf                             | The config path that keep all template `.yaml` files                       |
-| `WORKFLOW_CORE_TIMEZONE`            | Core      | Asia/Bangkok                     | A Timezone string value that will pass to `ZoneInfo` object                |
-| `WORKFLOW_CORE_STAGE_DEFAULT_ID`    | Core      | true                             | A flag that enable default stage ID that use for catch an execution output |
-| `WORKFLOW_CORE_STAGE_RAISE_ERROR`   | Core      | true                             | A flag that all stage raise StageException from stage execution            |
-| `WORKFLOW_CORE_MAX_NUM_POKING`      | Core      | 4                                |                                                                            |
-| `WORKFLOW_CORE_MAX_JOB_PARALLEL`    | Core      | 2                                | The maximum job number that able to run parallel in workflow executor      |
-| `WORKFLOW_LOG_DEBUG_MODE`           | Log       | true                             | A flag that enable logging with debug level mode                           |
-| `WORKFLOW_LOG_ENABLE_WRITE`         | Log       | true                             | A flag that enable logging object saving log to its destination            |
-| `WORKFLOW_APP_PROCESS_WORKER`       | Schedule  | 2                                | The maximum process worker number that run in scheduler app module         |
-| `WORKFLOW_APP_SCHEDULE_PER_PROCESS` | Schedule  | 100                              | A schedule per process that run parallel                                   |
-| `WORKFLOW_APP_STOP_BOUNDARY_DELTA`  | Schedule  | '{"minutes": 5, "seconds": 20}'  | A time delta value that use to stop scheduler app in json string format    |
+| Environment                             | Component | Default                          | Description                                                                                                        | Remark |
+|:----------------------------------------|-----------|----------------------------------|--------------------------------------------------------------------------------------------------------------------|--------|
+| `WORKFLOW_ROOT_PATH`                    | Core      | .                                | The root path of the workflow application                                                                          |        |
+| `WORKFLOW_CORE_REGISTRY`                | Core      | src.ddeutil.workflow,tests.utils | List of importable string for the hook stage                                                                       |        |
+| `WORKFLOW_CORE_REGISTRY_FILTER`         | Core      | ddeutil.workflow.utils           | List of importable string for the filter template                                                                  |        |
+| `WORKFLOW_CORE_PATH_CONF`               | Core      | conf                             | The config path that keep all template `.yaml` files                                                               |        |
+| `WORKFLOW_CORE_TIMEZONE`                | Core      | Asia/Bangkok                     | A Timezone string value that will pass to `ZoneInfo` object                                                        |        |
+| `WORKFLOW_CORE_STAGE_DEFAULT_ID`        | Core      | true                             | A flag that enable default stage ID that use for catch an execution output                                         |        |
+| `WORKFLOW_CORE_STAGE_RAISE_ERROR`       | Core      | false                            | A flag that all stage raise StageException from stage execution                                                    |        |
+| `WORKFLOW_CORE_JOB_DEFAULT_ID`          | Core      | false                            | A flag that enable default job ID that use for catch an execution output. The ID that use will be sequence number. |        |
+| `WORKFLOW_CORE_MAX_NUM_POKING`          | Core      | 4                                |                                                                                                                    |        |
+| `WORKFLOW_CORE_MAX_JOB_PARALLEL`        | Core      | 2                                | The maximum job number that able to run parallel in workflow executor                                              |        |
+| `WORKFLOW_CORE_WORKFLOW_ID_SIMPLE_MODE` | Core      | true                             |                                                                                                                    |        |
+| `WORKFLOW_LOG_DEBUG_MODE`               | Log       | true                             | A flag that enable logging with debug level mode                                                                   |        |
+| `WORKFLOW_LOG_ENABLE_WRITE`             | Log       | true                             | A flag that enable logging object saving log to its destination                                                    |        |
+| `WORKFLOW_APP_MAX_PROCESS`              | Schedule  | 2                                | The maximum process worker number that run in scheduler app module                                                 |        |
+| `WORKFLOW_APP_MAX_SCHEDULE_PER_PROCESS` | Schedule  | 100                              | A schedule per process that run parallel                                                                           |        |
+| `WORKFLOW_APP_STOP_BOUNDARY_DELTA`      | Schedule  | '{"minutes": 5, "seconds": 20}'  | A time delta value that use to stop scheduler app in json string format                                            |        |
 
 **API Application**:
 
-| Environment                          | Component | Default | Description                                                                       |
-|--------------------------------------|-----------|---------|-----------------------------------------------------------------------------------|
-| `WORKFLOW_API_ENABLE_ROUTE_WORKFLOW` | API       | true    | A flag that enable workflow route to manage execute manually and workflow logging |
-| `WORKFLOW_API_ENABLE_ROUTE_SCHEDULE` | API       | true    | A flag that enable run scheduler                                                  |
+| Environment                           | Component | Default | Description                                                                       | Remark |
+|:--------------------------------------|-----------|---------|-----------------------------------------------------------------------------------|--------|
+| `WORKFLOW_API_ENABLE_ROUTE_WORKFLOW`  | API       | true    | A flag that enable workflow route to manage execute manually and workflow logging |        |
+| `WORKFLOW_API_ENABLE_ROUTE_SCHEDULE`  | API       | true    | A flag that enable run scheduler                                                  |        |
 
 ## :rocket: Deployment
 
@@ -192,3 +194,17 @@ like crontab job but via Python API.
 > [!NOTE]
 > If this package already deploy, it able to use
 > `uvicorn ddeutil.workflow.api:app --host 127.0.0.1 --port 80 --workers 4`
+
+### Docker Container
+
+Create Docker image;
+
+```shell
+$ docker build -t ddeutil-workflow:latest -f .container/Dockerfile .
+```
+
+Run the above Docker image;
+
+```shell
+$ docker run -i ddeutil-workflow:latest
+```

{ddeutil_workflow-0.0.14 → ddeutil_workflow-0.0.16}/pyproject.toml

@@ -26,8 +26,8 @@ classifiers = [
 ]
 requires-python = ">=3.9.13"
 dependencies = [
-    "ddeutil>=0.4.0",
-    "ddeutil-io>=0.1.13",
+    "ddeutil>=0.4.3",
+    "ddeutil-io[yaml]>=0.2.3",
     "python-dotenv==1.0.1",
     "typer==0.12.5,<1.0.0",
     "schedule==1.2.2,<2.0.0",
@@ -60,7 +60,7 @@ changelog = "CHANGELOG.md"
 branch = true
 relative_files = true
 concurrency = ["thread", "multiprocessing"]
-source = ["ddeutil.workflow", "tests"]
+source = ["ddeutil.workflow"]
 omit = [
     "src/ddeutil/workflow/__about__.py",
     # Omit this files because it does not ready to production.
@@ -68,12 +68,13 @@ omit = [
     "src/ddeutil/workflow/cli.py",
     "src/ddeutil/workflow/repeat.py",
     "src/ddeutil/workflow/route.py",
-    "tests/utils.py",
-    "tests/tasks/dummy.py",
 ]
 
 [tool.coverage.report]
-exclude_lines = ["raise NotImplementedError"]
+exclude_lines = [
+    "raise NotImplementedError",
+    "pragma: no cov",
+]
 
 [tool.pytest.ini_options]
 pythonpath = ["src"]

ddeutil_workflow-0.0.16/src/ddeutil/workflow/__about__.py

@@ -0,0 +1 @@
+__version__: str = "0.0.16"

ddeutil_workflow-0.0.16/src/ddeutil/workflow/__types.py

@@ -0,0 +1,125 @@
+# ------------------------------------------------------------------------------
+# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
+# Licensed under the MIT License. See LICENSE in the project root for
+# license information.
+# ------------------------------------------------------------------------------
+from __future__ import annotations
+
+import re
+from collections.abc import Iterator
+from dataclasses import dataclass
+from re import (
+    IGNORECASE,
+    MULTILINE,
+    UNICODE,
+    VERBOSE,
+    Match,
+    Pattern,
+)
+from typing import Any, Optional, TypedDict, Union
+
+from typing_extensions import Self
+
+TupleStr = tuple[str, ...]
+DictData = dict[str, Any]
+DictStr = dict[str, str]
+Matrix = dict[str, Union[list[str], list[int]]]
+
+
+class Context(TypedDict):
+    params: dict[str, Any]
+    jobs: dict[str, Any]
+
+
+@dataclass(frozen=True)
+class CallerRe:
+    """Caller dataclass that catching result from the matching regex with the
+    Re.RE_CALLER value.
+    """
+
+    full: str
+    caller: str
+    caller_prefix: Optional[str]
+    caller_last: str
+    post_filters: str
+
+    @classmethod
+    def from_regex(cls, match: Match[str]) -> Self:
+        """Class construct from matching result.
+
+        :rtype: Self
+        """
+        return cls(full=match.group(0), **match.groupdict())
+
+
+class Re:
+    """Regular expression config for this package."""
+
+    # NOTE:
+    #   Regular expression:
+    #       - Version 1:
+    #         \${{\s*(?P<caller>[a-zA-Z0-9_.\s'\"\[\]\(\)\-\{}]+?)\s*(?P<post_filters>(?:\|\s*(?:[a-zA-Z0-9_]{3,}[a-zA-Z0-9_.,-\\%\s'\"[\]()\{}]+)\s*)*)}}
+    #       - Version 2: (2024-09-30):
+    #         \${{\s*(?P<caller>(?P<caller_prefix>(?:[a-zA-Z_-]+\.)*)(?P<caller_last>[a-zA-Z0-9_\-.'\"(\)[\]{}]+))\s*(?P<post_filters>(?:\|\s*(?:[a-zA-Z0-9_]{3,}[a-zA-Z0-9_.,-\\%\s'\"[\]()\{}]+)\s*)*)}}
+    #       - Version 3: (2024-10-05):
+    #         \${{\s*(?P<caller>(?P<caller_prefix>(?:[a-zA-Z_-]+\??\.)*)(?P<caller_last>[a-zA-Z0-9_\-.'\"(\)[\]{}]+\??))\s*(?P<post_filters>(?:\|\s*(?:[a-zA-Z0-9_]{3,}[a-zA-Z0-9_.,-\\%\s'\"[\]()\{}]+)\s*)*)}}
+    #
+    #   Examples:
+    #       - ${{ params.data_dt }}
+    #       - ${{ params.source.table }}
+    #       - ${{ params.datetime | fmt('%Y-%m-%d') }}
+    #       - ${{ params.source?.schema }}
+    #
+    __re_caller: str = r"""
+        \$
+        {{
+            \s*
+            (?P<caller>
+                (?P<caller_prefix>(?:[a-zA-Z_-]+\??\.)*)
+                (?P<caller_last>[a-zA-Z0-9_\-.'\"(\)[\]{}]+\??)
+            )
+            \s*
+            (?P<post_filters>
+                (?:
+                    \|\s*
+                    (?:
+                        [a-zA-Z0-9_]{3,}
+                        [a-zA-Z0-9_.,-\\%\s'\"[\]()\{}]*
+                    )\s*
+                )*
+            )
+        }}
+    """
+    RE_CALLER: Pattern = re.compile(
+        __re_caller, MULTILINE | IGNORECASE | UNICODE | VERBOSE
+    )
+
+    # NOTE:
+    #   Regular expression:
+    #       - Version 1:
+    #         ^(?P<path>[^/@]+)/(?P<func>[^@]+)@(?P<tag>.+)$
+    #
+    #   Examples:
+    #       - tasks/function@dummy
+    __re_task_fmt: str = r"""
+        ^
+            (?P<path>[^/@]+)
+            /
+            (?P<func>[^@]+)
+            @
+            (?P<tag>.+)
+        $
+    """
+    RE_TASK_FMT: Pattern = re.compile(
+        __re_task_fmt, MULTILINE | IGNORECASE | UNICODE | VERBOSE
+    )
+
+    @classmethod
+    def finditer_caller(cls, value) -> Iterator[CallerRe]:
+        """Generate CallerRe object that create from matching object that
+        extract with re.finditer function.
+
+        :rtype: Iterator[CallerRe]
+        """
+        for found in cls.RE_CALLER.finditer(value):
+            yield CallerRe.from_regex(found)

{ddeutil_workflow-0.0.14 → ddeutil_workflow-0.0.16}/src/ddeutil/workflow/api.py

@@ -7,7 +7,6 @@ from __future__ import annotations
 
 import asyncio
 import contextlib
-import os
 import uuid
 from collections.abc import AsyncIterator
 from datetime import datetime, timedelta
@@ -15,7 +14,6 @@ from queue import Empty, Queue
 from threading import Thread
 from typing import TypedDict
 
-from ddeutil.core import str2bool
 from dotenv import load_dotenv
 from fastapi import FastAPI
 from fastapi.middleware.gzip import GZipMiddleware
@@ -23,6 +21,7 @@ from fastapi.responses import UJSONResponse
 from pydantic import BaseModel
 
 from .__about__ import __version__
+from .conf import config
 from .log import get_logger
 from .repeat import repeat_at, repeat_every
 from .scheduler import WorkflowTaskData
@@ -131,12 +130,12 @@ async def message_upper(payload: Payload):
     return await get_result(request_id)
 
 
-if str2bool(os.getenv("WORKFLOW_API_ENABLE_ROUTE_WORKFLOW", "true")):
+if config.enable_route_workflow:
     from .route import workflow
 
     app.include_router(workflow)
 
-if str2bool(os.getenv("WORKFLOW_API_ENABLE_ROUTE_SCHEDULE", "true")):
+if config.enable_route_schedule:
     from .route import schedule
     from .scheduler import workflow_task
 

{ddeutil_workflow-0.0.14 → ddeutil_workflow-0.0.16}/src/ddeutil/workflow/cli.py

@@ -6,15 +6,14 @@
 from __future__ import annotations
 
 import json
-import os
 from datetime import datetime
 from enum import Enum
 from typing import Annotated, Optional
-from zoneinfo import ZoneInfo
 
 from ddeutil.core import str2list
 from typer import Argument, Option, Typer
 
+from .conf import config
 from .log import get_logger
 
 logger = get_logger("ddeutil.workflow")
@@ -73,9 +72,7 @@ def schedule(
     excluded: list[str] = str2list(excluded) if excluded else []
     externals: str = externals or "{}"
     if stop:
-        stop: datetime = stop.astimezone(
-            tz=ZoneInfo(os.getenv("WORKFLOW_CORE_TIMEZONE", "UTC"))
-        )
+        stop: datetime = stop.astimezone(tz=config.tz)
 
     from .scheduler import workflow_runner