ddeutil-workflow 0.0.34__py3-none-any.whl → 0.0.36__py3-none-any.whl

ddeutil/workflow/stages.py

@@ -14,8 +14,8 @@ use-case, and it does not worry when I want to create a new one.
 
     Execution   --> Ok      --> Result with 0
 
-                --> Error   --> Result with 1 (if env var was set)
-                            --> Raise StageException(...)
+                --> Error   ┬-> Result with 1 (if env var was set)
+                            ╰-> Raise StageException(...)
 
     On the context I/O that pass to a stage object at execute process. The
 execute method receives a `params={"params": {...}}` value for mapping to
@@ -42,7 +42,7 @@ from pydantic.functional_validators import model_validator
 from typing_extensions import Self
 
 from .__types import DictData, DictStr, TupleStr
-from .call import TagFunc, extract_call
+from .caller import TagFunc, extract_call
 from .conf import config, get_logger
 from .exceptions import StageException
 from .result import Result, Status
@@ -174,15 +174,12 @@ class BaseStage(BaseModel, ABC):
 
         :rtype: Result
         """
-        if result is None:
-            result: Result = Result(
-                run_id=(
-                    run_id or gen_id(self.name + (self.id or ""), unique=True)
-                ),
-                parent_run_id=parent_run_id,
-            )
-        elif parent_run_id:
-            result.set_parent_run_id(parent_run_id)
+        result: Result = Result.construct_with_rs_or_id(
+            result,
+            run_id=run_id,
+            parent_run_id=parent_run_id,
+            id_logic=(self.name + (self.id or "")),
+        )
 
         try:
             return self.execute(params, result=result)
@@ -736,6 +733,26 @@ Stage = Union[
 
 # TODO: Not implement this stages yet
 class ParallelStage(BaseStage):  # pragma: no cov
+    """Parallel execution stage that execute child stages with parallel.
+
+    Data Validate:
+        >>> stage = {
+        ...     "name": "Parallel stage execution.",
+        ...     "parallel": [
+        ...         {
+        ...             "name": "Echo first stage",
+        ...             "echo": "Start run with branch 1",
+        ...             "sleep": 3,
+        ...         },
+        ...         {
+        ...             "name": "Echo second stage",
+        ...             "echo": "Start run with branch 2",
+        ...             "sleep": 1,
+        ...         },
+        ...     ]
+        ... }
+    """
+
     parallel: list[Stage]
     max_parallel_core: int = Field(default=2)
 
@@ -746,6 +763,22 @@ class ParallelStage(BaseStage):  # pragma: no cov
 
 # TODO: Not implement this stages yet
 class ForEachStage(BaseStage):  # pragma: no cov
+    """For-Each execution stage that execute child stages with an item in list of
+    item values.
+
+    Data Validate:
+        >>> stage = {
+        ...     "name": "For-each stage execution",
+        ...     "foreach": [1, 2, 3]
+        ...     "stages": [
+        ...         {
+        ...             "name": "Echo stage",
+        ...             "echo": "Start run with item {{ item }}"
+        ...         },
+        ...     ],
+        ... }
+    """
+
     foreach: list[str]
     stages: list[Stage]
 
@@ -756,9 +789,30 @@ class ForEachStage(BaseStage):  # pragma: no cov
 
 # TODO: Not implement this stages yet
 class HookStage(BaseStage):  # pragma: no cov
-    foreach: list[str]
-    stages: list[Stage]
+    hook: str
+    args: DictData
+    callback: str
+
+    def execute(
+        self, params: DictData, *, result: Result | None = None
+    ) -> Result: ...
+
+
+# TODO: Not implement this stages yet
+class DockerStage(BaseStage):  # pragma: no cov
+    image: str
+    env: DictData = Field(default_factory=dict)
 
     def execute(
         self, params: DictData, *, result: Result | None = None
     ) -> Result: ...
+
+
+# TODO: Not implement this stages yet
+class VirtualPyStage(PyStage):  # pragma: no cov
+    """Python Virtual Environment stage execution."""
+
+    run: str
+    vars: DictData
+
+    def create_py_file(self, py: str, run_id: str | None): ...

ddeutil/workflow/utils.py

@@ -9,7 +9,7 @@ import logging
 import stat
 import time
 from collections.abc import Iterator, Mapping
-from datetime import datetime, timedelta
+from datetime import date, datetime, timedelta
 from hashlib import md5
 from inspect import isfunction
 from itertools import chain, islice, product
@@ -39,6 +39,12 @@ def get_dt_now(
     return datetime.now(tz=(tz or UTC)) - timedelta(seconds=offset)
 
 
+def get_d_now(
+    tz: ZoneInfo | None = None, offset: float = 0.0
+) -> date:  # pragma: no cov
+    return (datetime.now(tz=(tz or UTC)) - timedelta(seconds=offset)).date()
+
+
 def get_diff_sec(
     dt: datetime, tz: ZoneInfo | None = None, offset: float = 0.0
 ) -> int:  # pragma: no cov

ddeutil/workflow/workflow.py

@@ -3,21 +3,7 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
-"""The main schedule running is ``workflow_runner`` function that trigger the
-multiprocess of ``workflow_control`` function for listing schedules on the
-config by ``Loader.finds(Schedule)``.
-
-    The ``workflow_control`` is the scheduler function that release 2 schedule
-functions; ``workflow_task``, and ``workflow_monitor``.
-
-    ``workflow_control`` --- Every minute at :02 --> ``workflow_task``
-
-                         --- Every 5 minutes     --> ``workflow_monitor``
-
-    The ``workflow_task`` will run ``task.release`` method in threading object
-for multithreading strategy. This ``release`` method will run only one crontab
-value with the on field.
-"""
+"""A Workflow module."""
 from __future__ import annotations
 
 import copy
@@ -529,7 +515,7 @@ class Workflow(BaseModel):
                 run_id=(run_id or gen_id(name, unique=True)),
                 parent_run_id=parent_run_id,
             )
-        elif parent_run_id:
+        elif parent_run_id:  # pragma: no cov
             result.set_parent_run_id(parent_run_id)
 
         if queue is not None and not isinstance(queue, ReleaseQueue):

{ddeutil_workflow-0.0.34.dist-info → ddeutil_workflow-0.0.36.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: ddeutil-workflow
-Version: 0.0.34
+Version: 0.0.36
 Summary: Lightweight workflow orchestration
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT
@@ -29,6 +29,8 @@ Requires-Dist: python-dotenv==1.0.1
 Requires-Dist: schedule<2.0.0,==1.2.2
 Provides-Extra: api
 Requires-Dist: fastapi<1.0.0,>=0.115.0; extra == "api"
+Requires-Dist: httpx; extra == "api"
+Requires-Dist: ujson; extra == "api"
 
 # Workflow Orchestration
 
@@ -62,7 +64,7 @@ configuration. It called **Metadata Driven Data Workflow**.
 1. The Minimum frequency unit of scheduling is **1 minute** :warning:
 2. Can not re-run only failed stage and its pending downstream :rotating_light:
 3. All parallel tasks inside workflow engine use Multi-Threading
-   (Python 3.13 unlock GIL :unlock:)
+   (🐍 Python 3.13 unlock GIL :unlock:)
 
 ---
 
@@ -78,45 +80,48 @@ flowchart LR
 
     subgraph Docker Container
         direction TB
-        G@{ shape: rounded, label: "Observe<br>Application" }
+        G@{ shape: rounded, label: "📡Observe<br>Application" }
     end
 
     subgraph Docker Container
         direction TB
-        B@{ shape: rounded, label: "Workflow<br>Application" }
+        B@{ shape: rounded, label: "🏃Workflow<br>Application" }
     end
 
-    A <--->|action &<br>response| B
-    B -....-> |response| G
-    G -....-> |request| B
+    A <-->|action &<br>response| B
+    B -...-> |response| G
+    G -...-> |request| B
 
     subgraph Data Context
-    D@{ shape: processes, label: "Logs" }
-    E@{ shape: lin-cyl, label: "Audit<br>Logs" }
+        D@{ shape: processes, label: "Logs" }
+        E@{ shape: lin-cyl, label: "Audit<br>Logs" }
     end
 
-    subgraph Git Context
-    F@{ shape: tag-rect, label: "YAML<br>files" }
+    subgraph Config Context
+        F@{ shape: tag-rect, label: "YAML<br>files" }
     end
 
-    B --->|disable| F
-    F --->|read| B
+    A ---> |push| H(Repo)
+    H -.-> |pull| F
 
-    B --->|write| E
-    E --->|read| B
-    B --->|write| D
+    B <-->|disable &<br>read| F
+
+    B <-->|read &<br>write| E
+
+    B -->|write| D
 
     D -.->|read| G
     E -.->|read| G
 ```
 
 > [!WARNING]
-> _Disclaimer_: I inspire the dynamic statement from the [**GitHub Action**](https://github.com/features/actions)
-> with `.yml` files and all configs file from several data orchestration framework
-> tools from my experience on Data Engineer. :grimacing:
->
-> Other workflow tools that I interest on them and pick some interested feature
-> implement to this package:
+> _**Disclaimer**_: I inspire the dynamic YAML statement from the [**GitHub Action**](https://github.com/features/actions),
+> and all configs pattern from several data orchestration framework tools from
+> my data engineering experience. :grimacing:
+
+> [!NOTE]
+> Other workflow orchestration tools that I interest and pick them to be inspiration
+> some for this package:
 >
 > - [Google **Workflows**](https://cloud.google.com/workflows)
 > - [AWS **Step Functions**](https://aws.amazon.com/step-functions/)
@@ -127,10 +132,10 @@ This project need `ddeutil` and `ddeutil-io` extension namespace packages.
 If you want to install this package with application add-ons, you should add
 `app` in installation;
 
-| Use-case       | Install Optional         | Support            |
-|----------------|--------------------------|--------------------|
-| Python         | `ddeutil-workflow`       | :heavy_check_mark: |
-| FastAPI Server | `ddeutil-workflow[api]`  | :heavy_check_mark: |
+| Use-case       | Install Optional         |       Support       |
+|----------------|--------------------------|:-------------------:|
+| Python         | `ddeutil-workflow`       | :heavy_check_mark:  |
+| FastAPI Server | `ddeutil-workflow[api]`  | :heavy_check_mark:  |
 
 ## :beers: Usage
 
@@ -292,7 +297,7 @@ like crontab job but via Python API.
 ### API Server
 
 ```shell
-(venv) $ uvicorn src.ddeutil.workflow.api:app \
+(venv) $ uvicorn ddeutil.workflow.api:app \
   --host 127.0.0.1 \
   --port 80 \
   --no-access-log

ddeutil_workflow-0.0.36.dist-info/RECORD

@@ -0,0 +1,31 @@
+ddeutil/workflow/__about__.py,sha256=sUrITqcQ8T6XPgdiHuk6l8YqDYHRQk2RjtNlY0SjLcs,28
+ddeutil/workflow/__cron.py,sha256=3i-wmjTlh0ADCzN9pLKaWHzJkXzC72aIBmVEQSbyCCE,26895
+ddeutil/workflow/__init__.py,sha256=d643WDkk93MjCt9ujD46hX07Mb7yc9eTzKV3QwnEZQg,1845
+ddeutil/workflow/__types.py,sha256=CK1jfzyHP9P-MB0ElhpJZ59ZFGJC9MkQuAop5739_9k,4304
+ddeutil/workflow/audit.py,sha256=wx70RKRdHj1d2431ilpt9OPTInMByjqXkYff7l5pvF4,8230
+ddeutil/workflow/caller.py,sha256=qNfrr0B2ykLm6Y8JfhU1rnf5XGmp-usXrPmXM5uGRnM,5473
+ddeutil/workflow/conf.py,sha256=cFc2cd_SGXg9PMrkvCT7WWE85a5UN-DdH53_JIbFyzs,14031
+ddeutil/workflow/cron.py,sha256=j8EeoHst70toRfnD_frix41vrI-eLYVJkZ9yeJtpfnI,8871
+ddeutil/workflow/exceptions.py,sha256=5ghT443VLq0IeU87loHNEqqrrrctklP7YfxwJ51ImWU,949
+ddeutil/workflow/job.py,sha256=lpuWQdAFlFUmP0H7ha217sDwbvFlqSuqyAo4Q1yhAa8,24527
+ddeutil/workflow/logs.py,sha256=EJDb9Xt3XWjTGE8CeEvw0eDU8kyaeStALQNAtTl5HQw,10027
+ddeutil/workflow/params.py,sha256=qw9XJyjh2ocf9pf6h_XiYHLOvQN4R5TMqPElmItKnRM,8019
+ddeutil/workflow/result.py,sha256=fbM2An3VyweMjAy4Iw7h8H-KkoQsDrZe_KjGztXAFkE,4319
+ddeutil/workflow/scheduler.py,sha256=6KozyZZSXtDqQxpUwS4o-J1h_HhGbwFV5Abul3_I4W4,25940
+ddeutil/workflow/stages.py,sha256=euT_v_xk0iqxWOFVZCHuFN8zREgc3-d06j5LMY8AJaE,28020
+ddeutil/workflow/templates.py,sha256=A0JgZFGkBv-AX-EskZj656nG5zFd3j1PpLpyXihf6Xg,10967
+ddeutil/workflow/utils.py,sha256=Djzd7Bz5QYAw0GqTFv6a0yWm_D8Md0XDxjWudmscQI0,7406
+ddeutil/workflow/workflow.py,sha256=_LYfs15AcXWVpM8CaO4oH6SWoJnqzF5FU08QTDoHT5w,44529
+ddeutil/workflow/api/__init__.py,sha256=F53NMBWtb9IKaDWkPU5KvybGGfKAcbehgn6TLBwHuuM,21
+ddeutil/workflow/api/api.py,sha256=fBFfJtpf7CL4Vym4iiZtWsrrnICGnwYKHBTwLoatPB4,5151
+ddeutil/workflow/api/repeat.py,sha256=g7VAP4O8ocKj5Uts5Q9P-OAfeDhKk7a4VWQDlOttTXA,5280
+ddeutil/workflow/api/routes/__init__.py,sha256=qoGtOMyVgQ5nTUc8J8wH27A8isaxl3IFCX8qoyibeCY,484
+ddeutil/workflow/api/routes/job.py,sha256=BXh8ikYKicCgzt13MysbtGgLrAPuts4dpgkD7iHTjrs,1901
+ddeutil/workflow/api/routes/logs.py,sha256=sHhQuigOQ6YQ34V56d9Is6eHbV-zE45v6gPntD7lgS8,1909
+ddeutil/workflow/api/routes/schedules.py,sha256=Oj8QLJJ852Lv2y40wGhuXjY2IEdhjAoGBxSWCkxe3YY,4675
+ddeutil/workflow/api/routes/workflows.py,sha256=LQiLlB3tESUtOZWcuLUKIHKvnit_yX3fKzYB4FqhavI,4354
+ddeutil_workflow-0.0.36.dist-info/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
+ddeutil_workflow-0.0.36.dist-info/METADATA,sha256=OwDce5Gz0qVtJLQbmvbgthMfH6bD68Xaqm-B8RfLeRg,19354
+ddeutil_workflow-0.0.36.dist-info/WHEEL,sha256=52BFRY2Up02UkjOa29eZOS2VxUrpPORXg1pkohGGUS8,91
+ddeutil_workflow-0.0.36.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
+ddeutil_workflow-0.0.36.dist-info/RECORD,,

{ddeutil_workflow-0.0.34.dist-info → ddeutil_workflow-0.0.36.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.8.0)
+Generator: setuptools (76.0.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

ddeutil_workflow-0.0.34.dist-info/RECORD

@@ -1,26 +0,0 @@
-ddeutil/workflow/__about__.py,sha256=lXPz0LJuQJgNzG0V-1UIfdYsebVctya1i8YqUhNgkXU,28
-ddeutil/workflow/__cron.py,sha256=3i-wmjTlh0ADCzN9pLKaWHzJkXzC72aIBmVEQSbyCCE,26895
-ddeutil/workflow/__init__.py,sha256=ucsWC4fWdD3ZJu3mDLwe1BCS7wHR0-43jtxm8eZe53o,1779
-ddeutil/workflow/__types.py,sha256=CK1jfzyHP9P-MB0ElhpJZ59ZFGJC9MkQuAop5739_9k,4304
-ddeutil/workflow/audit.py,sha256=LZDgZbWioy0ctEWBmTUXJY_cAdFEMxvkvamJbkVAhdc,8066
-ddeutil/workflow/call.py,sha256=gkEmrYZT1KFVMVy6uyXcl2wBPQXyg1onPLkXN3q9JJk,5469
-ddeutil/workflow/conf.py,sha256=cFc2cd_SGXg9PMrkvCT7WWE85a5UN-DdH53_JIbFyzs,14031
-ddeutil/workflow/cron.py,sha256=j8EeoHst70toRfnD_frix41vrI-eLYVJkZ9yeJtpfnI,8871
-ddeutil/workflow/exceptions.py,sha256=5ghT443VLq0IeU87loHNEqqrrrctklP7YfxwJ51ImWU,949
-ddeutil/workflow/job.py,sha256=cm4u_l3fopL6tgUZYy-apZMGektaFyB5qu9uja3Nups,23776
-ddeutil/workflow/params.py,sha256=LKR7jXyxTb5NVrFav_fl2y9xo3p7qL1S9h-i6CtvNwE,5851
-ddeutil/workflow/result.py,sha256=z3C1V9C8D736gqTefBH0uUFr40pNU5Z2TwV2Pndoxmw,7800
-ddeutil/workflow/scheduler.py,sha256=5xlgzunC-eDOc0uSxJZhphywVOwrtmuF0mT5YYc83BY,24943
-ddeutil/workflow/stages.py,sha256=5UhmcDgJ214NSNyTwgYcX2hvSW1Rcgk_Qq71n1VMdZ4,26555
-ddeutil/workflow/templates.py,sha256=A0JgZFGkBv-AX-EskZj656nG5zFd3j1PpLpyXihf6Xg,10967
-ddeutil/workflow/utils.py,sha256=MctJmvklTYtiqZ-nZ7fazQeDoe77UvU0YUEqQZSlbCs,7225
-ddeutil/workflow/workflow.py,sha256=7SvF80y4GrrwL-0X46wlct76qPp_fttZpEgxu5xjcBY,45148
-ddeutil/workflow/api/__init__.py,sha256=F53NMBWtb9IKaDWkPU5KvybGGfKAcbehgn6TLBwHuuM,21
-ddeutil/workflow/api/api.py,sha256=TfgJtu-yREsrRveLcqTjxoJszuhq21qKkl4oyQpSzvQ,3959
-ddeutil/workflow/api/repeat.py,sha256=zyvsrXKk-3-_N8ZRZSki0Mueshugum2jtqctEOp9QSc,4927
-ddeutil/workflow/api/route.py,sha256=fKs5zlG4prwL8HplXYjYsyuxelOKgV-54VBgBufIBcQ,8629
-ddeutil_workflow-0.0.34.dist-info/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
-ddeutil_workflow-0.0.34.dist-info/METADATA,sha256=J-Y3CGyv6r39SVYlasXC6-fVNFYhAtmWjM46CDoYz_0,19221
-ddeutil_workflow-0.0.34.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-ddeutil_workflow-0.0.34.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
-ddeutil_workflow-0.0.34.dist-info/RECORD,,