ddeutil-workflow 0.0.51__tar.gz → 0.0.52__tar.gz

{ddeutil_workflow-0.0.51/src/ddeutil_workflow.egg-info → ddeutil_workflow-0.0.52}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddeutil-workflow
-Version: 0.0.51
+Version: 0.0.52
 Summary: Lightweight workflow orchestration
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT

ddeutil_workflow-0.0.52/src/ddeutil/workflow/__about__.py

@@ -0,0 +1 @@
+__version__: str = "0.0.52"

{ddeutil_workflow-0.0.51 → ddeutil_workflow-0.0.52}/src/ddeutil/workflow/__init__.py

@@ -60,33 +60,6 @@ from .scheduler import (
     schedule_runner,
     schedule_task,
 )
-from .stages import (
-    BashStage,
-    CallStage,
-    EmptyStage,
-    ForEachStage,
-    ParallelStage,
-    PyStage,
-    Stage,
-    TriggerStage,
-)
-from .utils import (
-    batch,
-    cross_product,
-    default_gen_id,
-    delay,
-    filter_func,
-    gen_id,
-    get_diff_sec,
-    get_dt_now,
-    make_exec,
-    reach_next_minute,
-    replace_sec,
-    wait_to_next_minute,
-)
-from .workflow import (
-    Release,
-    ReleaseQueue,
-    Workflow,
-    WorkflowTask,
-)
+from .stages import *
+from .utils import *
+from .workflow import *

{ddeutil_workflow-0.0.51 → ddeutil_workflow-0.0.52}/src/ddeutil/workflow/job.py

@@ -27,7 +27,7 @@ from threading import Event
 from typing import Annotated, Any, Literal, Optional, Union
 
 from ddeutil.core import freeze_args
-from pydantic import BaseModel, ConfigDict, Discriminator, Field, SecretStr, Tag
+from pydantic import BaseModel, Discriminator, Field, SecretStr, Tag
 from pydantic.functional_validators import field_validator, model_validator
 from typing_extensions import Self
 
@@ -178,7 +178,7 @@ class Strategy(BaseModel):
 
 
 class Rule(str, Enum):
-    """Trigger rules enum object."""
+    """Rule enum object for assign trigger option."""
 
     ALL_SUCCESS: str = "all_success"
     ALL_FAILED: str = "all_failed"
@@ -203,8 +203,6 @@ class BaseRunsOn(BaseModel):  # pragma: no cov
     object and override execute method.
     """
 
-    model_config = ConfigDict(use_enum_values=True)
-
     type: RunsOn = Field(description="A runs-on type.")
     args: DictData = Field(
         default_factory=dict,
@@ -219,7 +217,9 @@ class BaseRunsOn(BaseModel):  # pragma: no cov
 class OnLocal(BaseRunsOn):  # pragma: no cov
     """Runs-on local."""
 
-    type: Literal[RunsOn.LOCAL] = Field(default=RunsOn.LOCAL)
+    type: Literal[RunsOn.LOCAL] = Field(
+        default=RunsOn.LOCAL, validate_default=True
+    )
 
 
 class SelfHostedArgs(BaseModel):
@@ -231,7 +231,9 @@ class SelfHostedArgs(BaseModel):
 class OnSelfHosted(BaseRunsOn):  # pragma: no cov
     """Runs-on self-hosted."""
 
-    type: Literal[RunsOn.SELF_HOSTED] = Field(default=RunsOn.SELF_HOSTED)
+    type: Literal[RunsOn.SELF_HOSTED] = Field(
+        default=RunsOn.SELF_HOSTED, validate_default=True
+    )
     args: SelfHostedArgs = Field(alias="with")
 
 
@@ -245,7 +247,9 @@ class AzBatchArgs(BaseModel):
 
 class OnAzBatch(BaseRunsOn):  # pragma: no cov
 
-    type: Literal[RunsOn.AZ_BATCH] = Field(default=RunsOn.AZ_BATCH)
+    type: Literal[RunsOn.AZ_BATCH] = Field(
+        default=RunsOn.AZ_BATCH, validate_default=True
+    )
     args: AzBatchArgs = Field(alias="with")
 
 
@@ -264,13 +268,16 @@ class DockerArgs(BaseModel):
 class OnDocker(BaseRunsOn):  # pragma: no cov
     """Runs-on Docker container."""
 
-    type: Literal[RunsOn.DOCKER] = Field(default=RunsOn.DOCKER)
+    type: Literal[RunsOn.DOCKER] = Field(
+        default=RunsOn.DOCKER, validate_default=True
+    )
     args: DockerArgs = Field(alias="with", default_factory=DockerArgs)
 
 
-def get_discriminator_runs_on(model: dict[str, Any]) -> str:
+def get_discriminator_runs_on(model: dict[str, Any]) -> RunsOn:
     """Get discriminator of the RunsOn models."""
-    return model.get("type", "local")
+    t = model.get("type")
+    return RunsOn(t) if t else RunsOn.LOCAL
 
 
 RunsOnModel = Annotated[
@@ -336,6 +343,7 @@ class Job(BaseModel):
     )
     trigger_rule: Rule = Field(
         default=Rule.ALL_SUCCESS,
+        validate_default=True,
         description=(
             "A trigger rule of tracking needed jobs if feature will use when "
             "the `raise_error` did not set from job and stage executions."
@@ -609,7 +617,7 @@ class Job(BaseModel):
         )
 
         result.trace.info(
-            f"[JOB]: Execute: {self.id!r} on {self.runs_on.type!r}"
+            f"[JOB]: Execute: {self.id!r} on {self.runs_on.type.value!r}"
         )
         if self.runs_on.type == RunsOn.LOCAL:
             return local_execute(

{ddeutil_workflow-0.0.51 → ddeutil_workflow-0.0.52}/src/ddeutil/workflow/stages.py

@@ -1099,14 +1099,15 @@ class ParallelStage(BaseStage):  # pragma: no cov
         :rtype: DictData
         """
         result.trace.debug(f"... Execute branch: {branch!r}")
-        context: DictData = copy.deepcopy(params)
-        context.update({"branch": branch, "stages": {}})
+        _params: DictData = copy.deepcopy(params)
+        _params.update({"branch": branch})
+        context: DictData = {"branch": branch, "stages": {}}
         for stage in self.parallel[branch]:
 
             if extras:
                 stage.extras = extras
 
-            if stage.is_skipped(params=context):
+            if stage.is_skipped(params=_params):
                 result.trace.info(f"... Skip stage: {stage.iden!r}")
                 stage.set_outputs(output={"skipped": True}, to=context)
                 continue
@@ -1129,7 +1130,7 @@ class ParallelStage(BaseStage):  # pragma: no cov
 
             try:
                 rs: Result = stage.handler_execute(
-                    params=context,
+                    params=_params,
                     run_id=result.run_id,
                     parent_run_id=result.parent_run_id,
                     raise_error=True,
@@ -1294,14 +1295,15 @@ class ForEachStage(BaseStage):
         :rtype: Result
         """
         result.trace.debug(f"... Execute item: {item!r}")
-        context: DictData = copy.deepcopy(params)
-        context.update({"item": item, "stages": {}})
+        _params: DictData = copy.deepcopy(params)
+        _params.update({"item": item})
+        context: DictData = {"item": item, "stages": {}}
         for stage in self.stages:
 
             if self.extras:
                 stage.extras = self.extras
 
-            if stage.is_skipped(params=context):
+            if stage.is_skipped(params=_params):
                 result.trace.info(f"... Skip stage: {stage.iden!r}")
                 stage.set_outputs(output={"skipped": True}, to=context)
                 continue
@@ -1324,7 +1326,7 @@ class ForEachStage(BaseStage):
 
             try:
                 rs: Result = stage.handler_execute(
-                    params=context,
+                    params=_params,
                     run_id=result.run_id,
                     parent_run_id=result.parent_run_id,
                     raise_error=True,
@@ -1513,15 +1515,16 @@ class UntilStage(BaseStage):  # pragma: no cov
         :rtype: tuple[Result, T]
         """
         result.trace.debug(f"... Execute until item: {item!r}")
-        context: DictData = copy.deepcopy(params)
-        context.update({"loop": loop, "item": item, "stages": {}})
+        _params: DictData = copy.deepcopy(params)
+        _params.update({"item": item})
+        context: DictData = {"loop": loop, "item": item, "stages": {}}
         next_item: T = None
         for stage in self.stages:
 
             if self.extras:
                 stage.extras = self.extras
 
-            if stage.is_skipped(params=context):
+            if stage.is_skipped(params=_params):
                 result.trace.info(f"... Skip stage: {stage.iden!r}")
                 stage.set_outputs(output={"skipped": True}, to=context)
                 continue
@@ -1550,7 +1553,7 @@ class UntilStage(BaseStage):  # pragma: no cov
 
             try:
                 rs: Result = stage.handler_execute(
-                    params=context,
+                    params=_params,
                     run_id=result.run_id,
                     parent_run_id=result.parent_run_id,
                     raise_error=True,

{ddeutil_workflow-0.0.51 → ddeutil_workflow-0.0.52}/src/ddeutil/workflow/workflow.py

@@ -371,7 +371,9 @@ class Workflow(BaseModel):
     def from_conf(
         cls,
         name: str,
+        *,
         extras: DictData | None = None,
+        loader: type[Loader] = None,
     ) -> Self:
         """Create Workflow instance from the Loader object that only receive
         an input workflow name. The loader object will use this workflow name to
@@ -380,12 +382,13 @@ class Workflow(BaseModel):
         :param name: A workflow name that want to pass to Loader object.
         :param extras: An extra parameters that want to pass to Loader
             object.
+        :param loader: A loader class for override default loader object.
 
         :raise ValueError: If the type does not match with current object.
 
         :rtype: Self
         """
-        loader: Loader = Loader(name, externals=(extras or {}))
+        loader: Loader = (loader or Loader)(name, externals=(extras or {}))
 
         # NOTE: Validate the config type match with current connection model
         if loader.type != cls.__name__:
@@ -407,6 +410,7 @@ class Workflow(BaseModel):
         path: Path,
         *,
         extras: DictData | None = None,
+        loader: type[Loader] = None,
     ) -> Self:
         """Create Workflow instance from the specific path. The loader object
         will use this workflow name and path to searching configuration data of
@@ -416,12 +420,13 @@ class Workflow(BaseModel):
         :param path: (Path) A config path that want to search.
         :param extras: (DictData) An extra parameters that want to override core
             config values.
+        :param loader: A loader class for override default loader object.
 
         :raise ValueError: If the type does not match with current object.
 
         :rtype: Self
         """
-        loader: SimLoad = SimLoad(
+        loader: SimLoad = (loader or SimLoad)(
             name, conf_path=path, externals=(extras or {})
         )
         # NOTE: Validate the config type match with current connection model

{ddeutil_workflow-0.0.51 → ddeutil_workflow-0.0.52/src/ddeutil_workflow.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddeutil-workflow
-Version: 0.0.51
+Version: 0.0.52
 Summary: Lightweight workflow orchestration
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT

{ddeutil_workflow-0.0.51 → ddeutil_workflow-0.0.52}/tests/test_stage_handler_exec.py

@@ -680,6 +680,168 @@ def test_stage_exec_foreach_with_trigger(test_path):
         }
 
 
+def test_stage_exec_multi_foreach_nested_with_trigger(test_path):
+    with dump_yaml_context(
+        test_path / "conf/demo/01_99_wf_test_wf_foreach_with_trigger.yml",
+        data="""
+        tmp-wf-foreach-nested-trigger-task:
+          type: Workflow
+          params:
+            item: int
+          jobs:
+            first-job:
+              stages:
+                - name: "Echo"
+                  id: hello
+                  echo: "Run trigger with item: ${{ params.item }}"
+
+        tmp-wf-foreach-nested-trigger:
+          type: Workflow
+          jobs:
+            first-job:
+              stages:
+                - name: "Start run for-each stage"
+                  id: foreach-stage
+                  foreach: [1, 2]
+                  stages:
+
+                    - name: "Start run for-each stage inside foreach"
+                      id: foreach-nested
+                      foreach: [3, 4]
+                      stages:
+                        - name: "Stage trigger"
+                          trigger: tmp-wf-foreach-nested-trigger-task
+                          params:
+                            item: ${{ item }}
+        """,
+    ):
+        workflow = Workflow.from_conf(
+            name="tmp-wf-foreach-nested-trigger",
+            extras={"test": "demo"},
+        )
+        stage: Stage = workflow.job("first-job").stage("foreach-stage")
+        rs = stage.set_outputs(stage.handler_execute({}).context, to={})
+        assert rs == {
+            "stages": {
+                "foreach-stage": {
+                    "outputs": {
+                        "items": [1, 2],
+                        "foreach": {
+                            1: {
+                                "item": 1,
+                                "stages": {
+                                    "foreach-nested": {
+                                        "outputs": {
+                                            "items": [3, 4],
+                                            "foreach": {
+                                                3: {
+                                                    "item": 3,
+                                                    "stages": {
+                                                        "8713259197": {
+                                                            "outputs": {
+                                                                "params": {
+                                                                    "item": 3
+                                                                },
+                                                                "jobs": {
+                                                                    "first-job": {
+                                                                        "stages": {
+                                                                            "hello": {
+                                                                                "outputs": {}
+                                                                            }
+                                                                        }
+                                                                    }
+                                                                },
+                                                            }
+                                                        }
+                                                    },
+                                                },
+                                                4: {
+                                                    "item": 4,
+                                                    "stages": {
+                                                        "8713259197": {
+                                                            "outputs": {
+                                                                "params": {
+                                                                    "item": 4
+                                                                },
+                                                                "jobs": {
+                                                                    "first-job": {
+                                                                        "stages": {
+                                                                            "hello": {
+                                                                                "outputs": {}
+                                                                            }
+                                                                        }
+                                                                    }
+                                                                },
+                                                            }
+                                                        }
+                                                    },
+                                                },
+                                            },
+                                        }
+                                    }
+                                },
+                            },
+                            2: {
+                                "item": 2,
+                                "stages": {
+                                    "foreach-nested": {
+                                        "outputs": {
+                                            "items": [3, 4],
+                                            "foreach": {
+                                                3: {
+                                                    "item": 3,
+                                                    "stages": {
+                                                        "8713259197": {
+                                                            "outputs": {
+                                                                "params": {
+                                                                    "item": 3
+                                                                },
+                                                                "jobs": {
+                                                                    "first-job": {
+                                                                        "stages": {
+                                                                            "hello": {
+                                                                                "outputs": {}
+                                                                            }
+                                                                        }
+                                                                    }
+                                                                },
+                                                            }
+                                                        }
+                                                    },
+                                                },
+                                                4: {
+                                                    "item": 4,
+                                                    "stages": {
+                                                        "8713259197": {
+                                                            "outputs": {
+                                                                "params": {
+                                                                    "item": 4
+                                                                },
+                                                                "jobs": {
+                                                                    "first-job": {
+                                                                        "stages": {
+                                                                            "hello": {
+                                                                                "outputs": {}
+                                                                            }
+                                                                        }
+                                                                    }
+                                                                },
+                                                            }
+                                                        }
+                                                    },
+                                                },
+                                            },
+                                        }
+                                    }
+                                },
+                            },
+                        },
+                    }
+                }
+            }
+        }
+
+
 def test_stage_exec_parallel(test_path):
     with dump_yaml_context(
         test_path / "conf/demo/01_99_wf_test_wf_parallel.yml",

{ddeutil_workflow-0.0.51 → ddeutil_workflow-0.0.52}/tests/test_workflow_exec.py

@@ -599,6 +599,10 @@ def test_workflow_exec_foreach(test_path):
                 - name: "Get Items before run foreach"
                   id: get-items
                   uses: tasks/get-items@demo
+                - name: "Create variable"
+                  id: create-variable
+                  run: |
+                    foo: str = "bar"
                 - name: "For-each item"
                   id: foreach-stage
                   foreach: ${{ stages.get-items.outputs.items }}
@@ -606,6 +610,7 @@ def test_workflow_exec_foreach(test_path):
                     - name: "Echo stage"
                       echo: |
                         Start run with item ${{ item }}
+                        Import variable ${{ stages.create-variable.outputs.foo }}
                     - name: "Final Echo"
                       if: ${{ item }} == 4
                       echo: |
@@ -614,7 +619,62 @@ def test_workflow_exec_foreach(test_path):
     ):
         workflow = Workflow.from_conf(name="tmp-wf-foreach")
         rs = workflow.execute(params={})
-        print(rs)
+        assert rs.status == SUCCESS
+        assert rs.context == {
+            "params": {},
+            "jobs": {
+                "transform": {
+                    "stages": {
+                        "get-items": {"outputs": {"items": [1, 2, 3, 4]}},
+                        "create-variable": {"outputs": {"foo": "bar"}},
+                        "foreach-stage": {
+                            "outputs": {
+                                "items": [1, 2, 3, 4],
+                                "foreach": {
+                                    1: {
+                                        "item": 1,
+                                        "stages": {
+                                            "2709471980": {"outputs": {}},
+                                            "9263488742": {
+                                                "outputs": {},
+                                                "skipped": True,
+                                            },
+                                        },
+                                    },
+                                    2: {
+                                        "item": 2,
+                                        "stages": {
+                                            "2709471980": {"outputs": {}},
+                                            "9263488742": {
+                                                "outputs": {},
+                                                "skipped": True,
+                                            },
+                                        },
+                                    },
+                                    3: {
+                                        "item": 3,
+                                        "stages": {
+                                            "2709471980": {"outputs": {}},
+                                            "9263488742": {
+                                                "outputs": {},
+                                                "skipped": True,
+                                            },
+                                        },
+                                    },
+                                    4: {
+                                        "item": 4,
+                                        "stages": {
+                                            "2709471980": {"outputs": {}},
+                                            "9263488742": {"outputs": {}},
+                                        },
+                                    },
+                                },
+                            }
+                        },
+                    }
+                }
+            },
+        }
 
 
 @mock.patch.object(Config, "stage_raise_error", False)

ddeutil_workflow-0.0.51/src/ddeutil/workflow/__about__.py

@@ -1 +0,0 @@
-__version__: str = "0.0.51"