ddeutil-workflow 0.0.52__py3-none-any.whl → 0.0.53__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1 +1 @@
-__version__: str = "0.0.52"
+__version__: str = "0.0.53"

ddeutil/workflow/job.py

@@ -555,6 +555,7 @@ class Job(BaseModel):
             )
 
         _id: str = self.id or job_id
+        output: DictData = copy.deepcopy(output)
         errors: DictData = (
             {"errors": output.pop("errors", {})} if "errors" in output else {}
         )

ddeutil/workflow/stages.py

@@ -42,7 +42,7 @@ from concurrent.futures import (
     wait,
 )
 from datetime import datetime
-from inspect import Parameter
+from inspect import Parameter, isclass, isfunction, ismodule
 from pathlib import Path
 from subprocess import CompletedProcess
 from textwrap import dedent
@@ -266,7 +266,7 @@ class BaseStage(BaseModel, ABC):
                 param2template(self.name, params=to, extras=self.extras)
             )
         )
-
+        output: DictData = output.copy()
         errors: DictData = (
             {"errors": output.pop("errors", {})} if "errors" in output else {}
         )
@@ -302,7 +302,7 @@ class BaseStage(BaseModel, ABC):
                 param2template(self.name, params=outputs, extras=self.extras)
             )
         )
-        return outputs.get("stages", {}).get(_id, {})
+        return outputs.get("stages", {}).get(_id, {}).get("outputs", {})
 
     def is_skipped(self, params: DictData | None = None) -> bool:
         """Return true if condition of this stage do not correct. This process
@@ -704,12 +704,11 @@ class PyStage(BaseStage):
 
         :rtype: Iterator[str]
         """
-        from inspect import isclass, ismodule
-
         for value in values:
 
             if (
                 value == "__annotations__"
+                or (value.startswith("__") and value.endswith("__"))
                 or ismodule(values[value])
                 or isclass(values[value])
             ):
@@ -727,11 +726,10 @@ class PyStage(BaseStage):
 
         :rtype: DictData
         """
+        output: DictData = output.copy()
         lc: DictData = output.pop("locals", {})
         gb: DictData = output.pop("globals", {})
-        super().set_outputs(
-            {k: lc[k] for k in self.filter_locals(lc)} | output, to=to
-        )
+        super().set_outputs(lc | output, to=to)
         to.update({k: gb[k] for k in to if k in gb})
         return to
 
@@ -762,26 +760,36 @@ class PyStage(BaseStage):
         lc: DictData = {}
         gb: DictData = (
             globals()
-            | params
             | param2template(self.vars, params, extras=self.extras)
             | {"result": result}
         )
 
-        # NOTE: Start exec the run statement.
         result.trace.info(f"[STAGE]: Py-Execute: {self.name}")
-        # result.trace.warning(
-        #     "[STAGE]: This stage allow use `eval` function, so, please "
-        #     "check your statement be safe before execute."
-        # )
-        #
+
         # WARNING: The exec build-in function is very dangerous. So, it
         #   should use the re module to validate exec-string before running.
         exec(
-            param2template(dedent(self.run), params, extras=self.extras), gb, lc
+            param2template(dedent(self.run), params, extras=self.extras),
+            gb,
+            lc,
         )
 
         return result.catch(
-            status=SUCCESS, context={"locals": lc, "globals": gb}
+            status=SUCCESS,
+            context={
+                "locals": {k: lc[k] for k in self.filter_locals(lc)},
+                "globals": {
+                    k: gb[k]
+                    for k in gb
+                    if (
+                        not k.startswith("__")
+                        and k != "annotations"
+                        and not ismodule(gb[k])
+                        and not isclass(gb[k])
+                        and not isfunction(gb[k])
+                    )
+                },
+            },
         )
 
 
@@ -841,8 +849,8 @@ class CallStage(BaseStage):
 
         :raise ValueError: If necessary arguments does not pass from the `args`
             field.
-        :raise TypeError: If the result from the caller function does not by
-            a `dict` type.
+        :raise TypeError: If the result from the caller function does not match
+            with a `dict` type.
 
         :rtype: Result
         """
@@ -1099,17 +1107,17 @@ class ParallelStage(BaseStage):  # pragma: no cov
         :rtype: DictData
         """
         result.trace.debug(f"... Execute branch: {branch!r}")
-        _params: DictData = copy.deepcopy(params)
-        _params.update({"branch": branch})
-        context: DictData = {"branch": branch, "stages": {}}
+        context: DictData = copy.deepcopy(params)
+        context.update({"branch": branch})
+        output: DictData = {"branch": branch, "stages": {}}
         for stage in self.parallel[branch]:
 
             if extras:
                 stage.extras = extras
 
-            if stage.is_skipped(params=_params):
+            if stage.is_skipped(params=context):
                 result.trace.info(f"... Skip stage: {stage.iden!r}")
-                stage.set_outputs(output={"skipped": True}, to=context)
+                stage.set_outputs(output={"skipped": True}, to=output)
                 continue
 
             if event and event.is_set():
@@ -1122,7 +1130,7 @@ class ParallelStage(BaseStage):  # pragma: no cov
                     parallel={
                         branch: {
                             "branch": branch,
-                            "stages": filter_func(context.pop("stages", {})),
+                            "stages": filter_func(output.pop("stages", {})),
                             "errors": StageException(error_msg).to_dict(),
                         }
                     },
@@ -1130,14 +1138,15 @@ class ParallelStage(BaseStage):  # pragma: no cov
 
             try:
                 rs: Result = stage.handler_execute(
-                    params=_params,
+                    params=context,
                     run_id=result.run_id,
                     parent_run_id=result.parent_run_id,
                     raise_error=True,
                     event=event,
                 )
-                stage.set_outputs(rs.context, to=context)
-            except StageException as e:  # pragma: no cov
+                stage.set_outputs(rs.context, to=output)
+                stage.set_outputs(stage.get_outputs(output), to=context)
+            except (StageException, UtilException) as e:  # pragma: no cov
                 result.trace.error(f"[STAGE]: {e.__class__.__name__}: {e}")
                 raise StageException(
                     f"Sub-Stage execution error: {e.__class__.__name__}: {e}"
@@ -1153,7 +1162,7 @@ class ParallelStage(BaseStage):  # pragma: no cov
                     parallel={
                         branch: {
                             "branch": branch,
-                            "stages": filter_func(context.pop("stages", {})),
+                            "stages": filter_func(output.pop("stages", {})),
                             "errors": StageException(error_msg).to_dict(),
                         },
                     },
@@ -1164,7 +1173,7 @@ class ParallelStage(BaseStage):  # pragma: no cov
             parallel={
                 branch: {
                     "branch": branch,
-                    "stages": filter_func(context.pop("stages", {})),
+                    "stages": filter_func(output.pop("stages", {})),
                 },
             },
         )
@@ -1294,18 +1303,18 @@ class ForEachStage(BaseStage):
 
         :rtype: Result
         """
-        result.trace.debug(f"... Execute item: {item!r}")
-        _params: DictData = copy.deepcopy(params)
-        _params.update({"item": item})
-        context: DictData = {"item": item, "stages": {}}
+        result.trace.debug(f"[STAGE]: Execute item: {item!r}")
+        context: DictData = copy.deepcopy(params)
+        context.update({"item": item})
+        output: DictData = {"item": item, "stages": {}}
         for stage in self.stages:
 
             if self.extras:
                 stage.extras = self.extras
 
-            if stage.is_skipped(params=_params):
+            if stage.is_skipped(params=context):
                 result.trace.info(f"... Skip stage: {stage.iden!r}")
-                stage.set_outputs(output={"skipped": True}, to=context)
+                stage.set_outputs(output={"skipped": True}, to=output)
                 continue
 
             if event and event.is_set():  # pragma: no cov
@@ -1318,7 +1327,7 @@ class ForEachStage(BaseStage):
                     foreach={
                         item: {
                             "item": item,
-                            "stages": filter_func(context.pop("stages", {})),
+                            "stages": filter_func(output.pop("stages", {})),
                             "errors": StageException(error_msg).to_dict(),
                         }
                     },
@@ -1326,13 +1335,14 @@ class ForEachStage(BaseStage):
 
             try:
                 rs: Result = stage.handler_execute(
-                    params=_params,
+                    params=context,
                     run_id=result.run_id,
                     parent_run_id=result.parent_run_id,
                     raise_error=True,
                     event=event,
                 )
-                stage.set_outputs(rs.context, to=context)
+                stage.set_outputs(rs.context, to=output)
+                stage.set_outputs(stage.get_outputs(output), to=context)
             except (StageException, UtilException) as e:
                 result.trace.error(f"[STAGE]: {e.__class__.__name__}: {e}")
                 raise StageException(
@@ -1349,18 +1359,17 @@ class ForEachStage(BaseStage):
                     foreach={
                         item: {
                             "item": item,
-                            "stages": filter_func(context.pop("stages", {})),
+                            "stages": filter_func(output.pop("stages", {})),
                             "errors": StageException(error_msg).to_dict(),
                         },
                     },
                 )
-
         return result.catch(
             status=SUCCESS,
             foreach={
                 item: {
                     "item": item,
-                    "stages": filter_func(context.pop("stages", {})),
+                    "stages": filter_func(output.pop("stages", {})),
                 },
             },
         )
@@ -1515,18 +1524,18 @@ class UntilStage(BaseStage):  # pragma: no cov
         :rtype: tuple[Result, T]
         """
         result.trace.debug(f"... Execute until item: {item!r}")
-        _params: DictData = copy.deepcopy(params)
-        _params.update({"item": item})
-        context: DictData = {"loop": loop, "item": item, "stages": {}}
+        context: DictData = copy.deepcopy(params)
+        context.update({"item": item})
+        output: DictData = {"loop": loop, "item": item, "stages": {}}
         next_item: T = None
         for stage in self.stages:
 
             if self.extras:
                 stage.extras = self.extras
 
-            if stage.is_skipped(params=_params):
+            if stage.is_skipped(params=context):
                 result.trace.info(f"... Skip stage: {stage.iden!r}")
-                stage.set_outputs(output={"skipped": True}, to=context)
+                stage.set_outputs(output={"skipped": True}, to=output)
                 continue
 
             if event and event.is_set():
@@ -1541,9 +1550,7 @@ class UntilStage(BaseStage):  # pragma: no cov
                             loop: {
                                 "loop": loop,
                                 "item": item,
-                                "stages": filter_func(
-                                    context.pop("stages", {})
-                                ),
+                                "stages": filter_func(output.pop("stages", {})),
                                 "errors": StageException(error_msg).to_dict(),
                             }
                         },
@@ -1553,17 +1560,18 @@ class UntilStage(BaseStage):  # pragma: no cov
 
             try:
                 rs: Result = stage.handler_execute(
-                    params=_params,
+                    params=context,
                     run_id=result.run_id,
                     parent_run_id=result.parent_run_id,
                     raise_error=True,
                     event=event,
                 )
-                stage.set_outputs(rs.context, to=context)
-                if "item" in (
-                    outputs := stage.get_outputs(context).get("outputs", {})
-                ):
-                    next_item = outputs["item"]
+                stage.set_outputs(rs.context, to=output)
+
+                if "item" in (_output := stage.get_outputs(output)):
+                    next_item = _output["item"]
+
+                stage.set_outputs(_output, to=context)
             except (StageException, UtilException) as e:
                 result.trace.error(f"[STAGE]: {e.__class__.__name__}: {e}")
                 raise StageException(
@@ -1577,7 +1585,7 @@ class UntilStage(BaseStage):  # pragma: no cov
                     loop: {
                         "loop": loop,
                         "item": item,
-                        "stages": filter_func(context.pop("stages", {})),
+                        "stages": filter_func(output.pop("stages", {})),
                     }
                 },
             ),
@@ -1672,7 +1680,9 @@ class Match(BaseModel):
     """Match model for the Case Stage."""
 
     case: Union[str, int] = Field(description="A match case.")
-    stage: Stage = Field(description="A stage to execution for this case.")
+    stages: list[Stage] = Field(
+        description="A list of stage to execution for this case."
+    )
 
 
 class CaseStage(BaseStage):
@@ -1685,24 +1695,21 @@ class CaseStage(BaseStage):
         ...     "match": [
         ...         {
         ...             "case": "1",
-        ...             "stage": {
-        ...                 "name": "Stage case 1",
-        ...                 "eche": "Hello case 1",
-        ...             },
-        ...         },
-        ...         {
-        ...             "case": "2",
-        ...             "stage": {
-        ...                 "name": "Stage case 2",
-        ...                 "eche": "Hello case 2",
-        ...             },
+        ...             "stages": [
+        ...                 {
+        ...                     "name": "Stage case 1",
+        ...                     "eche": "Hello case 1",
+        ...                 },
+        ...             ],
         ...         },
         ...         {
         ...             "case": "_",
-        ...             "stage": {
-        ...                 "name": "Stage else",
-        ...                 "eche": "Hello case else",
-        ...             },
+        ...             "stages": [
+        ...                 {
+        ...                     "name": "Stage else",
+        ...                     "eche": "Hello case else",
+        ...                 },
+        ...             ],
         ...         },
         ...     ],
         ... }
@@ -1722,6 +1729,98 @@ class CaseStage(BaseStage):
         alias="skip-not-match",
     )
 
+    def execute_case(
+        self,
+        case: str,
+        stages: list[Stage],
+        params: DictData,
+        result: Result,
+        *,
+        event: Event | None = None,
+    ) -> Result:
+        """Execute case.
+
+        :param case: (str) A case that want to execution.
+        :param stages: (list[Stage]) A list of stage.
+        :param params: (DictData) A parameter that want to pass to stage
+            execution.
+        :param result: (Result) A result object for keeping context and status
+            data.
+        :param event: (Event) An event manager that use to track parent execute
+            was not force stopped.
+
+        :rtype: Result
+        """
+        context: DictData = copy.deepcopy(params)
+        context.update({"case": case})
+        output: DictData = {"case": case, "stages": {}}
+
+        for stage in stages:
+
+            if self.extras:
+                stage.extras = self.extras
+
+            if stage.is_skipped(params=context):
+                result.trace.info(f"... Skip stage: {stage.iden!r}")
+                stage.set_outputs(output={"skipped": True}, to=output)
+                continue
+
+            if event and event.is_set():  # pragma: no cov
+                error_msg: str = (
+                    "Case-Stage was canceled from event that had set before "
+                    "stage case execution."
+                )
+                return result.catch(
+                    status=CANCEL,
+                    context={
+                        "case": case,
+                        "stages": filter_func(output.pop("stages", {})),
+                        "errors": StageException(error_msg).to_dict(),
+                    },
+                )
+
+            try:
+                rs: Result = stage.handler_execute(
+                    params=context,
+                    run_id=result.run_id,
+                    parent_run_id=result.parent_run_id,
+                    raise_error=True,
+                    event=event,
+                )
+                stage.set_outputs(rs.context, to=output)
+                stage.set_outputs(stage.get_outputs(output), to=context)
+            except (StageException, UtilException) as e:  # pragma: no cov
+                result.trace.error(f"[STAGE]: {e.__class__.__name__}: {e}")
+                return result.catch(
+                    status=FAILED,
+                    context={
+                        "case": case,
+                        "stages": filter_func(output.pop("stages", {})),
+                        "errors": e.to_dict(),
+                    },
+                )
+
+            if rs.status == FAILED:
+                error_msg: str = (
+                    f"Case-Stage was break because it has a sub stage, "
+                    f"{stage.iden}, failed without raise error."
+                )
+                return result.catch(
+                    status=FAILED,
+                    context={
+                        "case": case,
+                        "stages": filter_func(output.pop("stages", {})),
+                        "errors": StageException(error_msg).to_dict(),
+                    },
+                )
+        return result.catch(
+            status=SUCCESS,
+            context={
+                "case": case,
+                "stages": filter_func(output.pop("stages", {})),
+            },
+        )
+
     def execute(
         self,
         params: DictData,
@@ -1751,17 +1850,17 @@ class CaseStage(BaseStage):
 
         result.trace.info(f"[STAGE]: Case-Execute: {_case!r}.")
         _else: Optional[Match] = None
-        stage: Optional[Stage] = None
+        stages: Optional[list[Stage]] = None
         for match in self.match:
             if (c := match.case) == "_":
                 _else: Match = match
                 continue
 
             _condition: str = param2template(c, params, extras=self.extras)
-            if stage is None and _case == _condition:
-                stage: Stage = match.stage
+            if stages is None and _case == _condition:
+                stages: list[Stage] = match.stages
 
-        if stage is None:
+        if stages is None:
             if _else is None:
                 if not self.skip_not_match:
                     raise StageException(
@@ -1779,10 +1878,8 @@ class CaseStage(BaseStage):
                     status=CANCEL,
                     context={"errors": StageException(error_msg).to_dict()},
                 )
-            stage: Stage = _else.stage
-
-        if self.extras:
-            stage.extras = self.extras
+            _case: str = "_"
+            stages: list[Stage] = _else.stages
 
         if event and event.is_set():  # pragma: no cov
             return result.catch(
@@ -1795,19 +1892,9 @@ class CaseStage(BaseStage):
                 },
             )
 
-        try:
-            return result.catch(
-                status=SUCCESS,
-                context=stage.handler_execute(
-                    params=params,
-                    run_id=result.run_id,
-                    parent_run_id=result.parent_run_id,
-                    event=event,
-                ).context,
-            )
-        except StageException as e:  # pragma: no cov
-            result.trace.error(f"[STAGE]: {e.__class__.__name__}:" f"\n\t{e}")
-            return result.catch(status=FAILED, context={"errors": e.to_dict()})
+        return self.execute_case(
+            case=_case, stages=stages, params=params, result=result, event=event
+        )
 
 
 class RaiseStage(BaseStage):  # pragma: no cov

{ddeutil_workflow-0.0.52.dist-info → ddeutil_workflow-0.0.53.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddeutil-workflow
-Version: 0.0.52
+Version: 0.0.53
 Summary: Lightweight workflow orchestration
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT

{ddeutil_workflow-0.0.52.dist-info → ddeutil_workflow-0.0.53.dist-info}/RECORD

@@ -1,4 +1,4 @@
-ddeutil/workflow/__about__.py,sha256=MtoJJkgninIGWDAMMmKrIvqN_o2rzywSeEdJGILbStY,28
+ddeutil/workflow/__about__.py,sha256=fOQi-49Q8-qLVO7us5t2StkrubZvI2LQkyYSQi-3P88,28
 ddeutil/workflow/__cron.py,sha256=h8rLeIUAAEB2SdZ4Jhch7LU1Yl3bbJ-iNNJ3tQ0eYVM,28095
 ddeutil/workflow/__init__.py,sha256=noE8LNRcgq32m9OnIFcQqh0P7PXWdp-SGmvBCYIXgf4,1338
 ddeutil/workflow/__main__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -6,13 +6,13 @@ ddeutil/workflow/__types.py,sha256=8jBdbfb3aZSetjz0mvNrpGHwwxJff7mK8_4v41cLqlc,4
 ddeutil/workflow/conf.py,sha256=80rgmJKFU7BlH5xTLnghGzGhE8C6LFAQykd9mjHSjo8,12528
 ddeutil/workflow/cron.py,sha256=WS2MInn0Sp5DKlZDZH5VFZ5AA0Q3_AnBnYEU4lZSv4I,9779
 ddeutil/workflow/exceptions.py,sha256=r4Jrf9qtVPALU4wh4bnb_OYqC-StqSQJEmFC-_QK934,1408
-ddeutil/workflow/job.py,sha256=l2YpbUQoXYQ46S7Ne3Ttefc_jKBy14q3_PKf3k_cmNU,35213
+ddeutil/workflow/job.py,sha256=Z1XP_9pj-RY64z3G4LYX-MppS99zQns9wtZy7zHuWbE,35262
 ddeutil/workflow/logs.py,sha256=rsoBrUGQrooou18fg2yvPsB8NOaXnUA5ThQpBr_WVMg,26598
 ddeutil/workflow/params.py,sha256=FKY4Oo1Ze4QZKRfAk7rqKsi44YaJQAbqAtXM6vlO2hI,11392
 ddeutil/workflow/result.py,sha256=27nPQq9CETLCVczv4vvFEF9w2TllHZ_ROfyDoLFxRWM,5647
 ddeutil/workflow/reusables.py,sha256=iXcS7Gg-71qVX4ln0ILTDx03cTtUnj_rNoXHTVdVrxc,17636
 ddeutil/workflow/scheduler.py,sha256=4G5AogkmnsTKe7jKYSfU35qjubR82WQ8CLtEe9kqPTE,28304
-ddeutil/workflow/stages.py,sha256=R6yVp5-9xw6fslY7oE2pNzpMmgHiBN-2AVG2hCc5-QI,69241
+ddeutil/workflow/stages.py,sha256=ZsGh8Wd-NqdAZC5cyJ6wXuF-UHqoCcFFedXvyHssSqc,72473
 ddeutil/workflow/utils.py,sha256=zbVttaMFMRLuuBJdSJf7D9qtz8bOnQIBq-rHI3Eqy4M,7821
 ddeutil/workflow/workflow.py,sha256=2ZBNW3-vcP8bpKrK184wSCukq3wpT6G0z25Su5bapR0,50832
 ddeutil/workflow/api/__init__.py,sha256=F53NMBWtb9IKaDWkPU5KvybGGfKAcbehgn6TLBwHuuM,21
@@ -24,8 +24,8 @@ ddeutil/workflow/api/routes/job.py,sha256=oPwBVP0Mxwxv-bGPlfmxQQ9PcVl0ev9HoPzndp
 ddeutil/workflow/api/routes/logs.py,sha256=U6vOni3wd-ZTOwd3yVdSOpgyRmNdcgfngU5KlLM3Cww,5383
 ddeutil/workflow/api/routes/schedules.py,sha256=EgUjyRGhsm6UNaMj5luh6TcY6l571sCHcla-BL1iOfY,4829
 ddeutil/workflow/api/routes/workflows.py,sha256=JcDOrn1deK8ztFRcMTNATQejG6KMA7JxZLVc4QeBsP4,4527
-ddeutil_workflow-0.0.52.dist-info/licenses/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
-ddeutil_workflow-0.0.52.dist-info/METADATA,sha256=DGXGAp4BlgOfkP-CWnMFkl8_ZwvoAtMZqB5uUpcoEBs,19425
-ddeutil_workflow-0.0.52.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-ddeutil_workflow-0.0.52.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
-ddeutil_workflow-0.0.52.dist-info/RECORD,,
+ddeutil_workflow-0.0.53.dist-info/licenses/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
+ddeutil_workflow-0.0.53.dist-info/METADATA,sha256=xNgYIlFQvS9VsF0agSPsYbJWin_s9c_llkRFnEUxyC0,19425
+ddeutil_workflow-0.0.53.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+ddeutil_workflow-0.0.53.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
+ddeutil_workflow-0.0.53.dist-info/RECORD,,