ddeutil-workflow 0.0.43__py3-none-any.whl → 0.0.44__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1 +1 @@
-__version__: str = "0.0.43"
+__version__: str = "0.0.44"

ddeutil/workflow/job.py

@@ -38,6 +38,7 @@ from .exceptions import (
     JobException,
     StageException,
     UtilException,
+    to_dict,
 )
 from .result import FAILED, SKIP, SUCCESS, WAIT, Result, Status
 from .reusables import has_template, param2template
@@ -415,6 +416,7 @@ class Job(BaseModel):
         need_exist: dict[str, Any] = {
             need: jobs[need] for need in self.needs if need in jobs
         }
+
         if len(need_exist) != len(self.needs):
             return WAIT
         elif all("skipped" in need_exist[job] for job in need_exist):
@@ -630,19 +632,6 @@ def local_execute_strategy(
         result: Result = Result(run_id=gen_id(job.id or "not-set", unique=True))
 
     strategy_id: str = gen_id(strategy)
-
-    # PARAGRAPH:
-    #
-    #       Create strategy execution context and update a matrix and copied
-    #   of params. So, the context value will have structure like;
-    #
-    #   {
-    #       "params": { ... },      <== Current input params
-    #       "jobs": { ... },        <== Current input params
-    #       "matrix": { ... }       <== Current strategy value
-    #       "stages": { ... }       <== Catching stage outputs
-    #   }
-    #
     context: DictData = copy.deepcopy(params)
     context.update({"matrix": strategy, "stages": {}})
 
@@ -650,7 +639,6 @@ def local_execute_strategy(
         result.trace.info(f"[JOB]: Execute Strategy ID: {strategy_id}")
         result.trace.info(f"[JOB]: ... Matrix: {strategy_id}")
 
-    # IMPORTANT: The stage execution only run sequentially one-by-one.
     for stage in job.stages:
 
         if stage.is_skipped(params=context):
@@ -674,34 +662,30 @@ def local_execute_strategy(
                 },
             )
 
-        # PARAGRAPH:
-        #
-        #       This step will add the stage result to `stages` key in that
-        #   stage id. It will have structure like;
-        #
-        #   {
-        #       "params": { ... },
-        #       "jobs": { ... },
-        #       "matrix": { ... },
-        #       "stages": { { "stage-id-01": { "outputs": { ... } } }, ... }
-        #   }
-        #
-        # IMPORTANT:
-        #   This execution change all stage running IDs to the current job
-        #   running ID, but it still trac log to the same parent running ID
-        #   (with passing `run_id` and `parent_run_id` to the stage
-        #   execution arguments).
-        #
         try:
-            stage.set_outputs(
-                stage.handler_execute(
-                    params=context,
-                    run_id=result.run_id,
-                    parent_run_id=result.parent_run_id,
-                    event=event,
-                ).context,
-                to=context,
+            rs: Result = stage.handler_execute(
+                params=context,
+                run_id=result.run_id,
+                parent_run_id=result.parent_run_id,
+                event=event,
             )
+            stage.set_outputs(rs.context, to=context)
+            if rs.status == FAILED:
+                error_msg: str = (
+                    f"Job strategy was break because it has a stage, "
+                    f"{stage.iden}, failed without raise error."
+                )
+                return result.catch(
+                    status=FAILED,
+                    context={
+                        strategy_id: {
+                            "matrix": strategy,
+                            "stages": context.pop("stages", {}),
+                            "errors": JobException(error_msg).to_dict(),
+                        },
+                    },
+                )
+
         except (StageException, UtilException) as err:
             result.trace.error(f"[JOB]: {err.__class__.__name__}: {err}")
             do_raise: bool = dynamic(
@@ -746,8 +730,8 @@ def local_execute(
     raise_error: bool | None = None,
 ) -> Result:
     """Local job execution with passing dynamic parameters from the workflow
-    execution. It will generate matrix values at the first step and run
-    multithread on this metrics to the `stages` field of this job.
+    execution or itself execution. It will generate matrix values at the first
+    step and run multithread on this metrics to the `stages` field of this job.
 
         This method does not raise any JobException if it runs with
     multi-threading strategy.
@@ -798,7 +782,7 @@ def local_execute(
                 raise_error=raise_error,
             )
 
-        return result.catch(status=SUCCESS)
+        return result.catch(status=result.status)
 
     fail_fast_flag: bool = job.strategy.fail_fast
     ls: str = "Fail-Fast" if fail_fast_flag else "All-Completed"
@@ -818,8 +802,6 @@ def local_execute(
             },
         )
 
-    # IMPORTANT: Start running strategy execution by multithreading because
-    #   it will run by strategy values without waiting previous execution.
     with ThreadPoolExecutor(
         max_workers=job.strategy.max_parallel,
         thread_name_prefix="job_strategy_exec_",
@@ -885,6 +867,22 @@ def self_hosted_execute(
     event: Event | None = None,
     raise_error: bool | None = None,
 ) -> Result:  # pragma: no cov
+    """Self-Hosted job execution with passing dynamic parameters from the
+    workflow execution or itself execution. It will make request to the
+    self-hosted host url.
+
+    :param job: (Job) A job model that want to execute.
+    :param params: (DictData) An input parameters that use on job execution.
+    :param run_id: (str) A job running ID for this execution.
+    :param parent_run_id: (str) A parent workflow running ID for this release.
+    :param result: (Result) A result object for keeping context and status
+        data.
+    :param event: (Event) An event manager that pass to the PoolThreadExecutor.
+    :param raise_error: (bool) A flag that all this method raise error to the
+        strategy execution.
+
+    :rtype: Result
+    """
     result: Result = Result.construct_with_rs_or_id(
         result,
         run_id=run_id,
@@ -893,14 +891,31 @@ def self_hosted_execute(
     )
 
     if event and event.is_set():
-        return result.catch(status=FAILED)
+        return result.catch(
+            status=FAILED,
+            context={
+                "errors": JobException(
+                    "Job self-hosted execution was canceled from event that "
+                    "had set before start execution."
+                ).to_dict()
+            },
+        )
 
     import requests
 
-    resp = requests.post(
-        job.runs_on.args.host,
-        data={"job": job.model_dump(), "params": params},
-    )
+    try:
+        resp = requests.post(
+            job.runs_on.args.host,
+            headers={"Auth": f"Barer {job.runs_on.args.token}"},
+            data={
+                "job": job.model_dump(),
+                "params": params,
+                "result": result.__dict__,
+                "raise_error": raise_error,
+            },
+        )
+    except requests.exceptions.RequestException as e:
+        return result.catch(status=FAILED, context={"errors": to_dict(e)})
 
     if resp.status_code != 200:
         do_raise: bool = dynamic(

ddeutil/workflow/stages.py

@@ -223,10 +223,11 @@ class BaseStage(BaseModel, ABC):
                 ) from e
 
             errors: DictData = {"errors": to_dict(e)}
-            if to is not None:
-                return self.set_outputs(errors, to=to)
-
-            return result.catch(status=FAILED, context=errors)
+            return (
+                self.set_outputs(errors, to=to)
+                if to is not None
+                else result.catch(status=FAILED, context=errors)
+            )
 
     def set_outputs(self, output: DictData, to: DictData) -> DictData:
         """Set an outputs from execution process to the received context. The
@@ -326,7 +327,10 @@ class BaseAsyncStage(BaseStage):
         *,
         result: Result | None = None,
         event: Event | None = None,
-    ) -> Result: ...
+    ) -> Result:
+        raise NotImplementedError(
+            "Async Stage should implement `execute` method."
+        )
 
     @abstractmethod
     async def axecute(

ddeutil/workflow/workflow.py

@@ -981,11 +981,11 @@ class Workflow(BaseModel):
                     ).context,
                     to=params,
                 )
-        except JobException as err:
-            result.trace.error(f"[WORKFLOW]: {err.__class__.__name__}: {err}")
+        except JobException as e:
+            result.trace.error(f"[WORKFLOW]: {e.__class__.__name__}: {e}")
             if raise_error:
                 raise WorkflowException(
-                    f"Get job execution error {job_id}: JobException: {err}"
+                    f"Get job execution error {job_id}: JobException: {e}"
                 ) from None
             raise NotImplementedError(
                 "Handle error from the job execution does not support yet."
@@ -1042,16 +1042,12 @@ class Workflow(BaseModel):
         )
 
         result.trace.info(f"[WORKFLOW]: Start Execute: {self.name!r} ...")
-
-        # NOTE: It should not do anything if it does not have job.
         if not self.jobs:
             result.trace.warning(
                 f"[WORKFLOW]: {self.name!r} does not have any jobs"
             )
             return result.catch(status=SUCCESS, context=params)
 
-        # NOTE: Create a job queue that keep the job that want to run after
-        #   its dependency condition.
         jq: Queue = Queue()
         for job_id in self.jobs:
             jq.put(job_id)
@@ -1090,9 +1086,9 @@ class Workflow(BaseModel):
                     timeout=timeout,
                     event=event,
                 )
-        except WorkflowException as err:
-            status = FAILED
-            context.update({"errors": err.to_dict()})
+        except WorkflowException as e:
+            status: Status = FAILED
+            context.update({"errors": e.to_dict()})
 
         return result.catch(status=status, context=context)
 
@@ -1149,7 +1145,8 @@ class Workflow(BaseModel):
                     continue
                 elif check == FAILED:  # pragma: no cov
                     raise WorkflowException(
-                        "Check job trigger rule was failed."
+                        f"Validate job trigger rule was failed with "
+                        f"{job.trigger_rule.value!r}."
                     )
                 elif check == SKIP:  # pragma: no cov
                     result.trace.info(f"[JOB]: Skip job: {job_id!r}")
@@ -1157,16 +1154,6 @@ class Workflow(BaseModel):
                     job_queue.task_done()
                     continue
 
-                # NOTE: Start workflow job execution with deep copy context data
-                #   before release.
-                #
-                #   Context:
-                #   ---
-                #   {
-                #       'params': <input-params>,
-                #       'jobs': { <job's-id>: ... },
-                #   }
-                #
                 futures.append(
                     executor.submit(
                         self.execute_job,
@@ -1177,19 +1164,14 @@ class Workflow(BaseModel):
                     ),
                 )
 
-                # NOTE: Mark this job queue done.
                 job_queue.task_done()
 
             if not_timeout_flag:
-
-                # NOTE: Wait for all items to finish processing by `task_done()`
-                #   method.
                 job_queue.join()
-
                 for future in as_completed(futures, timeout=thread_timeout):
-                    if err := future.exception():
-                        result.trace.error(f"[WORKFLOW]: {err}")
-                        raise WorkflowException(str(err))
+                    if e := future.exception():
+                        result.trace.error(f"[WORKFLOW]: {e}")
+                        raise WorkflowException(str(e))
 
                     future.result()
 
@@ -1235,78 +1217,74 @@ class Workflow(BaseModel):
             "max_job_exec_timeout", f=timeout, extras=self.extras
         )
         event: Event = event or Event()
-        future: Future | None = None
         result.trace.debug(f"[WORKFLOW]: Run {self.name!r} with non-threading.")
-
-        executor = ThreadPoolExecutor(
+        with ThreadPoolExecutor(
             max_workers=1,
             thread_name_prefix="wf_exec_non_threading_",
-        )
+        ) as executor:
+            future: Future | None = None
 
-        while not job_queue.empty() and (
-            not_timeout_flag := ((time.monotonic() - ts) < timeout)
-        ):
-            job_id: str = job_queue.get()
-            job: Job = self.jobs[job_id]
+            while not job_queue.empty() and (
+                not_timeout_flag := ((time.monotonic() - ts) < timeout)
+            ):
+                job_id: str = job_queue.get()
+                job: Job = self.jobs[job_id]
 
-            if (check := job.check_needs(context["jobs"])) == WAIT:
-                job_queue.task_done()
-                job_queue.put(job_id)
-                time.sleep(0.075)
-                continue
-            elif check == FAILED:  # pragma: no cov
-                raise WorkflowException("Check job trigger rule was failed.")
-            elif check == SKIP:  # pragma: no cov
-                result.trace.info(f"[JOB]: Skip job: {job_id!r}")
-                job.set_outputs({"SKIP": {"skipped": True}}, to=context)
-                job_queue.task_done()
-                continue
+                if (check := job.check_needs(context["jobs"])) == WAIT:
+                    job_queue.task_done()
+                    job_queue.put(job_id)
+                    time.sleep(0.075)
+                    continue
+                elif check == FAILED:
+                    raise WorkflowException(
+                        f"Validate job trigger rule was failed with "
+                        f"{job.trigger_rule.value!r}."
+                    )
+                elif check == SKIP:  # pragma: no cov
+                    result.trace.info(f"[JOB]: Skip job: {job_id!r}")
+                    job.set_outputs({"SKIP": {"skipped": True}}, to=context)
+                    job_queue.task_done()
+                    continue
 
-            # NOTE: Start workflow job execution with deep copy context data
-            #   before release. This job execution process will run until
-            #   done before checking all execution timeout or not.
-            #
-            #   {
-            #       'params': <input-params>,
-            #       'jobs': {},
-            #   }
-            if future is None:
-                future: Future = executor.submit(
-                    self.execute_job,
-                    job_id=job_id,
-                    params=context,
-                    result=result,
-                    event=event,
-                )
-                result.trace.debug(f"[WORKFLOW]: Make future: {future}")
-                time.sleep(0.025)
-            elif future.done():
-                if err := future.exception():
-                    result.trace.error(f"[WORKFLOW]: {err}")
-                    raise WorkflowException(str(err))
-
-                future = None
-                job_queue.put(job_id)
-            elif future.running():
-                time.sleep(0.075)
-                job_queue.put(job_id)
-            else:  # pragma: no cov
-                job_queue.put(job_id)
-                result.trace.debug(
-                    f"Execution non-threading does not handle case: {future} "
-                    f"that not running."
-                )
+                if future is None:
+                    future: Future = executor.submit(
+                        self.execute_job,
+                        job_id=job_id,
+                        params=context,
+                        result=result,
+                        event=event,
+                    )
+                    time.sleep(0.025)
+                elif future.done():
+                    if e := future.exception():
+                        result.trace.error(f"[WORKFLOW]: {e}")
+                        raise WorkflowException(str(e))
+
+                    future = None
+                    job_queue.put(job_id)
+                elif future.running():
+                    time.sleep(0.075)
+                    job_queue.put(job_id)
+                else:  # pragma: no cov
+                    job_queue.put(job_id)
+                    result.trace.debug(
+                        f"Execution non-threading does not handle case: {future} "
+                        f"that not running."
+                    )
 
-            job_queue.task_done()
+                job_queue.task_done()
+
+            if not_timeout_flag:
+                job_queue.join()
+                return context
 
-        if not_timeout_flag:
-            job_queue.join()
-            executor.shutdown()
-            return context
+            result.trace.error(
+                f"[WORKFLOW]: Execution: {self.name!r} was timeout."
+            )
+            event.set()
+            if future:
+                future.cancel()
 
-        result.trace.error(f"[WORKFLOW]: Execution: {self.name!r} was timeout.")
-        event.set()
-        executor.shutdown()
         raise WorkflowException(f"Execution: {self.name!r} was timeout.")
 
 

{ddeutil_workflow-0.0.43.dist-info → ddeutil_workflow-0.0.44.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddeutil-workflow
-Version: 0.0.43
+Version: 0.0.44
 Summary: Lightweight workflow orchestration
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT

{ddeutil_workflow-0.0.43.dist-info → ddeutil_workflow-0.0.44.dist-info}/RECORD

@@ -1,19 +1,19 @@
-ddeutil/workflow/__about__.py,sha256=feByUkEJfWx0d1Lb9oGvd2f3clHHuGXDT7T7Ki5zoEA,28
+ddeutil/workflow/__about__.py,sha256=babneIKueqHV5Z7O62u5nVOWXWHGZSyS5egKApTAA1I,28
 ddeutil/workflow/__cron.py,sha256=h8rLeIUAAEB2SdZ4Jhch7LU1Yl3bbJ-iNNJ3tQ0eYVM,28095
 ddeutil/workflow/__init__.py,sha256=m7ZTCuUOarcTKJuXOyuaXd5WTIO7NTkqCeCrNX3d5i8,1943
 ddeutil/workflow/__types.py,sha256=8jBdbfb3aZSetjz0mvNrpGHwwxJff7mK8_4v41cLqlc,4316
 ddeutil/workflow/conf.py,sha256=lDzWiVSNlNAhTzxbNIhIbQAIF1ggbmetAp0yn2fgnsc,12385
 ddeutil/workflow/cron.py,sha256=80SijzMdDOBxTWRsiF-Fmuz7Ym7leY0XT2lzRAPGdXc,8781
 ddeutil/workflow/exceptions.py,sha256=uLNxzav3HRcr4vaZnvbUIF_eTR6UXXZNaxroMWFOUL4,1418
-ddeutil/workflow/job.py,sha256=6CqLb1F_z3mHU_rOVRY7Z2V2B1tY3p7FZIJgb8DM1YE,29969
+ddeutil/workflow/job.py,sha256=NgEPgMAUL2mqDctLpdoEVx4g0ZsTj4RmkQluCh_ZUdM,30614
 ddeutil/workflow/logs.py,sha256=RkM5o_JPoWhFY7NrbYAARZQWjLC62YB_FYzTTcyDp8U,19816
 ddeutil/workflow/params.py,sha256=xCtFEh0-G-G-f8y_SXxyf31bU6Ox5p5Z-WbBFXrjy8M,9960
 ddeutil/workflow/result.py,sha256=iwkUzOubxhLCuO-ngWEWL6t-CpYBpINIIO_ubg4kz14,4701
 ddeutil/workflow/reusables.py,sha256=AtZO83HDFu1uK_azUinv5d8jsA36f2i3n_tqMrolbvc,17529
 ddeutil/workflow/scheduler.py,sha256=_MDsEHbBVOeF-381U8DfIMDyca_nG3XNXmgX4229_EU,27437
-ddeutil/workflow/stages.py,sha256=RMyOU9KqPhS-49kQduT6-iysGJZFwX0Cbgo9o8kzIAY,47304
+ddeutil/workflow/stages.py,sha256=cvSNt4IjbE4O9llCIL2bojnl4W50GHNa1ANQ4oGefUo,47438
 ddeutil/workflow/utils.py,sha256=sblje9qOtejCHVt8EVrbC0KY98vKqvxccaR5HIkRiTA,7363
-ddeutil/workflow/workflow.py,sha256=LPZzGNf55Tca0XXQZytTiyWbocC1T47cB_XsWpiXdyc,50482
+ddeutil/workflow/workflow.py,sha256=NxuSQqoNvDTKiOVLambTBD847BJW-748dm7YgfjGPoA,49731
 ddeutil/workflow/api/__init__.py,sha256=F53NMBWtb9IKaDWkPU5KvybGGfKAcbehgn6TLBwHuuM,21
 ddeutil/workflow/api/api.py,sha256=b-bMg0aRsEqt8Qb2hNUtamEt2Fq2CgNotF2oXSAdDu8,5226
 ddeutil/workflow/api/log.py,sha256=NMTnOnsBrDB5129329xF2myLdrb-z9k1MQrmrP7qXJw,1818
@@ -23,8 +23,8 @@ ddeutil/workflow/api/routes/job.py,sha256=YVta083i8vU8-o4WdKFwDpfdC9vN1dZ6goZSmN
 ddeutil/workflow/api/routes/logs.py,sha256=TeRDrEelbKS2Hu_EovgLh0bOdmSv9mfnrIZsrE7uPD4,5353
 ddeutil/workflow/api/routes/schedules.py,sha256=rUWBm5RgLS1PNBHSWwWXJ0l-c5mYWfl9os0BA9_OTEw,4810
 ddeutil/workflow/api/routes/workflows.py,sha256=ctgQGxXfpIV6bHFDM9IQ1_qaQHT6n5-HjJ1-D4GKWpc,4527
-ddeutil_workflow-0.0.43.dist-info/licenses/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
-ddeutil_workflow-0.0.43.dist-info/METADATA,sha256=eOwkhUh-w_bSW0LOlOCielBQTg0mvf2gEkkNMlbwIU4,19134
-ddeutil_workflow-0.0.43.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-ddeutil_workflow-0.0.43.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
-ddeutil_workflow-0.0.43.dist-info/RECORD,,
+ddeutil_workflow-0.0.44.dist-info/licenses/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
+ddeutil_workflow-0.0.44.dist-info/METADATA,sha256=PwXbwLjDtntF2uyvDbbARjc5lvgR1yYYkDvGWNVqb4A,19134
+ddeutil_workflow-0.0.44.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+ddeutil_workflow-0.0.44.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
+ddeutil_workflow-0.0.44.dist-info/RECORD,,