ddeutil-workflow 0.0.50__py3-none-any.whl → 0.0.51__py3-none-any.whl

ddeutil/workflow/workflow.py

@@ -6,6 +6,9 @@
 # [x] Use dynamic config
 """A Workflow module that is the core module of this package. It keeps Release
 and Workflow Pydantic models.
+
+    I will implement timeout on the workflow execution layer only because the
+main propose of this package in Workflow model.
 """
 from __future__ import annotations
 
@@ -36,7 +39,7 @@ from .__cron import CronJob, CronRunner
 from .__types import DictData, TupleStr
 from .conf import Loader, SimLoad, dynamic
 from .cron import On
-from .exceptions import JobException, WorkflowException
+from .exceptions import JobException, UtilException, WorkflowException
 from .job import Job
 from .logs import Audit, get_audit
 from .params import Param
@@ -636,20 +639,20 @@ class Workflow(BaseModel):
         run_id: str | None = None,
         parent_run_id: str | None = None,
         audit: type[Audit] = None,
-        queue: ReleaseQueue | None = None,
+        queue: Optional[ReleaseQueue] = None,
         override_log_name: str | None = None,
-        result: Result | None = None,
+        result: Optional[Result] = None,
+        timeout: int = 600,
     ) -> Result:
         """Release the workflow execution with overriding parameter with the
         release templating that include logical date (release date), execution
         date, or running id to the params.
 
             This method allow workflow use audit object to save the execution
-        result to audit destination like file audit to the local `/logs`
-        directory.
+        result to audit destination like file audit to the local `./logs` path.
 
         Steps:
-            - Initialize ReleaseQueue and Release if they do not pass.
+            - Initialize Release and validate ReleaseQueue.
             - Create release data for pass to parameter templating function.
             - Execute this workflow with mapping release data to its parameters.
             - Writing result audit
@@ -658,15 +661,15 @@ class Workflow(BaseModel):
 
         :param release: A release datetime or Release object.
         :param params: A workflow parameter that pass to execute method.
-        :param queue: A ReleaseQueue that use for mark complete.
-        :param run_id: A workflow running ID for this release.
-        :param parent_run_id: A parent workflow running ID for this release.
+        :param run_id: (str) A workflow running ID.
+        :param parent_run_id: (str) A parent workflow running ID.
         :param audit: An audit class that want to save the execution result.
-        :param queue: A ReleaseQueue object.
-        :param override_log_name: An override logging name that use instead
-            the workflow name.
+        :param queue: (ReleaseQueue) A ReleaseQueue object.
+        :param override_log_name: (str) An override logging name that use
+            instead the workflow name.
         :param result: (Result) A result object for keeping context and status
             data.
+        :param timeout: (int) A workflow execution time out in second unit.
 
         :raise TypeError: If a queue parameter does not match with ReleaseQueue
             type.
@@ -683,7 +686,8 @@ class Workflow(BaseModel):
             extras=self.extras,
         )
 
-        if queue is not None and not isinstance(queue, ReleaseQueue):
+        # VALIDATE: check type of queue that valid with ReleaseQueue.
+        if queue and not isinstance(queue, ReleaseQueue):
             raise TypeError(
                 "The queue argument should be ReleaseQueue object only."
             )
@@ -693,36 +697,29 @@ class Workflow(BaseModel):
             release: Release = Release.from_dt(release, extras=self.extras)
 
         result.trace.debug(
-            f"[RELEASE]: Start release - {name!r} : "
-            f"{release.date:%Y-%m-%d %H:%M:%S}"
+            f"[RELEASE]: Start {name!r} : {release.date:%Y-%m-%d %H:%M:%S}"
         )
-
-        # NOTE: Release parameters that use to templating on the schedule
-        #   config data.
-        release_params: DictData = {
-            "release": {
-                "logical_date": release.date,
-                "execute_date": datetime.now(
-                    tz=dynamic("tz", extras=self.extras)
-                ),
-                "run_id": result.run_id,
-                "timezone": dynamic("tz", extras=self.extras),
-            }
-        }
-
-        # NOTE: Execute workflow with templating params from release mapping.
-        #   The result context that return from execution method is:
-        #
-        #   ... {"params": ..., "jobs": ...}
-        #
         self.execute(
-            params=param2template(params, release_params, extras=self.extras),
+            params=param2template(
+                params,
+                params={
+                    "release": {
+                        "logical_date": release.date,
+                        "execute_date": datetime.now(
+                            tz=dynamic("tz", extras=self.extras)
+                        ),
+                        "run_id": result.run_id,
+                        "timezone": dynamic("tz", extras=self.extras),
+                    }
+                },
+                extras=self.extras,
+            ),
             result=result,
             parent_run_id=result.parent_run_id,
+            timeout=timeout,
         )
         result.trace.debug(
-            f"[RELEASE]: End release - {name!r} : "
-            f"{release.date:%Y-%m-%d %H:%M:%S}"
+            f"[RELEASE]: End {name!r} : {release.date:%Y-%m-%d %H:%M:%S}"
         )
 
         # NOTE: Saving execution result to destination of the input audit
@@ -741,19 +738,10 @@ class Workflow(BaseModel):
             ).save(excluded=None)
         )
 
-        # NOTE: Remove this release from running.
-        if queue is not None:
+        if queue:
             queue.remove_running(release)
             queue.mark_complete(release)
 
-        # NOTE: Remove the params key from the result context for deduplicate.
-        #   This step is prepare result context for this release method.
-        context: DictData = result.context
-        jobs: DictData = context.pop("jobs", {})
-        errors: DictData = (
-            {"errors": context.pop("errors", {})} if "errors" in context else {}
-        )
-
         return result.catch(
             status=SUCCESS,
             context={
@@ -763,8 +751,7 @@ class Workflow(BaseModel):
                     "logical_date": release.date,
                     "release": release,
                 },
-                "outputs": {"jobs": jobs},
-                **errors,
+                "outputs": {"jobs": result.context.pop("jobs", {})},
             },
         )
 
@@ -1004,6 +991,12 @@ class Workflow(BaseModel):
                 f"workflow."
             )
 
+        job: Job = self.job(name=job_id)
+        if job.is_skipped(params=params):
+            result.trace.info(f"[WORKFLOW]: Skip job: {job_id!r}")
+            job.set_outputs(output={"skipped": True}, to=params)
+            return result.catch(status=SKIP, context=params)
+
         if event and event.is_set():  # pragma: no cov
             raise WorkflowException(
                 "Workflow job was canceled from event that had set before "
@@ -1011,27 +1004,31 @@ class Workflow(BaseModel):
             )
 
         try:
-            job: Job = self.jobs[job_id]
-            if job.is_skipped(params=params):
-                result.trace.info(f"[WORKFLOW]: Skip job: {job_id!r}")
-                job.set_outputs(output={"SKIP": {"skipped": True}}, to=params)
-            else:
-                result.trace.info(f"[WORKFLOW]: Execute: {job_id!r}")
-                job.set_outputs(
-                    job.execute(
-                        params=params,
-                        run_id=result.run_id,
-                        parent_run_id=result.parent_run_id,
-                        event=event,
-                    ).context,
-                    to=params,
-                )
-        except JobException as e:
+            result.trace.info(f"[WORKFLOW]: Execute Job: {job_id!r}")
+            rs: Result = job.execute(
+                params=params,
+                run_id=result.run_id,
+                parent_run_id=result.parent_run_id,
+                event=event,
+            )
+            job.set_outputs(rs.context, to=params)
+        except (JobException, UtilException) as e:
             result.trace.error(f"[WORKFLOW]: {e.__class__.__name__}: {e}")
             raise WorkflowException(
                 f"Get job execution error {job_id}: JobException: {e}"
             ) from None
 
+        if rs.status == FAILED:
+            error_msg: str = (
+                f"Workflow job, {job.id}, failed without raise error."
+            )
+            return result.catch(
+                status=FAILED,
+                context={
+                    "errors": WorkflowException(error_msg).to_dict(),
+                    **params,
+                },
+            )
         return result.catch(status=SUCCESS, context=params)
 
     def execute(
@@ -1083,7 +1080,7 @@ class Workflow(BaseModel):
             extras=self.extras,
         )
 
-        result.trace.info(f"[WORKFLOW]: Start Execute: {self.name!r} ...")
+        result.trace.info(f"[WORKFLOW]: Execute: {self.name!r} ...")
         if not self.jobs:
             result.trace.warning(
                 f"[WORKFLOW]: {self.name!r} does not have any jobs"
@@ -1128,7 +1125,7 @@ class Workflow(BaseModel):
                     timeout=timeout,
                     event=event,
                 )
-        except WorkflowException as e:
+        except (WorkflowException, JobException) as e:
             status: Status = FAILED
             context.update({"errors": e.to_dict()})
 
@@ -1167,7 +1164,7 @@ class Workflow(BaseModel):
             "max_job_exec_timeout", f=timeout, extras=self.extras
         )
         event: Event = event or Event()
-        result.trace.debug(f"[WORKFLOW]: Run {self.name!r} with threading.")
+        result.trace.debug(f"... Run {self.name!r} with threading.")
         with ThreadPoolExecutor(
             max_workers=dynamic("max_job_parallel", extras=self.extras),
             thread_name_prefix="wf_exec_threading_",
@@ -1192,7 +1189,7 @@ class Workflow(BaseModel):
                     )
                 elif check == SKIP:  # pragma: no cov
                     result.trace.info(f"[JOB]: Skip job: {job_id!r}")
-                    job.set_outputs({"SKIP": {"skipped": True}}, to=context)
+                    job.set_outputs(output={"skipped": True}, to=context)
                     job_queue.task_done()
                     continue
 
@@ -1259,7 +1256,7 @@ class Workflow(BaseModel):
             "max_job_exec_timeout", f=timeout, extras=self.extras
         )
         event: Event = event or Event()
-        result.trace.debug(f"[WORKFLOW]: Run {self.name!r} with non-threading.")
+        result.trace.debug(f"... Run {self.name!r} with non-threading.")
         with ThreadPoolExecutor(
             max_workers=1,
             thread_name_prefix="wf_exec_non_threading_",
@@ -1284,7 +1281,7 @@ class Workflow(BaseModel):
                     )
                 elif check == SKIP:  # pragma: no cov
                     result.trace.info(f"[JOB]: Skip job: {job_id!r}")
-                    job.set_outputs({"SKIP": {"skipped": True}}, to=context)
+                    job.set_outputs(output={"skipped": True}, to=context)
                     job_queue.task_done()
                     continue
 
@@ -1297,7 +1294,7 @@ class Workflow(BaseModel):
                         event=event,
                     )
                     time.sleep(0.025)
-                elif future.done():
+                elif future.done() or future.cancelled():
                     if e := future.exception():
                         result.trace.error(f"[WORKFLOW]: {e}")
                         raise WorkflowException(str(e))
@@ -1317,6 +1314,13 @@ class Workflow(BaseModel):
 
             if not_timeout_flag:
                 job_queue.join()
+                if future:  # pragma: no cov
+                    if e := future.exception():
+                        result.trace.error(f"[WORKFLOW]: {e}")
+                        raise WorkflowException(str(e))
+
+                    future.result()
+
                 return context
 
             result.trace.error(

{ddeutil_workflow-0.0.50.dist-info → ddeutil_workflow-0.0.51.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddeutil-workflow
-Version: 0.0.50
+Version: 0.0.51
 Summary: Lightweight workflow orchestration
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT
@@ -22,8 +22,8 @@ Classifier: Programming Language :: Python :: 3.13
 Requires-Python: >=3.9.13
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: ddeutil[checksum]>=0.4.6
-Requires-Dist: ddeutil-io[toml,yaml]>=0.2.10
+Requires-Dist: ddeutil[checksum]>=0.4.7
+Requires-Dist: ddeutil-io[toml,yaml]>=0.2.11
 Requires-Dist: pydantic==2.11.1
 Requires-Dist: python-dotenv==1.1.0
 Requires-Dist: schedule<2.0.0,==1.2.2
@@ -139,7 +139,8 @@ flowchart LR
 
 ## 📦 Installation
 
-This project need `ddeutil` and `ddeutil-io` extension namespace packages.
+This project need `ddeutil` and `ddeutil-io` extension namespace packages to be
+the base deps.
 If you want to install this package with application add-ons, you should add
 `app` in installation;
 
@@ -148,7 +149,7 @@ If you want to install this package with application add-ons, you should add
 | Python         | `ddeutil-workflow`       | :heavy_check_mark:  |
 | FastAPI Server | `ddeutil-workflow[api]`  | :heavy_check_mark:  |
 
-## :beers: Usage
+## 🎯 Usage
 
 This is examples that use workflow file for running common Data Engineering
 use-case.
@@ -198,12 +199,54 @@ run-py-local:
 
                  # Arguments of target data that want to land.
                  writing_mode: flatten
-                 aws_s3_path: my-data/open-data/${{ params.source-extract }}
+                 aws:
+                    path: my-data/open-data/${{ params.source-extract }}
 
-                 # This Authentication code should implement with your custom call
-                 # function. The template allow you to use environment variable.
-                 aws_access_client_id: ${AWS_ACCESS_CLIENT_ID}
-                 aws_access_client_secret: ${AWS_ACCESS_CLIENT_SECRET}
+                    # This Authentication code should implement with your custom call
+                    # function. The template allow you to use environment variable.
+                    access_client_id: ${AWS_ACCESS_CLIENT_ID}
+                    access_client_secret: ${AWS_ACCESS_CLIENT_SECRET}
+```
+
+Before execute this workflow, you should implement caller function first.
+
+```text
+registry-caller/
+  ╰─ tasks.py
+```
+
+This function will store as module that will import from `WORKFLOW_CORE_REGISTRY_CALLER`
+value (This config can override by extra parameters with `registry_caller` key).
+
+```python
+from ddeutil.workflow import Result, tag
+from ddeutil.workflow.exceptions import StageException
+from pydantic import BaseModel, SecretStr
+
+class AwsCredential(BaseModel):
+    path: str
+    access_client_id: str
+    access_client_secret: SecretStr
+
+class RestAuth(BaseModel):
+    type: str
+    keys: SecretStr
+
+@tag("requests", alias="get-api-with-oauth-to-s3")
+def get_api_with_oauth_to_s3(
+    method: str,
+    url: str,
+    body: dict[str, str],
+    auth: RestAuth,
+    writing_node: str,
+    aws: AwsCredential,
+    result: Result,
+) -> dict[str, int]:
+    result.trace.info("[CALLER]: Start get data via RestAPI to S3.")
+    result.trace.info(f"... {method}: {url}")
+    if method != "post":
+       raise StageException(f"RestAPI does not support for {method} action.")
+    return {"records": 1000}
 ```
 
 The above workflow template is main executor pipeline that you want to do. If you
@@ -298,13 +341,15 @@ only.
 ## :rocket: Deployment
 
 This package able to run as an application service for receive manual trigger
-from the master node via RestAPI or use to be Scheduler background service
-like crontab job but via Python API.
+from any node via RestAPI or use to be Scheduler background application
+like crontab job but via Python API or FastAPI app.
 
 ### API Server
 
+This server use FastAPI package to be the base application.
+
 ```shell
-(venv) $ uvicorn ddeutil.workflow.api:app \
+(.venv) $ uvicorn ddeutil.workflow.api:app \
   --host 127.0.0.1 \
   --port 80 \
   --no-access-log
@@ -314,8 +359,19 @@ like crontab job but via Python API.
 > If this package already deploy, it is able to use multiprocess;
 > `uvicorn ddeutil.workflow.api:app --host 127.0.0.1 --port 80 --workers 4`
 
+### Local Schedule
+
+> [!WARNING]
+> This CLI does not implement yet.
+
+```shell
+(.venv) $ ddeutil-workflow schedule
+```
+
 ### Docker Container
 
+Build a Docker container from this package.
+
 ```shell
 $ docker build -t ddeutil-workflow:latest -f .container/Dockerfile .
 $ docker run -i ddeutil-workflow:latest ddeutil-workflow

{ddeutil_workflow-0.0.50.dist-info → ddeutil_workflow-0.0.51.dist-info}/RECORD

@@ -1,20 +1,20 @@
-ddeutil/workflow/__about__.py,sha256=K4g7cm4iInR43bd_K3fsuGDTVpz7fbAASmKh5_jH8_U,28
+ddeutil/workflow/__about__.py,sha256=lV_4ewekJpV5o1pQoqnMK3g475fbmyO855YKAFNFKF8,28
 ddeutil/workflow/__cron.py,sha256=h8rLeIUAAEB2SdZ4Jhch7LU1Yl3bbJ-iNNJ3tQ0eYVM,28095
-ddeutil/workflow/__init__.py,sha256=3u-yGnTyfY4BFrKqA5UGaMVe_Q4cZNODuC9qZ5meOXo,2048
+ddeutil/workflow/__init__.py,sha256=AMmM7Cq_jrO_MzIZjzQyFvcFkVj9ad7V1f3PtLrI030,1736
 ddeutil/workflow/__main__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ddeutil/workflow/__types.py,sha256=8jBdbfb3aZSetjz0mvNrpGHwwxJff7mK8_4v41cLqlc,4316
-ddeutil/workflow/conf.py,sha256=o6RyqcjjeQXdPBZi3lMs5sSQ5aYvsUgMdJMoRYMWcN0,12492
-ddeutil/workflow/cron.py,sha256=80SijzMdDOBxTWRsiF-Fmuz7Ym7leY0XT2lzRAPGdXc,8781
+ddeutil/workflow/conf.py,sha256=80rgmJKFU7BlH5xTLnghGzGhE8C6LFAQykd9mjHSjo8,12528
+ddeutil/workflow/cron.py,sha256=WS2MInn0Sp5DKlZDZH5VFZ5AA0Q3_AnBnYEU4lZSv4I,9779
 ddeutil/workflow/exceptions.py,sha256=r4Jrf9qtVPALU4wh4bnb_OYqC-StqSQJEmFC-_QK934,1408
-ddeutil/workflow/job.py,sha256=-VUsv6ub3T199GyugplRjI8Vs6CKQ9QHY6yhlzvUF9w,32495
-ddeutil/workflow/logs.py,sha256=GG8tqs2BQv-iXSPWqxfrRJvKwJBvXn86Uq2lW1HrM9U,26455
-ddeutil/workflow/params.py,sha256=xCtFEh0-G-G-f8y_SXxyf31bU6Ox5p5Z-WbBFXrjy8M,9960
+ddeutil/workflow/job.py,sha256=LydncguMUTTdcG_fImjoPFZBulmZT_PD3jYmnmcYskk,35041
+ddeutil/workflow/logs.py,sha256=rsoBrUGQrooou18fg2yvPsB8NOaXnUA5ThQpBr_WVMg,26598
+ddeutil/workflow/params.py,sha256=FKY4Oo1Ze4QZKRfAk7rqKsi44YaJQAbqAtXM6vlO2hI,11392
 ddeutil/workflow/result.py,sha256=27nPQq9CETLCVczv4vvFEF9w2TllHZ_ROfyDoLFxRWM,5647
-ddeutil/workflow/reusables.py,sha256=hIpehea6J4OWeXX55kjYzo-c9-_Cc0YRwLRRbcaUkZs,17539
-ddeutil/workflow/scheduler.py,sha256=F783QaJfPg8tvYyvJvkwl8Sa42vsJzj6BzzROZFvm9I,28153
-ddeutil/workflow/stages.py,sha256=rQtW8W8btMavJxnseusHOH4HAv7SA_WLm9zQsCK22f8,63237
+ddeutil/workflow/reusables.py,sha256=iXcS7Gg-71qVX4ln0ILTDx03cTtUnj_rNoXHTVdVrxc,17636
+ddeutil/workflow/scheduler.py,sha256=4G5AogkmnsTKe7jKYSfU35qjubR82WQ8CLtEe9kqPTE,28304
+ddeutil/workflow/stages.py,sha256=Ywwky6rJJ5cMvridG-1Y4rv-wJJ_WMbXysViR3WxQrY,69108
 ddeutil/workflow/utils.py,sha256=zbVttaMFMRLuuBJdSJf7D9qtz8bOnQIBq-rHI3Eqy4M,7821
-ddeutil/workflow/workflow.py,sha256=4jp7wm8TkSv8CXOKrCC-dlFgTP2d0OXgRHimoXnjSvY,50430
+ddeutil/workflow/workflow.py,sha256=v_O3Gm4J6SCN_1NfGd3_ClpJkatFCyoqXR3nAIMAoF4,50575
 ddeutil/workflow/api/__init__.py,sha256=F53NMBWtb9IKaDWkPU5KvybGGfKAcbehgn6TLBwHuuM,21
 ddeutil/workflow/api/api.py,sha256=CWtPLgOv2Jus9E7nzG5mG2Z32ZEkUK3JWQ2htZyMRpA,5244
 ddeutil/workflow/api/log.py,sha256=NMTnOnsBrDB5129329xF2myLdrb-z9k1MQrmrP7qXJw,1818
@@ -24,8 +24,8 @@ ddeutil/workflow/api/routes/job.py,sha256=oPwBVP0Mxwxv-bGPlfmxQQ9PcVl0ev9HoPzndp
 ddeutil/workflow/api/routes/logs.py,sha256=U6vOni3wd-ZTOwd3yVdSOpgyRmNdcgfngU5KlLM3Cww,5383
 ddeutil/workflow/api/routes/schedules.py,sha256=EgUjyRGhsm6UNaMj5luh6TcY6l571sCHcla-BL1iOfY,4829
 ddeutil/workflow/api/routes/workflows.py,sha256=JcDOrn1deK8ztFRcMTNATQejG6KMA7JxZLVc4QeBsP4,4527
-ddeutil_workflow-0.0.50.dist-info/licenses/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
-ddeutil_workflow-0.0.50.dist-info/METADATA,sha256=jh7H6NtEXQ5lKiuSpZ63LAwKDboVmV-AKGSZPiATwn4,18036
-ddeutil_workflow-0.0.50.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-ddeutil_workflow-0.0.50.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
-ddeutil_workflow-0.0.50.dist-info/RECORD,,
+ddeutil_workflow-0.0.51.dist-info/licenses/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
+ddeutil_workflow-0.0.51.dist-info/METADATA,sha256=xlAbUVZ9djYf1GImH9v0N-mR7XpxcQIkvwGuX5XmSrs,19425
+ddeutil_workflow-0.0.51.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+ddeutil_workflow-0.0.51.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
+ddeutil_workflow-0.0.51.dist-info/RECORD,,