ddeutil-workflow 0.0.36__py3-none-any.whl → 0.0.38__py3-none-any.whl

ddeutil/workflow/utils.py

@@ -5,10 +5,9 @@
 # ------------------------------------------------------------------------------
 from __future__ import annotations
 
-import logging
 import stat
 import time
-from collections.abc import Iterator, Mapping
+from collections.abc import Iterator
 from datetime import date, datetime, timedelta
 from hashlib import md5
 from inspect import isfunction
@@ -24,7 +23,6 @@ from .__types import DictData, Matrix
 
 T = TypeVar("T")
 UTC = ZoneInfo("UTC")
-logger = logging.getLogger("ddeutil.workflow")
 
 
 def get_dt_now(
@@ -32,8 +30,10 @@ def get_dt_now(
 ) -> datetime:  # pragma: no cov
     """Return the current datetime object.
 
-    :param tz:
-    :param offset:
+    :param tz: A ZoneInfo object for replace timezone of return datetime object.
+    :param offset: An offset second value.
+
+    :rtype: datetime
     :return: The current datetime object that use an input timezone or UTC.
     """
     return datetime.now(tz=(tz or UTC)) - timedelta(seconds=offset)
@@ -42,6 +42,14 @@ def get_dt_now(
 def get_d_now(
     tz: ZoneInfo | None = None, offset: float = 0.0
 ) -> date:  # pragma: no cov
+    """Return the current date object.
+
+    :param tz: A ZoneInfo object for replace timezone of return date object.
+    :param offset: An offset second value.
+
+    :rtype: date
+    :return: The current date object that use an input timezone or UTC.
+    """
     return (datetime.now(tz=(tz or UTC)) - timedelta(seconds=offset)).date()
 
 
@@ -52,8 +60,10 @@ def get_diff_sec(
     current datetime with specific timezone.
 
     :param dt:
-    :param tz:
-    :param offset:
+    :param tz: A ZoneInfo object for replace timezone of return datetime object.
+    :param offset: An offset second value.
+
+    :rtype: int
     """
     return round(
         (
@@ -67,6 +77,10 @@ def reach_next_minute(
 ) -> bool:
     """Check this datetime object is not in range of minute level on the current
     datetime.
+
+    :param dt:
+    :param tz: A ZoneInfo object for replace timezone of return datetime object.
+    :param offset: An offset second value.
     """
     diff: float = (
         dt.replace(second=0, microsecond=0)
@@ -128,13 +142,14 @@ def gen_id(
         value: str = str(value)
 
     if config.gen_id_simple_mode:
-        return hash_str(f"{(value if sensitive else value.lower())}", n=10) + (
-            f"{datetime.now(tz=config.tz):%Y%m%d%H%M%S%f}" if unique else ""
-        )
+        return (
+            f"{datetime.now(tz=config.tz):%Y%m%d%H%M%S%f}T" if unique else ""
+        ) + hash_str(f"{(value if sensitive else value.lower())}", n=10)
+
     return md5(
         (
-            f"{(value if sensitive else value.lower())}"
-            + (f"{datetime.now(tz=config.tz):%Y%m%d%H%M%S%f}" if unique else "")
+            (f"{datetime.now(tz=config.tz):%Y%m%d%H%M%S%f}T" if unique else "")
+            + f"{(value if sensitive else value.lower())}"
         ).encode()
     ).hexdigest()
 
@@ -171,25 +186,6 @@ def filter_func(value: T) -> T:
     return value
 
 
-def dash2underscore(
-    key: str,
-    values: DictData,
-    *,
-    fixed: str | None = None,
-) -> DictData:
-    """Change key name that has dash to underscore.
-
-    :param key
-    :param values
-    :param fixed
-
-    :rtype: DictData
-    """
-    if key in values:
-        values[(fixed or key.replace("-", "_"))] = values.pop(key)
-    return values
-
-
 def cross_product(matrix: Matrix) -> Iterator[DictData]:
     """Iterator of products value from matrix.
 
@@ -246,21 +242,3 @@ def cut_id(run_id: str, *, num: int = 6) -> str:
     :rtype: str
     """
     return run_id[-num:]
-
-
-def deep_update(origin: DictData, u: Mapping) -> DictData:
-    """Deep update dict.
-
-    Example:
-        >>> deep_update(
-        ...     origin={"jobs": {"job01": "foo"}},
-        ...     u={"jobs": {"job02": "bar"}},
-        ... )
-        {"jobs": {"job01": "foo", "job02": "bar"}}
-    """
-    for k, value in u.items():
-        if isinstance(value, Mapping) and value:
-            origin[k] = deep_update(origin.get(k, {}), value)
-        else:
-            origin[k] = value
-    return origin

ddeutil/workflow/workflow.py

@@ -20,6 +20,7 @@ from functools import partial, total_ordering
 from heapq import heappop, heappush
 from queue import Queue
 from textwrap import dedent
+from threading import Event
 from typing import Optional
 
 from pydantic import BaseModel, ConfigDict, Field
@@ -68,34 +69,45 @@ class ReleaseType(str, Enum):
     config=ConfigDict(arbitrary_types_allowed=True, use_enum_values=True)
 )
 class Release:
-    """Release Pydantic dataclass object that use for represent
-    the release data that use with the `workflow.release` method."""
+    """Release Pydantic dataclass object that use for represent the release data
+    that use with the `workflow.release` method.
+    """
 
-    date: datetime
-    offset: float
-    end_date: datetime
-    runner: CronRunner
+    date: datetime = field()
+    offset: float = field()
+    end_date: datetime = field()
+    runner: CronRunner = field()
     type: ReleaseType = field(default=ReleaseType.DEFAULT)
 
     def __repr__(self) -> str:
+        """Represent string"""
         return repr(f"{self.date:%Y-%m-%d %H:%M:%S}")
 
     def __str__(self) -> str:
+        """Override string value of this release object with the date field.
+
+        :rtype: str
+        """
         return f"{self.date:%Y-%m-%d %H:%M:%S}"
 
     @classmethod
     def from_dt(cls, dt: datetime | str) -> Self:
         """Construct Release via datetime object only.
 
-        :param dt: A datetime object.
+        :param dt: (datetime | str) A datetime object or string that want to
+            construct to the Release object.
 
-        :rtype: Self
+        :raise TypeError: If the type of the dt argument does not valid with
+            datetime or str object.
+
+        :rtype: Release
         """
         if isinstance(dt, str):
             dt: datetime = datetime.fromisoformat(dt)
         elif not isinstance(dt, datetime):
             raise TypeError(
-                "The `from_dt` need argument type be str or datetime only."
+                "The `from_dt` need the `dt` argument type be str or datetime "
+                "only."
             )
 
         return cls(
@@ -108,6 +120,8 @@ class Release:
     def __eq__(self, other: Release | datetime) -> bool:
         """Override equal property that will compare only the same type or
         datetime.
+
+        :rtype: bool
         """
         if isinstance(other, self.__class__):
             return self.date == other.date
@@ -118,6 +132,8 @@ class Release:
     def __lt__(self, other: Release | datetime) -> bool:
         """Override equal property that will compare only the same type or
         datetime.
+
+        :rtype: bool
         """
         if isinstance(other, self.__class__):
             return self.date < other.date
@@ -143,7 +159,7 @@ class ReleaseQueue:
 
         :raise TypeError: If the type of input queue does not valid.
 
-        :rtype: Self
+        :rtype: ReleaseQueue
         """
         if queue is None:
             return cls()
@@ -174,7 +190,7 @@ class ReleaseQueue:
         """Check an input Release object is the first value of the
         waiting queue.
 
-        :rtype: bool
+        :rtype: Release
         """
         return self.queue[0]
 
@@ -265,12 +281,12 @@ class Workflow(BaseModel):
         an input workflow name. The loader object will use this workflow name to
         searching configuration data of this workflow model in conf path.
 
-        :raise ValueError: If the type does not match with current object.
-
         :param name: A workflow name that want to pass to Loader object.
         :param externals: An external parameters that want to pass to Loader
             object.
 
+        :raise ValueError: If the type does not match with current object.
+
         :rtype: Self
         """
         loader: Loader = Loader(name, externals=(externals or {}))
@@ -285,11 +301,11 @@ class Workflow(BaseModel):
         loader_data["name"] = name.replace(" ", "_")
 
         # NOTE: Prepare `on` data
-        cls.__bypass_on(loader_data, externals=externals)
+        cls.__bypass_on__(loader_data, externals=externals)
         return cls.model_validate(obj=loader_data)
 
     @classmethod
-    def __bypass_on(
+    def __bypass_on__(
         cls,
         data: DictData,
         externals: DictData | None = None,
@@ -405,10 +421,13 @@ class Workflow(BaseModel):
         return self
 
     def job(self, name: str) -> Job:
-        """Return this workflow's jobs that passing with the Job model.
+        """Return the workflow's job model that searching with an input job's
+        name or job's ID.
 
-        :param name: A job name that want to get from a mapping of job models.
-        :type name: str
+        :param name: (str) A job name or ID that want to get from a mapping of
+            job models.
+
+        :raise ValueError: If a name or ID does not exist on the jobs field.
 
         :rtype: Job
         :return: A job model that exists on this workflow by input name.
@@ -505,18 +524,19 @@ class Workflow(BaseModel):
         :param result: (Result) A result object for keeping context and status
             data.
 
+        :raise TypeError: If a queue parameter does not match with ReleaseQueue
+            type.
+
         :rtype: Result
         """
         audit: type[Audit] = audit or get_audit()
         name: str = override_log_name or self.name
-
-        if result is None:
-            result: Result = Result(
-                run_id=(run_id or gen_id(name, unique=True)),
-                parent_run_id=parent_run_id,
-            )
-        elif parent_run_id:  # pragma: no cov
-            result.set_parent_run_id(parent_run_id)
+        result: Result = Result.construct_with_rs_or_id(
+            result,
+            run_id=run_id,
+            parent_run_id=parent_run_id,
+            id_logic=name,
+        )
 
         if queue is not None and not isinstance(queue, ReleaseQueue):
             raise TypeError(
@@ -795,9 +815,7 @@ class Workflow(BaseModel):
                     partial_queue(q)
                     continue
 
-                # NOTE: Push the latest Release to the running queue.
                 heappush(q.running, release)
-
                 futures.append(
                     executor.submit(
                         self.release,
@@ -827,6 +845,7 @@ class Workflow(BaseModel):
         params: DictData,
         *,
         result: Result | None = None,
+        event: Event | None = None,
         raise_error: bool = True,
     ) -> Result:
         """Job execution with passing dynamic parameters from the main workflow
@@ -842,10 +861,12 @@ class Workflow(BaseModel):
 
         :param job_id: A job ID that want to execute.
         :param params: A params that was parameterized from workflow execution.
-        :param raise_error: A flag that raise error instead catching to result
-            if it gets exception from job execution.
         :param result: (Result) A result object for keeping context and status
             data.
+        :param event: (Event) An event manager that pass to the
+            PoolThreadExecutor.
+        :param raise_error: A flag that raise error instead catching to result
+            if it gets exception from job execution.
 
         :rtype: Result
         :return: Return the result object that receive the job execution result
@@ -863,6 +884,12 @@ class Workflow(BaseModel):
 
         result.trace.info(f"[WORKFLOW]: Start execute job: {job_id!r}")
 
+        if event and event.is_set():  # pragma: no cov
+            raise WorkflowException(
+                "Workflow job was canceled from event that had set before "
+                "job execution."
+            )
+
         # IMPORTANT:
         #   This execution change all job running IDs to the current workflow
         #   running ID, but it still trac log to the same parent running ID
@@ -876,6 +903,7 @@ class Workflow(BaseModel):
                     params=params,
                     run_id=result.run_id,
                     parent_run_id=result.parent_run_id,
+                    event=event,
                 ).context,
                 to=params,
             )
@@ -889,7 +917,7 @@ class Workflow(BaseModel):
                 "Handle error from the job execution does not support yet."
             ) from None
 
-        return result.catch(status=0, context=params)
+        return result.catch(status=Status.SUCCESS, context=params)
 
     def execute(
         self,
@@ -927,22 +955,21 @@ class Workflow(BaseModel):
         :rtype: Result
         """
         ts: float = time.monotonic()
-        if result is None:  # pragma: no cov
-            result: Result = Result(
-                run_id=(run_id or gen_id(self.name, unique=True)),
-                parent_run_id=parent_run_id,
-            )
-        elif parent_run_id:
-            result.set_parent_run_id(parent_run_id)
+        result: Result = Result.construct_with_rs_or_id(
+            result,
+            run_id=run_id,
+            parent_run_id=parent_run_id,
+            id_logic=self.name,
+        )
 
         result.trace.info(f"[WORKFLOW]: Start Execute: {self.name!r} ...")
 
         # NOTE: It should not do anything if it does not have job.
         if not self.jobs:
             result.trace.warning(
-                f"[WORKFLOW]: Workflow: {self.name!r} does not have any jobs"
+                f"[WORKFLOW]: {self.name!r} does not have any jobs"
             )
-            return result.catch(status=0, context=params)
+            return result.catch(status=Status.SUCCESS, context=params)
 
         # NOTE: Create a job queue that keep the job that want to run after
         #   its dependency condition.
@@ -959,7 +986,7 @@ class Workflow(BaseModel):
         #   }
         #
         context: DictData = self.parameterize(params)
-        status: int = 0
+        status: Status = Status.SUCCESS
         try:
             if config.max_job_parallel == 1:
                 self.__exec_non_threading(
@@ -978,16 +1005,9 @@ class Workflow(BaseModel):
                     timeout=timeout,
                 )
         except WorkflowException as err:
-            status: int = 1
-            context.update(
-                {
-                    "errors": {
-                        "class": err,
-                        "name": err.__class__.__name__,
-                        "message": f"{err.__class__.__name__}: {err}",
-                    },
-                },
-            )
+            status = Status.FAILED
+            context.update({"errors": err.to_dict()})
+
         return result.catch(status=status, context=context)
 
     def __exec_threading(
@@ -1005,18 +1025,19 @@ class Workflow(BaseModel):
             If a job need dependency, it will check dependency job ID from
         context data before allow it run.
 
-        :param result: A result model.
+        :param result: (Result) A result model.
         :param context: A context workflow data that want to downstream passing.
         :param ts: A start timestamp that use for checking execute time should
             time out.
-        :param job_queue: A job queue object.
-        :param timeout: A second value unit that bounding running time.
+        :param job_queue: (Queue) A job queue object.
+        :param timeout: (int) A second value unit that bounding running time.
         :param thread_timeout: A timeout to waiting all futures complete.
 
         :rtype: DictData
         """
         not_timeout_flag: bool = True
         timeout: int = timeout or config.max_job_exec_timeout
+        event: Event = Event()
         result.trace.debug(f"[WORKFLOW]: Run {self.name!r} with threading.")
 
         # IMPORTANT: The job execution can run parallel and waiting by
@@ -1036,7 +1057,7 @@ class Workflow(BaseModel):
                 if not job.check_needs(context["jobs"]):
                     job_queue.task_done()
                     job_queue.put(job_id)
-                    time.sleep(0.25)
+                    time.sleep(0.15)
                     continue
 
                 # NOTE: Start workflow job execution with deep copy context data
@@ -1055,6 +1076,7 @@ class Workflow(BaseModel):
                         job_id=job_id,
                         params=context,
                         result=result,
+                        event=event,
                     ),
                 )
 
@@ -1077,11 +1099,13 @@ class Workflow(BaseModel):
 
                 return context
 
+            result.trace.error(
+                f"[WORKFLOW]: Execution: {self.name!r} was timeout."
+            )
+            event.set()
             for future in futures:
                 future.cancel()
 
-        # NOTE: Raise timeout error.
-        result.trace.error(f"[WORKFLOW]: Execution: {self.name!r} was timeout.")
         raise WorkflowException(f"Execution: {self.name!r} was timeout.")
 
     def __exec_non_threading(
@@ -1099,18 +1123,25 @@ class Workflow(BaseModel):
             If a job need dependency, it will check dependency job ID from
         context data before allow it run.
 
-        :param result: A result model.
+        :param result: (Result) A result model.
         :param context: A context workflow data that want to downstream passing.
-        :param ts: A start timestamp that use for checking execute time should
-            time out.
-        :param timeout: A second value unit that bounding running time.
+        :param ts: (float) A start timestamp that use for checking execute time
+            should time out.
+        :param timeout: (int) A second value unit that bounding running time.
 
         :rtype: DictData
         """
         not_timeout_flag: bool = True
         timeout: int = timeout or config.max_job_exec_timeout
+        event: Event = Event()
+        future: Future | None = None
         result.trace.debug(f"[WORKFLOW]: Run {self.name!r} with non-threading.")
 
+        executor = ThreadPoolExecutor(
+            max_workers=1,
+            thread_name_prefix="wf_exec_non_threading_",
+        )
+
         while not job_queue.empty() and (
             not_timeout_flag := ((time.monotonic() - ts) < timeout)
         ):
@@ -1132,7 +1163,32 @@ class Workflow(BaseModel):
             #       'params': <input-params>,
             #       'jobs': {},
             #   }
-            self.execute_job(job_id=job_id, params=context, result=result)
+            if future is None:
+                future: Future = executor.submit(
+                    self.execute_job,
+                    job_id=job_id,
+                    params=context,
+                    result=result,
+                    event=event,
+                )
+                result.trace.debug(f"[WORKFLOW]: Make future: {future}")
+                time.sleep(0.025)
+            elif future.done():
+                if err := future.exception():
+                    result.trace.error(f"[WORKFLOW]: {err}")
+                    raise WorkflowException(str(err))
+
+                future = None
+                job_queue.put(job_id)
+            elif future.running():
+                time.sleep(0.075)
+                job_queue.put(job_id)
+            else:  # pragma: no cov
+                job_queue.put(job_id)
+                result.trace.debug(
+                    f"Execution non-threading does not handle case: {future} "
+                    f"that not running."
+                )
 
             # NOTE: Mark this job queue done.
             job_queue.task_done()
@@ -1142,11 +1198,12 @@ class Workflow(BaseModel):
             # NOTE: Wait for all items to finish processing by `task_done()`
             #   method.
             job_queue.join()
-
+            executor.shutdown()
             return context
 
-        # NOTE: Raise timeout error.
         result.trace.error(f"[WORKFLOW]: Execution: {self.name!r} was timeout.")
+        event.set()
+        executor.shutdown()
         raise WorkflowException(f"Execution: {self.name!r} was timeout.")
 
 
@@ -1162,9 +1219,9 @@ class WorkflowTask:
     arguments before passing to the parent release method.
     """
 
-    alias: str
-    workflow: Workflow
-    runner: CronRunner
+    alias: str = field()
+    workflow: Workflow = field()
+    runner: CronRunner = field()
     values: DictData = field(default_factory=dict)
 
     def release(
@@ -1185,6 +1242,11 @@ class WorkflowTask:
         :param audit: An audit class that want to save the execution result.
         :param queue: A ReleaseQueue object that use to mark complete.
 
+        :raise ValueError: If a queue parameter does not pass while release
+            is None.
+        :raise TypeError: If a queue parameter does not match with ReleaseQueue
+            type.
+
         :rtype: Result
         """
         audit: type[Audit] = audit or get_audit()
@@ -1209,7 +1271,6 @@ class WorkflowTask:
             else:
                 release = self.runner.date
 
-        # NOTE: Call the workflow release method.
         return self.workflow.release(
             release=release,
             params=self.values,
@@ -1264,13 +1325,14 @@ class WorkflowTask:
         if self.runner.date > end_date:
             return queue
 
-        # NOTE: Push the Release object to queue.
         heappush(queue.queue, workflow_release)
-
         return queue
 
     def __repr__(self) -> str:
-        """Override the `__repr__` method."""
+        """Override the `__repr__` method.
+
+        :rtype: str
+        """
         return (
             f"{self.__class__.__name__}(alias={self.alias!r}, "
             f"workflow={self.workflow.name!r}, runner={self.runner!r}, "
@@ -1278,7 +1340,10 @@ class WorkflowTask:
         )
 
     def __eq__(self, other: WorkflowTask) -> bool:
-        """Override equal property that will compare only the same type."""
+        """Override the equal property that will compare only the same type.
+
+        :rtype: bool
+        """
         if isinstance(other, WorkflowTask):
             return (
                 self.workflow.name == other.workflow.name

{ddeutil_workflow-0.0.36.dist-info → ddeutil_workflow-0.0.38.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: ddeutil-workflow
-Version: 0.0.36
+Version: 0.0.38
 Summary: Lightweight workflow orchestration
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT
@@ -31,6 +31,10 @@ Provides-Extra: api
 Requires-Dist: fastapi<1.0.0,>=0.115.0; extra == "api"
 Requires-Dist: httpx; extra == "api"
 Requires-Dist: ujson; extra == "api"
+Provides-Extra: async
+Requires-Dist: aiofiles; extra == "async"
+Requires-Dist: aiohttp; extra == "async"
+Dynamic: license-file
 
 # Workflow Orchestration
 
@@ -61,10 +65,10 @@ configuration. It called **Metadata Driven Data Workflow**.
 
 **:pushpin: <u>Rules of This Workflow engine</u>**:
 
-1. The Minimum frequency unit of scheduling is **1 minute** :warning:
-2. Can not re-run only failed stage and its pending downstream :rotating_light:
-3. All parallel tasks inside workflow engine use Multi-Threading
-   (🐍 Python 3.13 unlock GIL :unlock:)
+1. The Minimum frequency unit of scheduling is **1 Minute** 🕘
+2. **Can not** re-run only failed stage and its pending downstream ↩️
+3. All parallel tasks inside workflow engine use **Multi-Threading**
+   (Python 3.13 unlock GIL 🐍🔓)
 
 ---
 
@@ -165,6 +169,8 @@ run-py-local:
       run-date: datetime
    jobs:
       getting-api-data:
+         runs-on:
+            type: local
          stages:
             - name: "Retrieve API Data"
               id: retrieve-api