ddeutil-workflow 0.0.58__py3-none-any.whl → 0.0.60__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1 +1 @@
-__version__: str = "0.0.58"
+__version__: str = "0.0.60"

ddeutil/workflow/__cron.py

@@ -699,9 +699,9 @@ class CronJob:
 
     def schedule(
         self,
-        date: datetime | None = None,
+        date: Optional[datetime] = None,
         *,
-        tz: str | None = None,
+        tz: Optional[str] = None,
     ) -> CronRunner:
         """Returns CronRunner instance that be datetime runner with this
         cronjob. It can use `next`, `prev`, or `reset` methods to generate
@@ -766,7 +766,7 @@ class CronRunner:
     def __init__(
         self,
         cron: CronJob | CronJobYear,
-        date: datetime | None = None,
+        date: Optional[datetime] = None,
         *,
         tz: str | ZoneInfo | None = None,
     ) -> None:

ddeutil/workflow/__types.py

@@ -20,6 +20,7 @@ from typing import Any, Optional, TypedDict, Union
 
 from typing_extensions import Self
 
+StrOrNone = Optional[str]
 StrOrInt = Union[str, int]
 TupleStr = tuple[str, ...]
 DictData = dict[str, Any]
@@ -42,7 +43,7 @@ class CallerRe:
 
     full: str
     caller: str
-    caller_prefix: Optional[str]
+    caller_prefix: StrOrNone
     caller_last: str
     post_filters: str
 
@@ -50,6 +51,9 @@ class CallerRe:
     def from_regex(cls, match: Match[str]) -> Self:
         """Class construct from matching result.
 
+        :param match: A match string object for contract this Caller regex data
+            class.
+
         :rtype: Self
         """
         return cls(full=match.group(0), **match.groupdict())
@@ -121,10 +125,13 @@ class Re:
     )
 
     @classmethod
-    def finditer_caller(cls, value) -> Iterator[CallerRe]:
+    def finditer_caller(cls, value: str) -> Iterator[CallerRe]:
         """Generate CallerRe object that create from matching object that
         extract with re.finditer function.
 
+        :param value: (str) A string value that want to finditer with the caller
+            regular expression.
+
         :rtype: Iterator[CallerRe]
         """
         for found in cls.RE_CALLER.finditer(value):

ddeutil/workflow/conf.py

@@ -5,6 +5,7 @@
 # ------------------------------------------------------------------------------
 from __future__ import annotations
 
+import copy
 import json
 import os
 from abc import ABC, abstractmethod
@@ -26,13 +27,13 @@ T = TypeVar("T")
 PREFIX: Final[str] = "WORKFLOW"
 
 
-def env(var: str, default: str | None = None) -> str | None:
+def env(var: str, default: Optional[str] = None) -> Optional[str]:
     """Get environment variable with uppercase and adding prefix string.
 
     :param var: (str) A env variable name.
-    :param default: (str | None) A default value if an env var does not set.
+    :param default: (Optional[str]) A default value if an env var does not set.
 
-    :rtype: str | None
+    :rtype: Optional[str]
     """
     return os.getenv(f"{PREFIX}_{var.upper().replace(' ', '_')}", default)
 
@@ -298,6 +299,7 @@ class FileLoad(BaseLoad):
                 f"Multi-config paths does not support for type: {type(paths)}"
             )
         else:
+            paths: list[Path] = copy.deepcopy(paths)
             paths.append(path)
 
         all_data: list[tuple[float, DictData]] = []
@@ -398,7 +400,7 @@ class FileLoad(BaseLoad):
         return is_ignored(file, read_ignore(path / ignore_filename))
 
     @classmethod
-    def filter_yaml(cls, file: Path, name: str | None = None) -> DictData:
+    def filter_yaml(cls, file: Path, name: Optional[str] = None) -> DictData:
         """Read a YAML file context from an input file path and specific name.
 
         :param file: (Path) A file path that want to extract YAML context.

ddeutil/workflow/event.py

@@ -3,14 +3,15 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
-"""Event module that store all event object. Now, it has only `Crontab` and
-`CrontabYear` model these are schedule with crontab event.
+"""Event module include all event object for trigger the Workflow to release.
+Now, it has only `Crontab` and `CrontabYear` event models on this module because
+I think it is the core event for workflow orchestration.
 """
 from __future__ import annotations
 
 from dataclasses import fields
 from datetime import datetime
-from typing import Annotated, Any, Literal, Union
+from typing import Annotated, Any, Literal, Optional, Union
 from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
 
 from pydantic import BaseModel, ConfigDict, Field, ValidationInfo
@@ -28,7 +29,7 @@ Interval = Literal["daily", "weekly", "monthly"]
 def interval2crontab(
     interval: Interval,
     *,
-    day: str | None = None,
+    day: Optional[str] = None,
     time: str = "00:00",
 ) -> str:
     """Return the crontab string that was generated from specific values.
@@ -86,7 +87,7 @@ class Crontab(BaseModel):
         CronJob,
         Field(
             description=(
-                "A Cronjob object that use for validate and generate datetime.",
+                "A Cronjob object that use for validate and generate datetime."
             ),
         ),
     ]
@@ -117,7 +118,6 @@ class Crontab(BaseModel):
         passing["cronjob"] = interval2crontab(
             **{v: value[v] for v in value if v in ("interval", "day", "time")}
         )
-        print(passing)
         return cls(extras=extras | passing.pop("extras", {}), **passing)
 
     @classmethod
@@ -170,9 +170,10 @@ class Crontab(BaseModel):
 
     @model_validator(mode="before")
     def __prepare_values(cls, data: Any) -> Any:
-        """Extract tz key from value and change name to timezone key.
+        """Extract a `tz` key from data and change the key name from `tz` to
+        `timezone`.
 
-        :param data: (DictData) A data that want to pass for create an Crontab
+        :param data: (DictData) A data that want to pass for create a Crontab
             model.
 
         :rtype: DictData
@@ -198,7 +199,7 @@ class Crontab(BaseModel):
         "cronjob", mode="before", json_schema_input_type=Union[CronJob, str]
     )
     def __prepare_cronjob(
-        cls, value: str | CronJob, info: ValidationInfo
+        cls, value: Union[str, CronJob], info: ValidationInfo
     ) -> CronJob:
         """Prepare crontab value that able to receive with string type.
         This step will get options kwargs from extras field and pass to the
@@ -234,7 +235,7 @@ class Crontab(BaseModel):
         """
         return str(value)
 
-    def generate(self, start: str | datetime) -> CronRunner:
+    def generate(self, start: Union[str, datetime]) -> CronRunner:
         """Return CronRunner object from an initial datetime.
 
         :param start: (str | datetime) A string or datetime for generate the
@@ -248,7 +249,7 @@ class Crontab(BaseModel):
             raise TypeError("start value should be str or datetime type.")
         return self.cronjob.schedule(date=start, tz=self.tz)
 
-    def next(self, start: str | datetime) -> CronRunner:
+    def next(self, start: Union[str, datetime]) -> CronRunner:
         """Return a next datetime from Cron runner object that start with any
         date that given from input.
 
@@ -277,16 +278,18 @@ class CrontabYear(Crontab):
         CronJobYear,
         Field(
             description=(
-                "A Cronjob object that use for validate and generate datetime.",
+                "A Cronjob object that use for validate and generate datetime."
             ),
         ),
     ]
 
     @field_validator(
-        "cronjob", mode="before", json_schema_input_type=Union[CronJobYear, str]
+        "cronjob",
+        mode="before",
+        json_schema_input_type=Union[CronJobYear, str],
     )
     def __prepare_cronjob(
-        cls, value: str | CronJobYear, info: ValidationInfo
+        cls, value: Union[CronJobYear, str], info: ValidationInfo
     ) -> CronJobYear:
         """Prepare crontab value that able to receive with string type.
         This step will get options kwargs from extras field and pass to the

ddeutil/workflow/exceptions.py

@@ -9,7 +9,7 @@ annotate for handle error only.
 """
 from __future__ import annotations
 
-from typing import Literal, TypedDict, overload
+from typing import Literal, Optional, TypedDict, Union, overload
 
 
 class ErrorData(TypedDict):
@@ -39,9 +39,9 @@ class BaseWorkflowException(Exception):
     making an error context to the result context.
     """
 
-    def __init__(self, message: str, *, refs: str | None = None):
+    def __init__(self, message: str, *, refs: Optional[str] = None):
         super().__init__(message)
-        self.refs: str | None = refs
+        self.refs: Optional[str] = refs
 
     @overload
     def to_dict(
@@ -55,8 +55,9 @@ class BaseWorkflowException(Exception):
 
     def to_dict(
         self, with_refs: bool = False
-    ) -> ErrorData | dict[str, ErrorData]:
-        """Return ErrorData data from the current exception object.
+    ) -> Union[ErrorData, dict[str, ErrorData]]:
+        """Return ErrorData data from the current exception object. If with_refs
+        flag was set, it will return mapping of refs and itself data.
 
         :rtype: ErrorData
         """

ddeutil/workflow/job.py

@@ -19,6 +19,7 @@ from __future__ import annotations
 
 import copy
 import time
+from collections.abc import Iterator
 from concurrent.futures import (
     FIRST_EXCEPTION,
     CancelledError,
@@ -38,7 +39,7 @@ from pydantic import BaseModel, Discriminator, Field, SecretStr, Tag
 from pydantic.functional_validators import field_validator, model_validator
 from typing_extensions import Self
 
-from .__types import DictData, DictStr, Matrix
+from .__types import DictData, DictStr, Matrix, StrOrNone
 from .exceptions import (
     JobException,
     StageException,
@@ -67,8 +68,8 @@ def make(
 
     :param matrix: (Matrix) A matrix values that want to cross product to
         possible parallelism values.
-    :param include: (A list of additional matrix that want to adds-in.
-    :param exclude: (A list of exclude matrix that want to filter-out.
+    :param include: A list of additional matrix that want to adds-in.
+    :param exclude: A list of exclude matrix that want to filter-out.
 
     :rtype: list[DictStr]
     """
@@ -191,22 +192,22 @@ class Strategy(BaseModel):
 class Rule(str, Enum):
     """Rule enum object for assign trigger option."""
 
-    ALL_SUCCESS: str = "all_success"
-    ALL_FAILED: str = "all_failed"
-    ALL_DONE: str = "all_done"
-    ONE_FAILED: str = "one_failed"
-    ONE_SUCCESS: str = "one_success"
-    NONE_FAILED: str = "none_failed"
-    NONE_SKIPPED: str = "none_skipped"
+    ALL_SUCCESS = "all_success"
+    ALL_FAILED = "all_failed"
+    ALL_DONE = "all_done"
+    ONE_FAILED = "one_failed"
+    ONE_SUCCESS = "one_success"
+    NONE_FAILED = "none_failed"
+    NONE_SKIPPED = "none_skipped"
 
 
 class RunsOn(str, Enum):
     """Runs-On enum object."""
 
-    LOCAL: str = "local"
-    SELF_HOSTED: str = "self_hosted"
-    AZ_BATCH: str = "azure_batch"
-    DOCKER: str = "docker"
+    LOCAL = "local"
+    SELF_HOSTED = "self_hosted"
+    AZ_BATCH = "azure_batch"
+    DOCKER = "docker"
 
 
 class BaseRunsOn(BaseModel):  # pragma: no cov
@@ -328,14 +329,14 @@ class Job(BaseModel):
         ... }
     """
 
-    id: Optional[str] = Field(
+    id: StrOrNone = Field(
         default=None,
         description=(
             "A job ID that was set from Workflow model after initialize step. "
             "If this model create standalone, it will be None."
         ),
     )
-    desc: Optional[str] = Field(
+    desc: StrOrNone = Field(
         default=None,
         description="A job description that can be markdown syntax.",
     )
@@ -344,7 +345,7 @@ class Job(BaseModel):
         description="A target node for this job to use for execution.",
         alias="runs-on",
     )
-    condition: Optional[str] = Field(
+    condition: StrOrNone = Field(
         default=None,
         description="A job condition statement to allow job executable.",
         alias="if",
@@ -525,7 +526,7 @@ class Job(BaseModel):
         output: DictData,
         to: DictData,
         *,
-        job_id: Optional[str] = None,
+        job_id: StrOrNone = None,
     ) -> DictData:
         """Set an outputs from execution result context to the received context
         with a `to` input parameter. The result context from job strategy
@@ -566,7 +567,7 @@ class Job(BaseModel):
         :param output: (DictData) A result data context that want to extract
             and transfer to the `strategies` key in receive context.
         :param to: (DictData) A received context data.
-        :param job_id: (str | None) A job ID if the `id` field does not set.
+        :param job_id: (StrOrNone) A job ID if the `id` field does not set.
 
         :rtype: DictData
         """
@@ -606,9 +607,9 @@ class Job(BaseModel):
         self,
         params: DictData,
         *,
-        run_id: str | None = None,
-        parent_run_id: str | None = None,
-        event: Event | None = None,
+        run_id: StrOrNone = None,
+        parent_run_id: StrOrNone = None,
+        event: Optional[Event] = None,
     ) -> Result:
         """Job execution with passing dynamic parameters from the workflow
         execution. It will generate matrix values at the first step and run
@@ -676,8 +677,8 @@ def local_execute_strategy(
     strategy: DictData,
     params: DictData,
     *,
-    result: Result | None = None,
-    event: Event | None = None,
+    result: Optional[Result] = None,
+    event: Optional[Event] = None,
 ) -> Result:
     """Local strategy execution with passing dynamic parameters from the
     job execution and strategy matrix.
@@ -799,9 +800,9 @@ def local_execute(
     job: Job,
     params: DictData,
     *,
-    run_id: str | None = None,
-    parent_run_id: str | None = None,
-    event: Event | None = None,
+    run_id: StrOrNone = None,
+    parent_run_id: StrOrNone = None,
+    event: Optional[Event] = None,
 ) -> Result:
     """Local job execution with passing dynamic parameters from the workflow
     execution or directly. It will generate matrix values at the first
@@ -874,10 +875,10 @@ def local_execute(
         status: Status = SUCCESS
 
         if not fail_fast_flag:
-            done: list[Future] = as_completed(futures)
+            done: Iterator[Future] = as_completed(futures)
         else:
             done, not_done = wait(futures, return_when=FIRST_EXCEPTION)
-            if len(done) != len(futures):
+            if len(list(done)) != len(futures):
                 result.trace.warning(
                     "[JOB]: Handler Fail-Fast: Got exception and set event."
                 )
@@ -895,7 +896,7 @@ def local_execute(
                 else ""
             )
             result.trace.debug(f"[JOB]: ... Job was set Fail-Fast{nd}")
-            done: list[Future] = as_completed(futures)
+            done: Iterator[Future] = as_completed(futures)
 
         for future in done:
             try:
@@ -918,9 +919,9 @@ def self_hosted_execute(
     job: Job,
     params: DictData,
     *,
-    run_id: str | None = None,
-    parent_run_id: str | None = None,
-    event: Event | None = None,
+    run_id: StrOrNone = None,
+    parent_run_id: StrOrNone = None,
+    event: Optional[Event] = None,
 ) -> Result:  # pragma: no cov
     """Self-Hosted job execution with passing dynamic parameters from the
     workflow execution or itself execution. It will make request to the
@@ -981,9 +982,9 @@ def azure_batch_execute(
     job: Job,
     params: DictData,
     *,
-    run_id: str | None = None,
-    parent_run_id: str | None = None,
-    event: Event | None = None,
+    run_id: StrOrNone = None,
+    parent_run_id: StrOrNone = None,
+    event: Optional[Event] = None,
 ) -> Result:  # pragma: no cov
     """Azure Batch job execution that will run all job's stages on the Azure
     Batch Node and extract the result file to be returning context result.
@@ -1035,9 +1036,9 @@ def docker_execution(
     job: Job,
     params: DictData,
     *,
-    run_id: str | None = None,
-    parent_run_id: str | None = None,
-    event: Event | None = None,
+    run_id: StrOrNone = None,
+    parent_run_id: StrOrNone = None,
+    event: Optional[Event] = None,
 ):  # pragma: no cov
     """Docker job execution.