ddeutil-workflow 0.0.56__py3-none-any.whl → 0.0.58__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1 +1 @@
-__version__: str = "0.0.56"
+__version__: str = "0.0.58"

ddeutil/workflow/__cron.py

@@ -502,10 +502,10 @@ class CronPart:
             except IndexError:
                 next_value: int = -1
             if value != (next_value - 1):
-                # NOTE: ``next_value`` is not the subsequent number
+                # NOTE: `next_value` is not the subsequent number
                 if start_number is None:
                     # NOTE:
-                    #   The last number of the list ``self.values`` is not in a
+                    #   The last number of the list `self.values` is not in a
                     #   range.
                     multi_dim_values.append(value)
                 else:
@@ -703,11 +703,14 @@ class CronJob:
         *,
         tz: str | None = None,
     ) -> CronRunner:
-        """Returns the schedule datetime runner with this cronjob. It would run
-        ``next``, ``prev``, or ``reset`` to generate running date that you want.
+        """Returns CronRunner instance that be datetime runner with this
+        cronjob. It can use `next`, `prev`, or `reset` methods to generate
+        running date.
 
-        :param date: An initial date that want to mark as the start point.
-        :param tz: A string timezone that want to change on runner.
+        :param date: (datetime) An initial date that want to mark as the start
+            point. (Default is use the current datetime)
+        :param tz: (str) A string timezone that want to change on runner.
+            (Default is None)
 
         :rtype: CronRunner
         """
@@ -743,6 +746,10 @@ class CronJobYear(CronJob):
 class CronRunner:
     """Create an instance of Date Runner object for datetime generate with
     cron schedule object value.
+
+    :param cron: (CronJob | CronJobYear)
+    :param date: (datetime)
+    :param tz: (str)
     """
 
     shift_limit: ClassVar[int] = 25
@@ -761,11 +768,17 @@ class CronRunner:
         cron: CronJob | CronJobYear,
         date: datetime | None = None,
         *,
-        tz: str | None = None,
+        tz: str | ZoneInfo | None = None,
     ) -> None:
-        # NOTE: Prepare timezone if this value does not set, it will use UTC.
-        self.tz: ZoneInfo = ZoneInfo("UTC")
+        self.tz: ZoneInfo | None = None
         if tz:
+            if isinstance(tz, ZoneInfo):
+                self.tz = tz
+            elif not isinstance(tz, str):
+                raise TypeError(
+                    "Invalid type of `tz` parameter, it should be str or "
+                    "ZoneInfo instance."
+                )
             try:
                 self.tz = ZoneInfo(tz)
             except ZoneInfoNotFoundError as err:
@@ -777,9 +790,10 @@ class CronRunner:
                 raise ValueError(
                     "Input schedule start time is not a valid datetime object."
                 )
-            if tz is None:
-                self.tz = date.tzinfo
-            self.date: datetime = date.astimezone(self.tz)
+            if tz is not None:
+                self.date: datetime = date.astimezone(self.tz)
+            else:
+                self.date: datetime = date
         else:
             self.date: datetime = datetime.now(tz=self.tz)
 

ddeutil/workflow/__types.py

@@ -20,6 +20,7 @@ from typing import Any, Optional, TypedDict, Union
 
 from typing_extensions import Self
 
+StrOrInt = Union[str, int]
 TupleStr = tuple[str, ...]
 DictData = dict[str, Any]
 DictStr = dict[str, str]

ddeutil/workflow/conf.py

@@ -200,7 +200,10 @@ class APIConfig:
         return str2bool(env("API_ENABLE_ROUTE_SCHEDULE", "true"))
 
 
-class BaseLoad(ABC):
+class BaseLoad(ABC):  # pragma: no cov
+    """Base Load object is the abstraction object for any Load object that
+    should to inherit from this base class.
+    """
 
     @classmethod
     @abstractmethod
@@ -215,7 +218,7 @@ class BaseLoad(ABC):
 
 class FileLoad(BaseLoad):
     """Base Load object that use to search config data by given some identity
-    value like name of `Workflow` or `On` templates.
+    value like name of `Workflow` or `Crontab` templates.
 
     :param name: (str) A name of key of config data that read with YAML
         Environment object.
@@ -335,8 +338,13 @@ class FileLoad(BaseLoad):
         """
         excluded: list[str] = excluded or []
         path: Path = dynamic("conf_path", f=path, extras=extras)
+        paths: Optional[list[Path]] = paths or (extras or {}).get("conf_paths")
         if not paths:
             paths: list[Path] = [path]
+        elif not isinstance(paths, list):
+            raise TypeError(
+                f"Multi-config paths does not support for type: {type(paths)}"
+            )
         else:
             paths.append(path)
 
@@ -431,17 +439,21 @@ def dynamic(
     """Dynamic get config if extra value was passed at run-time.
 
     :param key: (str) A config key that get from Config object.
-    :param f: An inner config function scope.
+    :param f: (T) An inner config function scope.
     :param extras: An extra values that pass at run-time.
+
+    :rtype: T
     """
-    rsx: Optional[T] = extras[key] if extras and key in extras else None
-    rs: Optional[T] = getattr(config, key, None) if f is None else f
-    if rsx is not None and not isinstance(rsx, type(rs)):
+    extra: Optional[T] = (extras or {}).get(key, None)
+    conf: Optional[T] = getattr(config, key, None) if f is None else f
+    if extra is None:
+        return conf
+    if not isinstance(extra, type(conf)):
         raise TypeError(
-            f"Type of config {key!r} from extras: {rsx!r} does not valid "
-            f"as config {type(rs)}."
+            f"Type of config {key!r} from extras: {extra!r} does not valid "
+            f"as config {type(conf)}."
         )
-    return rsx if rsx is not None else rs
+    return extra
 
 
 class Loader(Protocol):  # pragma: no cov

ddeutil/workflow/event.py

@@ -3,8 +3,8 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
-"""Event module that store all event object. Now, it has only `On` and `OnYear`
-model these are schedule with crontab event.
+"""Event module that store all event object. Now, it has only `Crontab` and
+`CrontabYear` model these are schedule with crontab event.
 """
 from __future__ import annotations
 
@@ -63,9 +63,9 @@ def interval2crontab(
     return f"{h} {m} {'1' if interval == 'monthly' else '*'} * {d}"
 
 
-class On(BaseModel):
-    """On model (Warped crontab object by Pydantic model) to keep crontab value
-    and generate CronRunner object from this crontab value.
+class Crontab(BaseModel):
+    """Cron event model (Warped the CronJob object by Pydantic model) to keep
+    crontab value and generate CronRunner object from this crontab value.
 
     Methods:
         - generate: is the main use-case of this schedule object.
@@ -117,6 +117,7 @@ class On(BaseModel):
         passing["cronjob"] = interval2crontab(
             **{v: value[v] for v in value if v in ("interval", "day", "time")}
         )
+        print(passing)
         return cls(extras=extras | passing.pop("extras", {}), **passing)
 
     @classmethod
@@ -127,7 +128,7 @@ class On(BaseModel):
         extras: DictData | None = None,
     ) -> Self:
         """Constructor from the name of config loader that will use loader
-        object for getting the `On` data.
+        object for getting the `Crontab` data.
 
         :param name: (str) A name of config that will get from loader.
         :param extras: (DictData) An extra parameter that use to override core
@@ -171,7 +172,7 @@ class On(BaseModel):
     def __prepare_values(cls, data: Any) -> Any:
         """Extract tz key from value and change name to timezone key.
 
-        :param data: (DictData) A data that want to pass for create an On
+        :param data: (DictData) A data that want to pass for create an Crontab
             model.
 
         :rtype: DictData
@@ -264,9 +265,9 @@ class On(BaseModel):
         return runner
 
 
-class YearOn(On):
-    """On with enhance Year Pydantic model for limit year matrix that use by
-    some data schedule tools like AWS Glue.
+class CrontabYear(Crontab):
+    """Cron event with enhance Year Pydantic model for limit year matrix that
+    use by some data schedule tools like AWS Glue.
     """
 
     model_config = ConfigDict(arbitrary_types_allowed=True)

ddeutil/workflow/exceptions.py

@@ -9,16 +9,16 @@ annotate for handle error only.
 """
 from __future__ import annotations
 
-from typing import TypedDict
+from typing import Literal, TypedDict, overload
 
-ErrorData = TypedDict(
-    "ErrorData",
-    {
-        "class": Exception,
-        "name": str,
-        "message": str,
-    },
-)
+
+class ErrorData(TypedDict):
+    """Error data type dict for typing necessary keys of return of to_dict func
+    and method.
+    """
+
+    name: str
+    message: str
 
 
 def to_dict(exception: Exception) -> ErrorData:  # pragma: no cov
@@ -29,20 +29,41 @@ def to_dict(exception: Exception) -> ErrorData:  # pragma: no cov
     :rtype: ErrorData
     """
     return {
-        "class": exception,
         "name": exception.__class__.__name__,
         "message": str(exception),
     }
 
 
 class BaseWorkflowException(Exception):
+    """Base Workflow exception class will implement the `refs` argument for
+    making an error context to the result context.
+    """
+
+    def __init__(self, message: str, *, refs: str | None = None):
+        super().__init__(message)
+        self.refs: str | None = refs
+
+    @overload
+    def to_dict(
+        self, with_refs: Literal[True] = ...
+    ) -> dict[str, ErrorData]: ...  # pragma: no cov
+
+    @overload
+    def to_dict(
+        self, with_refs: Literal[False] = ...
+    ) -> ErrorData: ...  # pragma: no cov
 
-    def to_dict(self) -> ErrorData:
+    def to_dict(
+        self, with_refs: bool = False
+    ) -> ErrorData | dict[str, ErrorData]:
         """Return ErrorData data from the current exception object.
 
         :rtype: ErrorData
         """
-        return to_dict(self)
+        data: ErrorData = to_dict(self)
+        if with_refs and (self.refs is not None and self.refs != "EMPTY"):
+            return {self.refs: data}
+        return data
 
 
 class UtilException(BaseWorkflowException): ...

ddeutil/workflow/job.py

@@ -18,8 +18,10 @@ method.
 from __future__ import annotations
 
 import copy
+import time
 from concurrent.futures import (
     FIRST_EXCEPTION,
+    CancelledError,
     Future,
     ThreadPoolExecutor,
     as_completed,
@@ -40,13 +42,12 @@ from .__types import DictData, DictStr, Matrix
 from .exceptions import (
     JobException,
     StageException,
-    UtilException,
     to_dict,
 )
 from .result import CANCEL, FAILED, SKIP, SUCCESS, WAIT, Result, Status
 from .reusables import has_template, param2template
 from .stages import Stage
-from .utils import NEWLINE, cross_product, filter_func, gen_id
+from .utils import cross_product, filter_func, gen_id
 
 MatrixFilter = list[dict[str, Union[str, int]]]
 
@@ -380,7 +381,7 @@ class Job(BaseModel):
 
         :rtype: str
         """
-        return dedent(value)
+        return dedent(value.lstrip("\n"))
 
     @field_validator("stages", mode="after")
     def __validate_stage_id__(cls, value: list[Stage]) -> list[Stage]:
@@ -429,11 +430,14 @@ class Job(BaseModel):
                 return stage
         raise ValueError(f"Stage {stage_id!r} does not exists in this job.")
 
-    def check_needs(self, jobs: dict[str, Any]) -> Status:  # pragma: no cov
+    def check_needs(
+        self, jobs: dict[str, DictData]
+    ) -> Status:  # pragma: no cov
         """Return trigger status from checking job's need trigger rule logic was
         valid. The return status should be SUCCESS, FAILED, WAIT, or SKIP.
 
-        :param jobs: A mapping of job ID and its context data.
+        :param jobs: (dict[str, DictData]) A mapping of job ID and its context
+            data that return from execution process.
 
         :raise NotImplementedError: If the job trigger rule out of scope.
 
@@ -450,28 +454,34 @@ class Job(BaseModel):
         }
         if len(need_exist) != len(self.needs):
             return WAIT
-        elif all("skipped" in need_exist[job] for job in need_exist):
+        elif all(need_exist[job].get("skipped", False) for job in need_exist):
             return SKIP
         elif self.trigger_rule == Rule.ALL_DONE:
             return SUCCESS
         elif self.trigger_rule == Rule.ALL_SUCCESS:
             rs = all(
-                k not in need_exist[job]
-                for k in ("errors", "skipped")
+                (
+                    "errors" not in need_exist[job]
+                    and not need_exist[job].get("skipped", False)
+                )
                 for job in need_exist
             )
         elif self.trigger_rule == Rule.ALL_FAILED:
             rs = all("errors" in need_exist[job] for job in need_exist)
         elif self.trigger_rule == Rule.ONE_SUCCESS:
             rs = sum(
-                k not in need_exist[job]
-                for k in ("errors", "skipped")
+                (
+                    "errors" not in need_exist[job]
+                    and not need_exist[job].get("skipped", False)
+                )
                 for job in need_exist
             ) + 1 == len(self.needs)
         elif self.trigger_rule == Rule.ONE_FAILED:
             rs = sum("errors" in need_exist[job] for job in need_exist) == 1
         elif self.trigger_rule == Rule.NONE_SKIPPED:
-            rs = all("skipped" not in need_exist[job] for job in need_exist)
+            rs = all(
+                not need_exist[job].get("skipped", False) for job in need_exist
+            )
         elif self.trigger_rule == Rule.NONE_FAILED:
             rs = all("errors" not in need_exist[job] for job in need_exist)
         else:  # pragma: no cov
@@ -613,20 +623,19 @@ class Job(BaseModel):
         :param event: (Event) An Event manager instance that use to cancel this
             execution if it forces stopped by parent execution.
 
-        :raise NotImplementedError: If the `runs-on` value does not implement on
-            this execution.
-
         :rtype: Result
         """
         result: Result = Result.construct_with_rs_or_id(
             run_id=run_id,
             parent_run_id=parent_run_id,
-            id_logic=(self.id or "not-set"),
+            id_logic=(self.id or "EMPTY"),
             extras=self.extras,
         )
 
         result.trace.info(
-            f"[JOB]: Execute: {self.id!r} on {self.runs_on.type.value!r}"
+            f"[JOB]: Execute "
+            f"{''.join(self.runs_on.type.value.split('_')).title()}: "
+            f"{self.id!r}"
         )
         if self.runs_on.type == RunsOn.LOCAL:
             return local_execute(
@@ -647,12 +656,18 @@ class Job(BaseModel):
                 event=event,
             )
 
-        # pragma: no cov
         result.trace.error(
-            f"[JOB]: Execute not support runs-on: {self.runs_on.type!r} yet."
+            f"[JOB]: Execute not support runs-on: {self.runs_on.type.value!r} "
+            f"yet."
         )
-        raise NotImplementedError(
-            f"Execute runs-on type: {self.runs_on.type} does not support yet."
+        return result.catch(
+            status=FAILED,
+            context={
+                "errors": JobException(
+                    f"Execute runs-on type: {self.runs_on.type.value!r} does "
+                    f"not support yet."
+                ).to_dict(),
+            },
         )
 
 
@@ -664,10 +679,10 @@ def local_execute_strategy(
     result: Result | None = None,
     event: Event | None = None,
 ) -> Result:
-    """Local job strategy execution with passing dynamic parameters from the
-    workflow execution to strategy matrix.
+    """Local strategy execution with passing dynamic parameters from the
+    job execution and strategy matrix.
 
-        This execution is the minimum level of execution of this job model.
+        This execution is the minimum level of job execution.
     It different with `self.execute` because this method run only one
     strategy and return with context of this strategy data.
 
@@ -684,22 +699,22 @@ def local_execute_strategy(
     :param event: (Event) An Event manager instance that use to cancel this
         execution if it forces stopped by parent execution.
 
-    :raise JobException: If stage execution raise any error as `StageException`
-        or `UtilException`.
+    :raise JobException: If event was set.
+    :raise JobException: If stage execution raise any error as `StageException`.
+    :raise JobException: If the result from execution has `FAILED` status.
 
     :rtype: Result
     """
     result: Result = result or Result(
-        run_id=gen_id(job.id or "not-set", unique=True),
+        run_id=gen_id(job.id or "EMPTY", unique=True),
         extras=job.extras,
     )
     if strategy:
         strategy_id: str = gen_id(strategy)
-        result.trace.info(f"[JOB]: Start Strategy: {strategy_id!r}")
+        result.trace.info(f"[JOB]: Execute Strategy: {strategy_id!r}")
         result.trace.info(f"[JOB]: ... matrix: {strategy!r}")
     else:
         strategy_id: str = "EMPTY"
-        result.trace.info("[JOB]: Start Strategy: 'EMPTY'")
 
     context: DictData = copy.deepcopy(params)
     context.update({"matrix": strategy, "stages": {}})
@@ -714,11 +729,8 @@ def local_execute_strategy(
             continue
 
         if event and event.is_set():
-            error_msg: str = (
-                "Job strategy was canceled from event that had set before "
-                "job strategy execution."
-            )
-            return result.catch(
+            error_msg: str = "Job strategy was canceled because event was set."
+            result.catch(
                 status=CANCEL,
                 context={
                     strategy_id: {
@@ -728,6 +740,7 @@ def local_execute_strategy(
                     },
                 },
             )
+            raise JobException(error_msg, refs=strategy_id)
 
         try:
             result.trace.info(f"[JOB]: Execute Stage: {stage.iden!r}")
@@ -738,8 +751,7 @@ def local_execute_strategy(
                 event=event,
             )
             stage.set_outputs(rs.context, to=context)
-        except (StageException, UtilException) as e:
-            result.trace.error(f"[JOB]: {e.__class__.__name__}: {e}")
+        except StageException as e:
             result.catch(
                 status=FAILED,
                 context={
@@ -751,15 +763,15 @@ def local_execute_strategy(
                 },
             )
             raise JobException(
-                f"Stage raise: {e.__class__.__name__}: {e}"
+                message=f"Handler Error: {e.__class__.__name__}: {e}",
+                refs=strategy_id,
             ) from e
 
         if rs.status == FAILED:
             error_msg: str = (
-                f"Strategy break because stage, {stage.iden!r}, return FAILED "
-                f"status."
+                f"Strategy break because stage, {stage.iden!r}, return "
+                f"`FAILED` status."
             )
-            result.trace.warning(f"[JOB]: {error_msg}")
             result.catch(
                 status=FAILED,
                 context={
@@ -770,7 +782,7 @@ def local_execute_strategy(
                     },
                 },
             )
-            raise JobException(error_msg)
+            raise JobException(error_msg, refs=strategy_id)
 
     return result.catch(
         status=SUCCESS,
@@ -792,11 +804,19 @@ def local_execute(
     event: Event | None = None,
 ) -> Result:
     """Local job execution with passing dynamic parameters from the workflow
-    execution or itself execution. It will generate matrix values at the first
+    execution or directly. It will generate matrix values at the first
     step and run multithread on this metrics to the `stages` field of this job.
 
-        This method does not raise any `JobException` if it runs with
-    multi-threading strategy.
+    Important:
+        This method does not raise any `JobException` because it allows run
+    parallel mode. If it raises error from strategy execution, it will catch
+    that error and store it in the `errors` key with list of error.
+
+        {
+            "errors": [
+                {"name": "...", "message": "..."}, ...
+            ]
+        }
 
     :param job: (Job) A job model.
     :param params: (DictData) A parameter data.
@@ -810,20 +830,20 @@ def local_execute(
     result: Result = Result.construct_with_rs_or_id(
         run_id=run_id,
         parent_run_id=parent_run_id,
-        id_logic=(job.id or "not-set"),
+        id_logic=(job.id or "EMPTY"),
         extras=job.extras,
     )
 
-    event: Event = Event() if event is None else event
+    event: Event = event or Event()
     fail_fast_flag: bool = job.strategy.fail_fast
     ls: str = "Fail-Fast" if fail_fast_flag else "All-Completed"
     workers: int = job.strategy.max_parallel
     result.trace.info(
-        f"[JOB]: {ls}-Execute: {job.id} with {workers} "
+        f"[JOB]: Execute {ls}: {job.id!r} with {workers} "
         f"worker{'s' if workers > 1 else ''}."
     )
 
-    if event and event.is_set():  # pragma: no cov
+    if event and event.is_set():
         return result.catch(
             status=CANCEL,
             context={
@@ -859,14 +879,23 @@ def local_execute(
             done, not_done = wait(futures, return_when=FIRST_EXCEPTION)
             if len(done) != len(futures):
                 result.trace.warning(
-                    "[JOB]: Set event for stop pending stage future."
+                    "[JOB]: Handler Fail-Fast: Got exception and set event."
                 )
                 event.set()
                 for future in not_done:
                     future.cancel()
+                time.sleep(0.075)
 
-            nd: str = f", strategies not run: {not_done}" if not_done else ""
-            result.trace.debug(f"... Strategy set Fail-Fast{nd}")
+            nd: str = (
+                (
+                    f", {len(not_done)} strateg"
+                    f"{'ies' if len(not_done) > 1 else 'y'} not run!!!"
+                )
+                if not_done
+                else ""
+            )
+            result.trace.debug(f"[JOB]: ... Job was set Fail-Fast{nd}")
+            done: list[Future] = as_completed(futures)
 
         for future in done:
             try:
@@ -874,12 +903,14 @@ def local_execute(
             except JobException as e:
                 status = FAILED
                 result.trace.error(
-                    f"[JOB]: {ls}: {e.__class__.__name__}:{NEWLINE}{e}"
+                    f"[JOB]: {ls} Error Handler:||{e.__class__.__name__}:||{e}"
                 )
                 if "errors" in context:
-                    context["errors"].append(e.to_dict())
+                    context["errors"][e.refs] = e.to_dict()
                 else:
-                    context["errors"] = [e.to_dict()]
+                    context["errors"] = e.to_dict(with_refs=True)
+            except CancelledError:
+                pass
     return result.catch(status=status, context=context)
 
 
@@ -907,7 +938,7 @@ def self_hosted_execute(
     result: Result = Result.construct_with_rs_or_id(
         run_id=run_id,
         parent_run_id=parent_run_id,
-        id_logic=(job.id or "not-set"),
+        id_logic=(job.id or "EMPTY"),
         extras=job.extras,
     )
 
@@ -953,7 +984,7 @@ def azure_batch_execute(
     run_id: str | None = None,
     parent_run_id: str | None = None,
     event: Event | None = None,
-) -> Result:  # pragma no cov
+) -> Result:  # pragma: no cov
     """Azure Batch job execution that will run all job's stages on the Azure
     Batch Node and extract the result file to be returning context result.
 
@@ -983,7 +1014,7 @@ def azure_batch_execute(
     result: Result = Result.construct_with_rs_or_id(
         run_id=run_id,
         parent_run_id=parent_run_id,
-        id_logic=(job.id or "not-set"),
+        id_logic=(job.id or "EMPTY"),
         extras=job.extras,
     )
     if event and event.is_set():
@@ -1007,7 +1038,7 @@ def docker_execution(
     run_id: str | None = None,
     parent_run_id: str | None = None,
     event: Event | None = None,
-):
+):  # pragma: no cov
     """Docker job execution.
 
     Steps:
@@ -1018,7 +1049,7 @@ def docker_execution(
     result: Result = Result.construct_with_rs_or_id(
         run_id=run_id,
         parent_run_id=parent_run_id,
-        id_logic=(job.id or "not-set"),
+        id_logic=(job.id or "EMPTY"),
         extras=job.extras,
     )
     if event and event.is_set():