ddeutil-workflow 0.0.54__py3-none-any.whl → 0.0.56__py3-none-any.whl

ddeutil/workflow/conf.py

@@ -7,24 +7,26 @@ from __future__ import annotations
 
 import json
 import os
+from abc import ABC, abstractmethod
 from collections.abc import Iterator
 from datetime import timedelta
 from functools import cached_property
+from inspect import isclass
 from pathlib import Path
-from typing import Final, Optional, TypeVar
+from typing import Final, Optional, Protocol, TypeVar, Union
 from zoneinfo import ZoneInfo
 
 from ddeutil.core import str2bool
 from ddeutil.io import YamlFlResolve
 from ddeutil.io.paths import glob_files, is_ignored, read_ignore
 
-from .__types import DictData, TupleStr
+from .__types import DictData
 
 T = TypeVar("T")
 PREFIX: Final[str] = "WORKFLOW"
 
 
-def env(var: str, default: str | None = None) -> str | None:  # pragma: no cov
+def env(var: str, default: str | None = None) -> str | None:
     """Get environment variable with uppercase and adding prefix string.
 
     :param var: (str) A env variable name.
@@ -35,17 +37,6 @@ def env(var: str, default: str | None = None) -> str | None:  # pragma: no cov
     return os.getenv(f"{PREFIX}_{var.upper().replace(' ', '_')}", default)
 
 
-__all__: TupleStr = (
-    "api_config",
-    "env",
-    "Config",
-    "SimLoad",
-    "Loader",
-    "config",
-    "dynamic",
-)
-
-
 class Config:  # pragma: no cov
     """Config object for keeping core configurations on the current session
     without changing when if the application still running.
@@ -188,7 +179,7 @@ class Config:  # pragma: no cov
             return timedelta(**json.loads(stop_boundary_delta_str))
         except Exception as err:
             raise ValueError(
-                "Config ``WORKFLOW_APP_STOP_BOUNDARY_DELTA`` can not parsing to"
+                "Config `WORKFLOW_APP_STOP_BOUNDARY_DELTA` can not parsing to"
                 f"timedelta with {stop_boundary_delta_str}."
             ) from err
 
@@ -209,110 +200,194 @@ class APIConfig:
         return str2bool(env("API_ENABLE_ROUTE_SCHEDULE", "true"))
 
 
-class SimLoad:
-    """Simple Load Object that will search config data by given some identity
-    value like name of workflow or on.
+class BaseLoad(ABC):
+
+    @classmethod
+    @abstractmethod
+    def find(cls, name: str, *args, **kwargs) -> DictData: ...
+
+    @classmethod
+    @abstractmethod
+    def finds(
+        cls, obj: object, *args, **kwargs
+    ) -> Iterator[tuple[str, DictData]]: ...
 
-    :param name: A name of config data that will read by Yaml Loader object.
-    :param conf_path: A config path object.
-    :param externals: An external parameters
+
+class FileLoad(BaseLoad):
+    """Base Load object that use to search config data by given some identity
+    value like name of `Workflow` or `On` templates.
+
+    :param name: (str) A name of key of config data that read with YAML
+        Environment object.
+    :param path: (Path) A config path object.
+    :param externals: (DictData) An external config data that want to add to
+        loaded config data.
+    :param extras: (DictDdata) An extra parameters that use to override core
+        config values.
+
+    :raise ValueError: If the data does not find on the config path with the
+        name parameter.
 
     Noted:
-        The config data should have ``type`` key for modeling validation that
+        The config data should have `type` key for modeling validation that
     make this loader know what is config should to do pass to.
 
         ... <identity-key>:
         ...     type: <importable-object>
         ...     <key-data-1>: <value-data-1>
         ...     <key-data-2>: <value-data-2>
+
+        This object support multiple config paths if you pass the `conf_paths`
+    key to the `extras` parameter.
     """
 
     def __init__(
         self,
         name: str,
-        conf_path: Path,
+        *,
+        path: Optional[Union[str, Path]] = None,
         externals: DictData | None = None,
+        extras: DictData | None = None,
     ) -> None:
-        self.conf_path: Path = conf_path
+        self.path: Path = Path(dynamic("conf_path", f=path, extras=extras))
         self.externals: DictData = externals or {}
-
-        self.data: DictData = {}
-        for file in glob_files(conf_path):
-
-            if self.is_ignore(file, conf_path):
-                continue
-
-            if data := self.filter_yaml(file, name=name):
-                self.data = data
+        self.extras: DictData = extras or {}
+        self.data: DictData = self.find(
+            name,
+            path=path,
+            paths=self.extras.get("conf_paths"),
+            extras=extras,
+        )
 
         # VALIDATE: check the data that reading should not empty.
         if not self.data:
             raise ValueError(
-                f"Config {name!r} does not found on conf path: "
-                f"{self.conf_path}."
+                f"Config {name!r} does not found on the conf path: {self.path}."
             )
 
         self.data.update(self.externals)
 
+    @classmethod
+    def find(
+        cls,
+        name: str,
+        *,
+        path: Optional[Path] = None,
+        paths: Optional[list[Path]] = None,
+        extras: Optional[DictData] = None,
+    ) -> DictData:
+        """Find data with specific key and return the latest modify date data if
+        this key exists multiple files.
+
+        :param name: (str) A name of data that want to find.
+        :param path: (Path) A config path object.
+        :param paths: (list[Path]) A list of config path object.
+        :param extras: (DictData)  An extra parameter that use to override core
+            config values.
+
+        :rtype: DictData
+        """
+        path: Path = dynamic("conf_path", f=path, extras=extras)
+        if not paths:
+            paths: list[Path] = [path]
+        elif not isinstance(paths, list):
+            raise TypeError(
+                f"Multi-config paths does not support for type: {type(paths)}"
+            )
+        else:
+            paths.append(path)
+
+        all_data: list[tuple[float, DictData]] = []
+        for path in paths:
+            for file in glob_files(path):
+
+                if cls.is_ignore(file, path):
+                    continue
+
+                if data := cls.filter_yaml(file, name=name):
+                    all_data.append((file.lstat().st_mtime, data))
+
+        return {} if not all_data else max(all_data, key=lambda x: x[0])[1]
+
     @classmethod
     def finds(
         cls,
         obj: object,
-        conf_path: Path,
         *,
-        included: list[str] | None = None,
+        path: Optional[Path] = None,
+        paths: Optional[list[Path]] = None,
         excluded: list[str] | None = None,
+        extras: Optional[DictData] = None,
     ) -> Iterator[tuple[str, DictData]]:
         """Find all data that match with object type in config path. This class
         method can use include and exclude list of identity name for filter and
         adds-on.
 
         :param obj: An object that want to validate matching before return.
-        :param conf_path: A config object.
-        :param included: An excluded list of data key that want to reject this
-            data if any key exist.
+        :param path: A config path object.
+        :param paths: (list[Path]) A list of config path object.
         :param excluded: An included list of data key that want to filter from
             data.
+        :param extras: (DictData) An extra parameter that use to override core
+            config values.
 
         :rtype: Iterator[tuple[str, DictData]]
         """
-        exclude: list[str] = excluded or []
-        for file in glob_files(conf_path):
+        excluded: list[str] = excluded or []
+        path: Path = dynamic("conf_path", f=path, extras=extras)
+        if not paths:
+            paths: list[Path] = [path]
+        else:
+            paths.append(path)
+
+        all_data: dict[str, list[tuple[float, DictData]]] = {}
+        for path in paths:
+            for file in glob_files(path):
+
+                if cls.is_ignore(file, path):
+                    continue
 
-            if cls.is_ignore(file, conf_path):
-                continue
+                for key, data in cls.filter_yaml(file).items():
 
-            for key, data in cls.filter_yaml(file).items():
+                    if key in excluded:
+                        continue
 
-                if key in exclude:
-                    continue
+                    if (
+                        data.get("type", "")
+                        == (obj if isclass(obj) else obj.__class__).__name__
+                    ):
+                        marking: tuple[float, DictData] = (
+                            file.lstat().st_mtime,
+                            data,
+                        )
+                        if key in all_data:
+                            all_data[key].append(marking)
+                        else:
+                            all_data[key] = [marking]
 
-                if data.get("type", "") == obj.__name__:
-                    yield key, (
-                        {k: data[k] for k in data if k in included}
-                        if included
-                        else data
-                    )
+        for key in all_data:
+            yield key, max(all_data[key], key=lambda x: x[0])[1]
 
     @classmethod
     def is_ignore(
         cls,
         file: Path,
-        conf_path: Path,
+        path: Path,
         *,
         ignore_filename: Optional[str] = None,
     ) -> bool:
         """Check this file was ignored.
 
         :param file: (Path) A file path that want to check.
-        :param conf_path: (Path) A config path that want to read the config
+        :param path: (Path) A config path that want to read the config
             ignore file.
-        :param ignore_filename: (str) An ignore filename.
+        :param ignore_filename: (str) An ignore filename. Default is
+            `.confignore` filename.
 
         :rtype: bool
         """
         ignore_filename: str = ignore_filename or ".confignore"
-        return is_ignored(file, read_ignore(conf_path / ignore_filename))
+        return is_ignored(file, read_ignore(path / ignore_filename))
 
     @classmethod
     def filter_yaml(cls, file: Path, name: str | None = None) -> DictData:
@@ -369,44 +444,19 @@ def dynamic(
     return rsx if rsx is not None else rs
 
 
-class Loader(SimLoad):
-    """Loader Object that get the config `yaml` file from current path.
+class Loader(Protocol):  # pragma: no cov
+    type: str
+    path: Path
+    data: DictData
+    extras: DictData
+    externals: DictData
 
-    :param name: (str) A name of config data that will read by Yaml Loader object.
-    :param externals: (DictData) An external parameters
-    """
+    def __init__(self, *args, **kwargs) -> None: ...
 
     @classmethod
-    def finds(
-        cls,
-        obj: object,
-        *,
-        path: Path | None = None,
-        included: list[str] | None = None,
-        excluded: list[str] | None = None,
-        **kwargs,
-    ) -> Iterator[tuple[str, DictData]]:
-        """Override the find class method from the Simple Loader object.
+    def find(cls, name: str, *args, **kwargs) -> DictData: ...
 
-        :param obj: An object that want to validate matching before return.
-        :param path: (Path) A override config path.
-        :param included: An excluded list of data key that want to reject this
-            data if any key exist.
-        :param excluded: An included list of data key that want to filter from
-            data.
-
-        :rtype: Iterator[tuple[str, DictData]]
-        """
-        return super().finds(
-            obj=obj,
-            conf_path=(path or config.conf_path),
-            included=included,
-            excluded=excluded,
-        )
-
-    def __init__(self, name: str, externals: DictData) -> None:
-        super().__init__(
-            name,
-            conf_path=dynamic("conf_path", extras=externals),
-            externals=externals,
-        )
+    @classmethod
+    def finds(
+        cls, obj: object, *args, **kwargs
+    ) -> Iterator[tuple[str, DictData]]: ...

ddeutil/workflow/{cron.py → event.py}

@@ -3,11 +3,14 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
+"""Event module that store all event object. Now, it has only `On` and `OnYear`
+model these are schedule with crontab event.
+"""
 from __future__ import annotations
 
 from dataclasses import fields
 from datetime import datetime
-from typing import Annotated, Literal, Union
+from typing import Annotated, Any, Literal, Union
 from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
 
 from pydantic import BaseModel, ConfigDict, Field, ValidationInfo
@@ -17,11 +20,13 @@ from typing_extensions import Self
 
 from .__cron import WEEKDAYS, CronJob, CronJobYear, CronRunner, Options
 from .__types import DictData, DictStr
-from .conf import Loader
+from .conf import FileLoad
+
+Interval = Literal["daily", "weekly", "monthly"]
 
 
 def interval2crontab(
-    interval: Literal["daily", "weekly", "monthly"],
+    interval: Interval,
     *,
     day: str | None = None,
     time: str = "00:00",
@@ -59,10 +64,11 @@ def interval2crontab(
 
 
 class On(BaseModel):
-    """On Pydantic model (Warped crontab object by model).
+    """On model (Warped crontab object by Pydantic model) to keep crontab value
+    and generate CronRunner object from this crontab value.
 
-    See Also:
-        * `generate()` is the main use-case of this schedule object.
+    Methods:
+        - generate: is the main use-case of this schedule object.
     """
 
     model_config = ConfigDict(arbitrary_types_allowed=True)
@@ -90,19 +96,24 @@ class On(BaseModel):
             description="A timezone string value",
             alias="timezone",
         ),
-    ] = "Etc/UTC"
+    ] = "UTC"
 
     @classmethod
     def from_value(cls, value: DictStr, extras: DictData) -> Self:
         """Constructor from values that will generate crontab by function.
 
-        :param value: A mapping value that will generate crontab before create
-            schedule model.
-        :param extras: An extras parameter that will keep in extras.
+        :param value: (DictStr) A mapping value that will generate crontab
+            before create schedule model.
+        :param extras: (DictData) An extra parameter that use to override core
+            config value.
         """
         passing: DictStr = {}
+
         if "timezone" in value:
             passing["tz"] = value.pop("timezone")
+        elif "tz" in value:
+            passing["tz"] = value.pop("tz")
+
         passing["cronjob"] = interval2crontab(
             **{v: value[v] for v in value if v in ("interval", "day", "time")}
         )
@@ -112,18 +123,20 @@ class On(BaseModel):
     def from_conf(
         cls,
         name: str,
+        *,
         extras: DictData | None = None,
     ) -> Self:
-        """Constructor from the name of config that will use loader object for
-        getting the data.
+        """Constructor from the name of config loader that will use loader
+        object for getting the `On` data.
 
-        :param name: A name of config that will get from loader.
-        :param extras: An extra parameter that will keep in extras.
+        :param name: (str) A name of config that will get from loader.
+        :param extras: (DictData) An extra parameter that use to override core
+            config values.
 
         :rtype: Self
         """
         extras: DictData = extras or {}
-        loader: Loader = Loader(name, externals=extras)
+        loader: FileLoad = FileLoad(name, extras=extras)
 
         # NOTE: Validate the config type match with current connection model
         if loader.type != cls.__name__:
@@ -155,17 +168,17 @@ class On(BaseModel):
         )
 
     @model_validator(mode="before")
-    def __prepare_values(cls, values: DictData) -> DictData:
+    def __prepare_values(cls, data: Any) -> Any:
         """Extract tz key from value and change name to timezone key.
 
-        :param values: (DictData) A data that want to pass for create an On
+        :param data: (DictData) A data that want to pass for create an On
             model.
 
         :rtype: DictData
         """
-        if tz := values.pop("tz", None):
-            values["timezone"] = tz
-        return values
+        if isinstance(data, dict) and (tz := data.pop("tz", None)):
+            data["timezone"] = tz
+        return data
 
     @field_validator("tz")
     def __validate_tz(cls, value: str) -> str:
@@ -238,6 +251,9 @@ class On(BaseModel):
         """Return a next datetime from Cron runner object that start with any
         date that given from input.
 
+        :param start: (str | datetime) A start datetime that use to generate
+            the CronRunner object.
+
         :rtype: CronRunner
         """
         runner: CronRunner = self.generate(start=start)

ddeutil/workflow/exceptions.py

@@ -22,7 +22,12 @@ ErrorData = TypedDict(
 
 
 def to_dict(exception: Exception) -> ErrorData:  # pragma: no cov
-    """Create dict data from exception instance."""
+    """Create dict data from exception instance.
+
+    :param exception: An exception object.
+
+    :rtype: ErrorData
+    """
     return {
         "class": exception,
         "name": exception.__class__.__name__,
@@ -33,6 +38,10 @@ def to_dict(exception: Exception) -> ErrorData:  # pragma: no cov
 class BaseWorkflowException(Exception):
 
     def to_dict(self) -> ErrorData:
+        """Return ErrorData data from the current exception object.
+
+        :rtype: ErrorData
+        """
         return to_dict(self)
 
 

ddeutil/workflow/job.py

@@ -140,14 +140,19 @@ class Strategy(BaseModel):
 
     fail_fast: bool = Field(
         default=False,
+        description=(
+            "A fail-fast flag that use to cancel strategy execution when it "
+            "has some execution was failed."
+        ),
         alias="fail-fast",
     )
     max_parallel: int = Field(
         default=1,
         gt=0,
+        lt=10,
         description=(
             "The maximum number of executor thread pool that want to run "
-            "parallel"
+            "parallel. This value should gather than 0 and less than 10."
         ),
         alias="max-parallel",
     )
@@ -540,6 +545,11 @@ class Job(BaseModel):
                         }
                     }
 
+            The keys that will set to the received context is `strategies`,
+        `errors`, and `skipped` keys. The `errors` and `skipped` keys will
+        extract from the result context if it exists. If it does not found, it
+        will not set on the received context.
+
         :raise JobException: If the job's ID does not set and the setting
             default job ID flag does not set.
 
@@ -599,7 +609,7 @@ class Job(BaseModel):
 
         :param params: (DictData) A parameter data.
         :param run_id: (str) A job running ID.
-        :param parent_run_id: (str) A parent workflow running ID.
+        :param parent_run_id: (str) A parent running ID.
         :param event: (Event) An Event manager instance that use to cancel this
             execution if it forces stopped by parent execution.
 
@@ -667,15 +677,15 @@ def local_execute_strategy(
     `set_outputs` method for reconstruct result context data.
 
     :param job: (Job) A job model that want to execute.
-    :param strategy: A strategy metrix value that use on this execution.
-        This value will pass to the `matrix` key for templating.
+    :param strategy: (DictData) A strategy metrix value. This value will pass
+        to the `matrix` key for templating in context data.
     :param params: (DictData) A parameter data.
     :param result: (Result) A Result instance for return context and status.
     :param event: (Event) An Event manager instance that use to cancel this
         execution if it forces stopped by parent execution.
 
-    :raise JobException: If it has any error from `StageException` or
-        `UtilException`.
+    :raise JobException: If stage execution raise any error as `StageException`
+        or `UtilException`.
 
     :rtype: Result
     """
@@ -683,17 +693,16 @@ def local_execute_strategy(
         run_id=gen_id(job.id or "not-set", unique=True),
         extras=job.extras,
     )
-
-    strategy_id: str = gen_id(strategy)
-    context: DictData = copy.deepcopy(params)
-    context.update({"matrix": strategy, "stages": {}})
-
     if strategy:
+        strategy_id: str = gen_id(strategy)
         result.trace.info(f"[JOB]: Start Strategy: {strategy_id!r}")
         result.trace.info(f"[JOB]: ... matrix: {strategy!r}")
     else:
-        result.trace.info("[JOB]: Start Strategy: EMPTY")
+        strategy_id: str = "EMPTY"
+        result.trace.info("[JOB]: Start Strategy: 'EMPTY'")
 
+    context: DictData = copy.deepcopy(params)
+    context.update({"matrix": strategy, "stages": {}})
     for stage in job.stages:
 
         if job.extras:
@@ -707,7 +716,7 @@ def local_execute_strategy(
         if event and event.is_set():
             error_msg: str = (
                 "Job strategy was canceled from event that had set before "
-                "strategy execution."
+                "job strategy execution."
             )
             return result.catch(
                 status=CANCEL,
@@ -820,7 +829,7 @@ def local_execute(
             context={
                 "errors": JobException(
                     "Job was canceled from event that had set before "
-                    "local execution."
+                    "local job execution."
                 ).to_dict()
             },
         )

ddeutil/workflow/result.py

@@ -98,6 +98,7 @@ class Result:
             return cls(
                 run_id=(run_id or gen_id(id_logic or "", unique=True)),
                 parent_run_id=parent_run_id,
+                ts=get_dt_now(dynamic("tz", extras=extras)),
                 extras=(extras or {}),
             )
         elif parent_run_id: