ddeutil-workflow 0.0.55__tar.gz → 0.0.56__tar.gz

{ddeutil_workflow-0.0.55/src/ddeutil_workflow.egg-info → ddeutil_workflow-0.0.56}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddeutil-workflow
-Version: 0.0.55
+Version: 0.0.56
 Summary: Lightweight workflow orchestration
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT

{ddeutil_workflow-0.0.55 → ddeutil_workflow-0.0.56}/pyproject.toml

@@ -57,6 +57,9 @@ docker = [
 Homepage = "https://github.com/ddeutils/ddeutil-workflow/"
 "Source Code" = "https://github.com/ddeutils/ddeutil-workflow/"
 
+[project.scripts]
+workflow-cli = "ddeutil.workflow.__main__:app"
+
 [tool.setuptools.dynamic]
 version = {attr = "ddeutil.workflow.__about__.__version__"}
 
@@ -81,7 +84,7 @@ omit = [
     "src/ddeutil/workflow/__cron.py",
     "src/ddeutil/workflow/api/__init__.py",
     "src/ddeutil/workflow/api/log.py",
-    "src/ddeutil/workflow/api/repeat.py",
+    "src/ddeutil/workflow/api/utils.py",
     "src/ddeutil/workflow/api/routes/__init__.py",
     "src/ddeutil/workflow/api/routes/job.py",
     "src/ddeutil/workflow/api/routes/logs.py",

ddeutil_workflow-0.0.56/src/ddeutil/workflow/__about__.py

@@ -0,0 +1 @@
+__version__: str = "0.0.56"

{ddeutil_workflow-0.0.55 → ddeutil_workflow-0.0.56}/src/ddeutil/workflow/__init__.py

@@ -7,16 +7,18 @@ from .__cron import CronJob, CronRunner
 from .__types import DictData, DictStr, Matrix, Re, TupleStr
 from .conf import (
     Config,
-    Loader,
+    FileLoad,
     config,
     env,
 )
-from .cron import *
+from .event import *
 from .exceptions import *
 from .job import *
 from .logs import (
     Audit,
     AuditModel,
+    FileAudit,
+    FileTrace,
     Trace,
     TraceData,
     TraceMeta,

ddeutil_workflow-0.0.56/src/ddeutil/workflow/__main__.py

@@ -0,0 +1,30 @@
+import typer
+
+app = typer.Typer()
+
+
+@app.callback()
+def callback():
+    """
+    Awesome Portal Gun
+    """
+
+
+@app.command()
+def provision():
+    """
+    Shoot the portal gun
+    """
+    typer.echo("Shooting portal gun")
+
+
+@app.command()
+def job():
+    """
+    Load the portal gun
+    """
+    typer.echo("Loading portal gun")
+
+
+if __name__ == "__main__":
+    app()

{ddeutil_workflow-0.0.55 → ddeutil_workflow-0.0.56}/src/ddeutil/workflow/conf.py

@@ -7,24 +7,26 @@ from __future__ import annotations
 
 import json
 import os
+from abc import ABC, abstractmethod
 from collections.abc import Iterator
 from datetime import timedelta
 from functools import cached_property
+from inspect import isclass
 from pathlib import Path
-from typing import Final, Optional, TypeVar
+from typing import Final, Optional, Protocol, TypeVar, Union
 from zoneinfo import ZoneInfo
 
 from ddeutil.core import str2bool
 from ddeutil.io import YamlFlResolve
 from ddeutil.io.paths import glob_files, is_ignored, read_ignore
 
-from .__types import DictData, TupleStr
+from .__types import DictData
 
 T = TypeVar("T")
 PREFIX: Final[str] = "WORKFLOW"
 
 
-def env(var: str, default: str | None = None) -> str | None:  # pragma: no cov
+def env(var: str, default: str | None = None) -> str | None:
     """Get environment variable with uppercase and adding prefix string.
 
     :param var: (str) A env variable name.
@@ -35,17 +37,6 @@ def env(var: str, default: str | None = None) -> str | None:  # pragma: no cov
     return os.getenv(f"{PREFIX}_{var.upper().replace(' ', '_')}", default)
 
 
-__all__: TupleStr = (
-    "api_config",
-    "env",
-    "Config",
-    "SimLoad",
-    "Loader",
-    "config",
-    "dynamic",
-)
-
-
 class Config:  # pragma: no cov
     """Config object for keeping core configurations on the current session
     without changing when if the application still running.
@@ -188,7 +179,7 @@ class Config:  # pragma: no cov
             return timedelta(**json.loads(stop_boundary_delta_str))
         except Exception as err:
             raise ValueError(
-                "Config ``WORKFLOW_APP_STOP_BOUNDARY_DELTA`` can not parsing to"
+                "Config `WORKFLOW_APP_STOP_BOUNDARY_DELTA` can not parsing to"
                 f"timedelta with {stop_boundary_delta_str}."
             ) from err
 
@@ -209,110 +200,194 @@ class APIConfig:
         return str2bool(env("API_ENABLE_ROUTE_SCHEDULE", "true"))
 
 
-class SimLoad:
-    """Simple Load Object that will search config data by given some identity
-    value like name of workflow or on.
+class BaseLoad(ABC):
+
+    @classmethod
+    @abstractmethod
+    def find(cls, name: str, *args, **kwargs) -> DictData: ...
+
+    @classmethod
+    @abstractmethod
+    def finds(
+        cls, obj: object, *args, **kwargs
+    ) -> Iterator[tuple[str, DictData]]: ...
 
-    :param name: A name of config data that will read by Yaml Loader object.
-    :param conf_path: A config path object.
-    :param externals: An external parameters
+
+class FileLoad(BaseLoad):
+    """Base Load object that use to search config data by given some identity
+    value like name of `Workflow` or `On` templates.
+
+    :param name: (str) A name of key of config data that read with YAML
+        Environment object.
+    :param path: (Path) A config path object.
+    :param externals: (DictData) An external config data that want to add to
+        loaded config data.
+    :param extras: (DictDdata) An extra parameters that use to override core
+        config values.
+
+    :raise ValueError: If the data does not find on the config path with the
+        name parameter.
 
     Noted:
-        The config data should have ``type`` key for modeling validation that
+        The config data should have `type` key for modeling validation that
     make this loader know what is config should to do pass to.
 
         ... <identity-key>:
         ...     type: <importable-object>
         ...     <key-data-1>: <value-data-1>
         ...     <key-data-2>: <value-data-2>
+
+        This object support multiple config paths if you pass the `conf_paths`
+    key to the `extras` parameter.
     """
 
     def __init__(
         self,
         name: str,
-        conf_path: Path,
+        *,
+        path: Optional[Union[str, Path]] = None,
         externals: DictData | None = None,
+        extras: DictData | None = None,
     ) -> None:
-        self.conf_path: Path = conf_path
+        self.path: Path = Path(dynamic("conf_path", f=path, extras=extras))
         self.externals: DictData = externals or {}
-
-        self.data: DictData = {}
-        for file in glob_files(conf_path):
-
-            if self.is_ignore(file, conf_path):
-                continue
-
-            if data := self.filter_yaml(file, name=name):
-                self.data = data
+        self.extras: DictData = extras or {}
+        self.data: DictData = self.find(
+            name,
+            path=path,
+            paths=self.extras.get("conf_paths"),
+            extras=extras,
+        )
 
         # VALIDATE: check the data that reading should not empty.
         if not self.data:
             raise ValueError(
-                f"Config {name!r} does not found on conf path: "
-                f"{self.conf_path}."
+                f"Config {name!r} does not found on the conf path: {self.path}."
             )
 
         self.data.update(self.externals)
 
+    @classmethod
+    def find(
+        cls,
+        name: str,
+        *,
+        path: Optional[Path] = None,
+        paths: Optional[list[Path]] = None,
+        extras: Optional[DictData] = None,
+    ) -> DictData:
+        """Find data with specific key and return the latest modify date data if
+        this key exists multiple files.
+
+        :param name: (str) A name of data that want to find.
+        :param path: (Path) A config path object.
+        :param paths: (list[Path]) A list of config path object.
+        :param extras: (DictData)  An extra parameter that use to override core
+            config values.
+
+        :rtype: DictData
+        """
+        path: Path = dynamic("conf_path", f=path, extras=extras)
+        if not paths:
+            paths: list[Path] = [path]
+        elif not isinstance(paths, list):
+            raise TypeError(
+                f"Multi-config paths does not support for type: {type(paths)}"
+            )
+        else:
+            paths.append(path)
+
+        all_data: list[tuple[float, DictData]] = []
+        for path in paths:
+            for file in glob_files(path):
+
+                if cls.is_ignore(file, path):
+                    continue
+
+                if data := cls.filter_yaml(file, name=name):
+                    all_data.append((file.lstat().st_mtime, data))
+
+        return {} if not all_data else max(all_data, key=lambda x: x[0])[1]
+
     @classmethod
     def finds(
         cls,
         obj: object,
-        conf_path: Path,
         *,
-        included: list[str] | None = None,
+        path: Optional[Path] = None,
+        paths: Optional[list[Path]] = None,
         excluded: list[str] | None = None,
+        extras: Optional[DictData] = None,
     ) -> Iterator[tuple[str, DictData]]:
         """Find all data that match with object type in config path. This class
         method can use include and exclude list of identity name for filter and
         adds-on.
 
         :param obj: An object that want to validate matching before return.
-        :param conf_path: A config object.
-        :param included: An excluded list of data key that want to reject this
-            data if any key exist.
+        :param path: A config path object.
+        :param paths: (list[Path]) A list of config path object.
         :param excluded: An included list of data key that want to filter from
             data.
+        :param extras: (DictData) An extra parameter that use to override core
+            config values.
 
         :rtype: Iterator[tuple[str, DictData]]
         """
-        exclude: list[str] = excluded or []
-        for file in glob_files(conf_path):
+        excluded: list[str] = excluded or []
+        path: Path = dynamic("conf_path", f=path, extras=extras)
+        if not paths:
+            paths: list[Path] = [path]
+        else:
+            paths.append(path)
+
+        all_data: dict[str, list[tuple[float, DictData]]] = {}
+        for path in paths:
+            for file in glob_files(path):
+
+                if cls.is_ignore(file, path):
+                    continue
 
-            if cls.is_ignore(file, conf_path):
-                continue
+                for key, data in cls.filter_yaml(file).items():
 
-            for key, data in cls.filter_yaml(file).items():
+                    if key in excluded:
+                        continue
 
-                if key in exclude:
-                    continue
+                    if (
+                        data.get("type", "")
+                        == (obj if isclass(obj) else obj.__class__).__name__
+                    ):
+                        marking: tuple[float, DictData] = (
+                            file.lstat().st_mtime,
+                            data,
+                        )
+                        if key in all_data:
+                            all_data[key].append(marking)
+                        else:
+                            all_data[key] = [marking]
 
-                if data.get("type", "") == obj.__name__:
-                    yield key, (
-                        {k: data[k] for k in data if k in included}
-                        if included
-                        else data
-                    )
+        for key in all_data:
+            yield key, max(all_data[key], key=lambda x: x[0])[1]
 
     @classmethod
     def is_ignore(
         cls,
         file: Path,
-        conf_path: Path,
+        path: Path,
         *,
         ignore_filename: Optional[str] = None,
     ) -> bool:
         """Check this file was ignored.
 
         :param file: (Path) A file path that want to check.
-        :param conf_path: (Path) A config path that want to read the config
+        :param path: (Path) A config path that want to read the config
             ignore file.
-        :param ignore_filename: (str) An ignore filename.
+        :param ignore_filename: (str) An ignore filename. Default is
+            `.confignore` filename.
 
         :rtype: bool
         """
         ignore_filename: str = ignore_filename or ".confignore"
-        return is_ignored(file, read_ignore(conf_path / ignore_filename))
+        return is_ignored(file, read_ignore(path / ignore_filename))
 
     @classmethod
     def filter_yaml(cls, file: Path, name: str | None = None) -> DictData:
@@ -369,44 +444,19 @@ def dynamic(
     return rsx if rsx is not None else rs
 
 
-class Loader(SimLoad):
-    """Loader Object that get the config `yaml` file from current path.
+class Loader(Protocol):  # pragma: no cov
+    type: str
+    path: Path
+    data: DictData
+    extras: DictData
+    externals: DictData
 
-    :param name: (str) A name of config data that will read by Yaml Loader object.
-    :param externals: (DictData) An external parameters
-    """
+    def __init__(self, *args, **kwargs) -> None: ...
 
     @classmethod
-    def finds(
-        cls,
-        obj: object,
-        *,
-        path: Path | None = None,
-        included: list[str] | None = None,
-        excluded: list[str] | None = None,
-        **kwargs,
-    ) -> Iterator[tuple[str, DictData]]:
-        """Override the find class method from the Simple Loader object.
+    def find(cls, name: str, *args, **kwargs) -> DictData: ...
 
-        :param obj: An object that want to validate matching before return.
-        :param path: (Path) A override config path.
-        :param included: An excluded list of data key that want to reject this
-            data if any key exist.
-        :param excluded: An included list of data key that want to filter from
-            data.
-
-        :rtype: Iterator[tuple[str, DictData]]
-        """
-        return super().finds(
-            obj=obj,
-            conf_path=(path or config.conf_path),
-            included=included,
-            excluded=excluded,
-        )
-
-    def __init__(self, name: str, externals: DictData) -> None:
-        super().__init__(
-            name,
-            conf_path=dynamic("conf_path", extras=externals),
-            externals=externals,
-        )
+    @classmethod
+    def finds(
+        cls, obj: object, *args, **kwargs
+    ) -> Iterator[tuple[str, DictData]]: ...

ddeutil_workflow-0.0.55/src/ddeutil/workflow/cron.py → ddeutil_workflow-0.0.56/src/ddeutil/workflow/event.py

@@ -3,11 +3,14 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
+"""Event module that store all event object. Now, it has only `On` and `OnYear`
+model these are schedule with crontab event.
+"""
 from __future__ import annotations
 
 from dataclasses import fields
 from datetime import datetime
-from typing import Annotated, Literal, Union
+from typing import Annotated, Any, Literal, Union
 from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
 
 from pydantic import BaseModel, ConfigDict, Field, ValidationInfo
@@ -17,11 +20,13 @@ from typing_extensions import Self
 
 from .__cron import WEEKDAYS, CronJob, CronJobYear, CronRunner, Options
 from .__types import DictData, DictStr
-from .conf import Loader
+from .conf import FileLoad
+
+Interval = Literal["daily", "weekly", "monthly"]
 
 
 def interval2crontab(
-    interval: Literal["daily", "weekly", "monthly"],
+    interval: Interval,
     *,
     day: str | None = None,
     time: str = "00:00",
@@ -59,10 +64,11 @@ def interval2crontab(
 
 
 class On(BaseModel):
-    """On Pydantic model (Warped crontab object by model).
+    """On model (Warped crontab object by Pydantic model) to keep crontab value
+    and generate CronRunner object from this crontab value.
 
-    See Also:
-        * `generate()` is the main use-case of this schedule object.
+    Methods:
+        - generate: is the main use-case of this schedule object.
     """
 
     model_config = ConfigDict(arbitrary_types_allowed=True)
@@ -90,19 +96,24 @@ class On(BaseModel):
             description="A timezone string value",
             alias="timezone",
         ),
-    ] = "Etc/UTC"
+    ] = "UTC"
 
     @classmethod
     def from_value(cls, value: DictStr, extras: DictData) -> Self:
         """Constructor from values that will generate crontab by function.
 
-        :param value: A mapping value that will generate crontab before create
-            schedule model.
-        :param extras: An extras parameter that will keep in extras.
+        :param value: (DictStr) A mapping value that will generate crontab
+            before create schedule model.
+        :param extras: (DictData) An extra parameter that use to override core
+            config value.
         """
         passing: DictStr = {}
+
         if "timezone" in value:
             passing["tz"] = value.pop("timezone")
+        elif "tz" in value:
+            passing["tz"] = value.pop("tz")
+
         passing["cronjob"] = interval2crontab(
             **{v: value[v] for v in value if v in ("interval", "day", "time")}
         )
@@ -112,18 +123,20 @@ class On(BaseModel):
     def from_conf(
         cls,
         name: str,
+        *,
         extras: DictData | None = None,
     ) -> Self:
-        """Constructor from the name of config that will use loader object for
-        getting the data.
+        """Constructor from the name of config loader that will use loader
+        object for getting the `On` data.
 
-        :param name: A name of config that will get from loader.
-        :param extras: An extra parameter that will keep in extras.
+        :param name: (str) A name of config that will get from loader.
+        :param extras: (DictData) An extra parameter that use to override core
+            config values.
 
         :rtype: Self
         """
         extras: DictData = extras or {}
-        loader: Loader = Loader(name, externals=extras)
+        loader: FileLoad = FileLoad(name, extras=extras)
 
         # NOTE: Validate the config type match with current connection model
         if loader.type != cls.__name__:
@@ -155,17 +168,17 @@ class On(BaseModel):
         )
 
     @model_validator(mode="before")
-    def __prepare_values(cls, values: DictData) -> DictData:
+    def __prepare_values(cls, data: Any) -> Any:
         """Extract tz key from value and change name to timezone key.
 
-        :param values: (DictData) A data that want to pass for create an On
+        :param data: (DictData) A data that want to pass for create an On
             model.
 
         :rtype: DictData
         """
-        if tz := values.pop("tz", None):
-            values["timezone"] = tz
-        return values
+        if isinstance(data, dict) and (tz := data.pop("tz", None)):
+            data["timezone"] = tz
+        return data
 
     @field_validator("tz")
     def __validate_tz(cls, value: str) -> str:
@@ -238,6 +251,9 @@ class On(BaseModel):
         """Return a next datetime from Cron runner object that start with any
         date that given from input.
 
+        :param start: (str | datetime) A start datetime that use to generate
+            the CronRunner object.
+
         :rtype: CronRunner
         """
         runner: CronRunner = self.generate(start=start)

{ddeutil_workflow-0.0.55 → ddeutil_workflow-0.0.56}/src/ddeutil/workflow/exceptions.py

@@ -22,7 +22,12 @@ ErrorData = TypedDict(
 
 
 def to_dict(exception: Exception) -> ErrorData:  # pragma: no cov
-    """Create dict data from exception instance."""
+    """Create dict data from exception instance.
+
+    :param exception: An exception object.
+
+    :rtype: ErrorData
+    """
     return {
         "class": exception,
         "name": exception.__class__.__name__,
@@ -33,6 +38,10 @@ def to_dict(exception: Exception) -> ErrorData:  # pragma: no cov
 class BaseWorkflowException(Exception):
 
     def to_dict(self) -> ErrorData:
+        """Return ErrorData data from the current exception object.
+
+        :rtype: ErrorData
+        """
         return to_dict(self)
 
 

{ddeutil_workflow-0.0.55 → ddeutil_workflow-0.0.56}/src/ddeutil/workflow/job.py

@@ -140,14 +140,19 @@ class Strategy(BaseModel):
 
     fail_fast: bool = Field(
         default=False,
+        description=(
+            "A fail-fast flag that use to cancel strategy execution when it "
+            "has some execution was failed."
+        ),
         alias="fail-fast",
     )
     max_parallel: int = Field(
         default=1,
         gt=0,
+        lt=10,
         description=(
             "The maximum number of executor thread pool that want to run "
-            "parallel"
+            "parallel. This value should gather than 0 and less than 10."
         ),
         alias="max-parallel",
     )