ddeutil-workflow 0.0.55__py3-none-any.whl → 0.0.57__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1 +1 @@
-__version__: str = "0.0.55"
+__version__: str = "0.0.57"

ddeutil/workflow/__cron.py

@@ -502,10 +502,10 @@ class CronPart:
             except IndexError:
                 next_value: int = -1
             if value != (next_value - 1):
-                # NOTE: ``next_value`` is not the subsequent number
+                # NOTE: `next_value` is not the subsequent number
                 if start_number is None:
                     # NOTE:
-                    #   The last number of the list ``self.values`` is not in a
+                    #   The last number of the list `self.values` is not in a
                     #   range.
                     multi_dim_values.append(value)
                 else:
@@ -703,11 +703,14 @@ class CronJob:
         *,
         tz: str | None = None,
     ) -> CronRunner:
-        """Returns the schedule datetime runner with this cronjob. It would run
-        ``next``, ``prev``, or ``reset`` to generate running date that you want.
+        """Returns CronRunner instance that be datetime runner with this
+        cronjob. It can use `next`, `prev`, or `reset` methods to generate
+        running date.
 
-        :param date: An initial date that want to mark as the start point.
-        :param tz: A string timezone that want to change on runner.
+        :param date: (datetime) An initial date that want to mark as the start
+            point. (Default is use the current datetime)
+        :param tz: (str) A string timezone that want to change on runner.
+            (Default is None)
 
         :rtype: CronRunner
         """
@@ -743,6 +746,10 @@ class CronJobYear(CronJob):
 class CronRunner:
     """Create an instance of Date Runner object for datetime generate with
     cron schedule object value.
+
+    :param cron: (CronJob | CronJobYear)
+    :param date: (datetime)
+    :param tz: (str)
     """
 
     shift_limit: ClassVar[int] = 25
@@ -761,11 +768,17 @@ class CronRunner:
         cron: CronJob | CronJobYear,
         date: datetime | None = None,
         *,
-        tz: str | None = None,
+        tz: str | ZoneInfo | None = None,
     ) -> None:
-        # NOTE: Prepare timezone if this value does not set, it will use UTC.
-        self.tz: ZoneInfo = ZoneInfo("UTC")
+        self.tz: ZoneInfo | None = None
         if tz:
+            if isinstance(tz, ZoneInfo):
+                self.tz = tz
+            elif not isinstance(tz, str):
+                raise TypeError(
+                    "Invalid type of `tz` parameter, it should be str or "
+                    "ZoneInfo instance."
+                )
             try:
                 self.tz = ZoneInfo(tz)
             except ZoneInfoNotFoundError as err:
@@ -777,9 +790,10 @@ class CronRunner:
                 raise ValueError(
                     "Input schedule start time is not a valid datetime object."
                 )
-            if tz is None:
-                self.tz = date.tzinfo
-            self.date: datetime = date.astimezone(self.tz)
+            if tz is not None:
+                self.date: datetime = date.astimezone(self.tz)
+            else:
+                self.date: datetime = date
         else:
             self.date: datetime = datetime.now(tz=self.tz)
 

ddeutil/workflow/__init__.py

@@ -7,16 +7,18 @@ from .__cron import CronJob, CronRunner
 from .__types import DictData, DictStr, Matrix, Re, TupleStr
 from .conf import (
     Config,
-    Loader,
+    FileLoad,
     config,
     env,
 )
-from .cron import *
+from .event import *
 from .exceptions import *
 from .job import *
 from .logs import (
     Audit,
     AuditModel,
+    FileAudit,
+    FileTrace,
     Trace,
     TraceData,
     TraceMeta,

ddeutil/workflow/__main__.py

@@ -0,0 +1,30 @@
+import typer
+
+app = typer.Typer()
+
+
+@app.callback()
+def callback():
+    """
+    Awesome Portal Gun
+    """
+
+
+@app.command()
+def provision():
+    """
+    Shoot the portal gun
+    """
+    typer.echo("Shooting portal gun")
+
+
+@app.command()
+def job():
+    """
+    Load the portal gun
+    """
+    typer.echo("Loading portal gun")
+
+
+if __name__ == "__main__":
+    app()

ddeutil/workflow/__types.py

@@ -20,6 +20,7 @@ from typing import Any, Optional, TypedDict, Union
 
 from typing_extensions import Self
 
+StrOrInt = Union[str, int]
 TupleStr = tuple[str, ...]
 DictData = dict[str, Any]
 DictStr = dict[str, str]

ddeutil/workflow/conf.py

@@ -7,24 +7,26 @@ from __future__ import annotations
 
 import json
 import os
+from abc import ABC, abstractmethod
 from collections.abc import Iterator
 from datetime import timedelta
 from functools import cached_property
+from inspect import isclass
 from pathlib import Path
-from typing import Final, Optional, TypeVar
+from typing import Final, Optional, Protocol, TypeVar, Union
 from zoneinfo import ZoneInfo
 
 from ddeutil.core import str2bool
 from ddeutil.io import YamlFlResolve
 from ddeutil.io.paths import glob_files, is_ignored, read_ignore
 
-from .__types import DictData, TupleStr
+from .__types import DictData
 
 T = TypeVar("T")
 PREFIX: Final[str] = "WORKFLOW"
 
 
-def env(var: str, default: str | None = None) -> str | None:  # pragma: no cov
+def env(var: str, default: str | None = None) -> str | None:
     """Get environment variable with uppercase and adding prefix string.
 
     :param var: (str) A env variable name.
@@ -35,17 +37,6 @@ def env(var: str, default: str | None = None) -> str | None:  # pragma: no cov
     return os.getenv(f"{PREFIX}_{var.upper().replace(' ', '_')}", default)
 
 
-__all__: TupleStr = (
-    "api_config",
-    "env",
-    "Config",
-    "SimLoad",
-    "Loader",
-    "config",
-    "dynamic",
-)
-
-
 class Config:  # pragma: no cov
     """Config object for keeping core configurations on the current session
     without changing when if the application still running.
@@ -188,7 +179,7 @@ class Config:  # pragma: no cov
             return timedelta(**json.loads(stop_boundary_delta_str))
         except Exception as err:
             raise ValueError(
-                "Config ``WORKFLOW_APP_STOP_BOUNDARY_DELTA`` can not parsing to"
+                "Config `WORKFLOW_APP_STOP_BOUNDARY_DELTA` can not parsing to"
                 f"timedelta with {stop_boundary_delta_str}."
             ) from err
 
@@ -209,110 +200,202 @@ class APIConfig:
         return str2bool(env("API_ENABLE_ROUTE_SCHEDULE", "true"))
 
 
-class SimLoad:
-    """Simple Load Object that will search config data by given some identity
-    value like name of workflow or on.
+class BaseLoad(ABC):  # pragma: no cov
+    """Base Load object is the abstraction object for any Load object that
+    should to inherit from this base class.
+    """
+
+    @classmethod
+    @abstractmethod
+    def find(cls, name: str, *args, **kwargs) -> DictData: ...
+
+    @classmethod
+    @abstractmethod
+    def finds(
+        cls, obj: object, *args, **kwargs
+    ) -> Iterator[tuple[str, DictData]]: ...
 
-    :param name: A name of config data that will read by Yaml Loader object.
-    :param conf_path: A config path object.
-    :param externals: An external parameters
+
+class FileLoad(BaseLoad):
+    """Base Load object that use to search config data by given some identity
+    value like name of `Workflow` or `On` templates.
+
+    :param name: (str) A name of key of config data that read with YAML
+        Environment object.
+    :param path: (Path) A config path object.
+    :param externals: (DictData) An external config data that want to add to
+        loaded config data.
+    :param extras: (DictDdata) An extra parameters that use to override core
+        config values.
+
+    :raise ValueError: If the data does not find on the config path with the
+        name parameter.
 
     Noted:
-        The config data should have ``type`` key for modeling validation that
+        The config data should have `type` key for modeling validation that
     make this loader know what is config should to do pass to.
 
         ... <identity-key>:
         ...     type: <importable-object>
         ...     <key-data-1>: <value-data-1>
         ...     <key-data-2>: <value-data-2>
+
+        This object support multiple config paths if you pass the `conf_paths`
+    key to the `extras` parameter.
     """
 
     def __init__(
         self,
         name: str,
-        conf_path: Path,
+        *,
+        path: Optional[Union[str, Path]] = None,
         externals: DictData | None = None,
+        extras: DictData | None = None,
     ) -> None:
-        self.conf_path: Path = conf_path
+        self.path: Path = Path(dynamic("conf_path", f=path, extras=extras))
         self.externals: DictData = externals or {}
-
-        self.data: DictData = {}
-        for file in glob_files(conf_path):
-
-            if self.is_ignore(file, conf_path):
-                continue
-
-            if data := self.filter_yaml(file, name=name):
-                self.data = data
+        self.extras: DictData = extras or {}
+        self.data: DictData = self.find(
+            name,
+            path=path,
+            paths=self.extras.get("conf_paths"),
+            extras=extras,
+        )
 
         # VALIDATE: check the data that reading should not empty.
         if not self.data:
             raise ValueError(
-                f"Config {name!r} does not found on conf path: "
-                f"{self.conf_path}."
+                f"Config {name!r} does not found on the conf path: {self.path}."
             )
 
         self.data.update(self.externals)
 
+    @classmethod
+    def find(
+        cls,
+        name: str,
+        *,
+        path: Optional[Path] = None,
+        paths: Optional[list[Path]] = None,
+        extras: Optional[DictData] = None,
+    ) -> DictData:
+        """Find data with specific key and return the latest modify date data if
+        this key exists multiple files.
+
+        :param name: (str) A name of data that want to find.
+        :param path: (Path) A config path object.
+        :param paths: (list[Path]) A list of config path object.
+        :param extras: (DictData)  An extra parameter that use to override core
+            config values.
+
+        :rtype: DictData
+        """
+        path: Path = dynamic("conf_path", f=path, extras=extras)
+        if not paths:
+            paths: list[Path] = [path]
+        elif not isinstance(paths, list):
+            raise TypeError(
+                f"Multi-config paths does not support for type: {type(paths)}"
+            )
+        else:
+            paths.append(path)
+
+        all_data: list[tuple[float, DictData]] = []
+        for path in paths:
+            for file in glob_files(path):
+
+                if cls.is_ignore(file, path):
+                    continue
+
+                if data := cls.filter_yaml(file, name=name):
+                    all_data.append((file.lstat().st_mtime, data))
+
+        return {} if not all_data else max(all_data, key=lambda x: x[0])[1]
+
     @classmethod
     def finds(
         cls,
         obj: object,
-        conf_path: Path,
         *,
-        included: list[str] | None = None,
+        path: Optional[Path] = None,
+        paths: Optional[list[Path]] = None,
         excluded: list[str] | None = None,
+        extras: Optional[DictData] = None,
     ) -> Iterator[tuple[str, DictData]]:
         """Find all data that match with object type in config path. This class
         method can use include and exclude list of identity name for filter and
         adds-on.
 
         :param obj: An object that want to validate matching before return.
-        :param conf_path: A config object.
-        :param included: An excluded list of data key that want to reject this
-            data if any key exist.
+        :param path: A config path object.
+        :param paths: (list[Path]) A list of config path object.
         :param excluded: An included list of data key that want to filter from
             data.
+        :param extras: (DictData) An extra parameter that use to override core
+            config values.
 
         :rtype: Iterator[tuple[str, DictData]]
         """
-        exclude: list[str] = excluded or []
-        for file in glob_files(conf_path):
-
-            if cls.is_ignore(file, conf_path):
-                continue
+        excluded: list[str] = excluded or []
+        path: Path = dynamic("conf_path", f=path, extras=extras)
+        paths: Optional[list[Path]] = paths or (extras or {}).get("conf_paths")
+        if not paths:
+            paths: list[Path] = [path]
+        elif not isinstance(paths, list):
+            raise TypeError(
+                f"Multi-config paths does not support for type: {type(paths)}"
+            )
+        else:
+            paths.append(path)
 
-            for key, data in cls.filter_yaml(file).items():
+        all_data: dict[str, list[tuple[float, DictData]]] = {}
+        for path in paths:
+            for file in glob_files(path):
 
-                if key in exclude:
+                if cls.is_ignore(file, path):
                     continue
 
-                if data.get("type", "") == obj.__name__:
-                    yield key, (
-                        {k: data[k] for k in data if k in included}
-                        if included
-                        else data
-                    )
+                for key, data in cls.filter_yaml(file).items():
+
+                    if key in excluded:
+                        continue
+
+                    if (
+                        data.get("type", "")
+                        == (obj if isclass(obj) else obj.__class__).__name__
+                    ):
+                        marking: tuple[float, DictData] = (
+                            file.lstat().st_mtime,
+                            data,
+                        )
+                        if key in all_data:
+                            all_data[key].append(marking)
+                        else:
+                            all_data[key] = [marking]
+
+        for key in all_data:
+            yield key, max(all_data[key], key=lambda x: x[0])[1]
 
     @classmethod
     def is_ignore(
         cls,
         file: Path,
-        conf_path: Path,
+        path: Path,
         *,
         ignore_filename: Optional[str] = None,
     ) -> bool:
         """Check this file was ignored.
 
         :param file: (Path) A file path that want to check.
-        :param conf_path: (Path) A config path that want to read the config
+        :param path: (Path) A config path that want to read the config
             ignore file.
-        :param ignore_filename: (str) An ignore filename.
+        :param ignore_filename: (str) An ignore filename. Default is
+            `.confignore` filename.
 
         :rtype: bool
         """
         ignore_filename: str = ignore_filename or ".confignore"
-        return is_ignored(file, read_ignore(conf_path / ignore_filename))
+        return is_ignored(file, read_ignore(path / ignore_filename))
 
     @classmethod
     def filter_yaml(cls, file: Path, name: str | None = None) -> DictData:
@@ -356,57 +439,36 @@ def dynamic(
     """Dynamic get config if extra value was passed at run-time.
 
     :param key: (str) A config key that get from Config object.
-    :param f: An inner config function scope.
+    :param f: (T) An inner config function scope.
     :param extras: An extra values that pass at run-time.
+
+    :rtype: T
     """
-    rsx: Optional[T] = extras[key] if extras and key in extras else None
-    rs: Optional[T] = getattr(config, key, None) if f is None else f
-    if rsx is not None and not isinstance(rsx, type(rs)):
+    extra: Optional[T] = (extras or {}).get(key, None)
+    conf: Optional[T] = getattr(config, key, None) if f is None else f
+    if extra is None:
+        return conf
+    if not isinstance(extra, type(conf)):
         raise TypeError(
-            f"Type of config {key!r} from extras: {rsx!r} does not valid "
-            f"as config {type(rs)}."
+            f"Type of config {key!r} from extras: {extra!r} does not valid "
+            f"as config {type(conf)}."
         )
-    return rsx if rsx is not None else rs
+    return extra
 
 
-class Loader(SimLoad):
-    """Loader Object that get the config `yaml` file from current path.
+class Loader(Protocol):  # pragma: no cov
+    type: str
+    path: Path
+    data: DictData
+    extras: DictData
+    externals: DictData
 
-    :param name: (str) A name of config data that will read by Yaml Loader object.
-    :param externals: (DictData) An external parameters
-    """
+    def __init__(self, *args, **kwargs) -> None: ...
 
     @classmethod
-    def finds(
-        cls,
-        obj: object,
-        *,
-        path: Path | None = None,
-        included: list[str] | None = None,
-        excluded: list[str] | None = None,
-        **kwargs,
-    ) -> Iterator[tuple[str, DictData]]:
-        """Override the find class method from the Simple Loader object.
-
-        :param obj: An object that want to validate matching before return.
-        :param path: (Path) A override config path.
-        :param included: An excluded list of data key that want to reject this
-            data if any key exist.
-        :param excluded: An included list of data key that want to filter from
-            data.
-
-        :rtype: Iterator[tuple[str, DictData]]
-        """
-        return super().finds(
-            obj=obj,
-            conf_path=(path or config.conf_path),
-            included=included,
-            excluded=excluded,
-        )
+    def find(cls, name: str, *args, **kwargs) -> DictData: ...
 
-    def __init__(self, name: str, externals: DictData) -> None:
-        super().__init__(
-            name,
-            conf_path=dynamic("conf_path", extras=externals),
-            externals=externals,
-        )
+    @classmethod
+    def finds(
+        cls, obj: object, *args, **kwargs
+    ) -> Iterator[tuple[str, DictData]]: ...

ddeutil/workflow/{cron.py → event.py}

@@ -3,11 +3,14 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
+"""Event module that store all event object. Now, it has only `On` and `OnYear`
+model these are schedule with crontab event.
+"""
 from __future__ import annotations
 
 from dataclasses import fields
 from datetime import datetime
-from typing import Annotated, Literal, Union
+from typing import Annotated, Any, Literal, Union
 from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
 
 from pydantic import BaseModel, ConfigDict, Field, ValidationInfo
@@ -17,11 +20,13 @@ from typing_extensions import Self
 
 from .__cron import WEEKDAYS, CronJob, CronJobYear, CronRunner, Options
 from .__types import DictData, DictStr
-from .conf import Loader
+from .conf import FileLoad
+
+Interval = Literal["daily", "weekly", "monthly"]
 
 
 def interval2crontab(
-    interval: Literal["daily", "weekly", "monthly"],
+    interval: Interval,
     *,
     day: str | None = None,
     time: str = "00:00",
@@ -59,10 +64,11 @@ def interval2crontab(
 
 
 class On(BaseModel):
-    """On Pydantic model (Warped crontab object by model).
+    """On model (Warped crontab object by Pydantic model) to keep crontab value
+    and generate CronRunner object from this crontab value.
 
-    See Also:
-        * `generate()` is the main use-case of this schedule object.
+    Methods:
+        - generate: is the main use-case of this schedule object.
     """
 
     model_config = ConfigDict(arbitrary_types_allowed=True)
@@ -90,40 +96,48 @@ class On(BaseModel):
             description="A timezone string value",
             alias="timezone",
         ),
-    ] = "Etc/UTC"
+    ] = "UTC"
 
     @classmethod
     def from_value(cls, value: DictStr, extras: DictData) -> Self:
         """Constructor from values that will generate crontab by function.
 
-        :param value: A mapping value that will generate crontab before create
-            schedule model.
-        :param extras: An extras parameter that will keep in extras.
+        :param value: (DictStr) A mapping value that will generate crontab
+            before create schedule model.
+        :param extras: (DictData) An extra parameter that use to override core
+            config value.
         """
         passing: DictStr = {}
+
         if "timezone" in value:
             passing["tz"] = value.pop("timezone")
+        elif "tz" in value:
+            passing["tz"] = value.pop("tz")
+
         passing["cronjob"] = interval2crontab(
             **{v: value[v] for v in value if v in ("interval", "day", "time")}
         )
+        print(passing)
         return cls(extras=extras | passing.pop("extras", {}), **passing)
 
     @classmethod
     def from_conf(
         cls,
         name: str,
+        *,
         extras: DictData | None = None,
     ) -> Self:
-        """Constructor from the name of config that will use loader object for
-        getting the data.
+        """Constructor from the name of config loader that will use loader
+        object for getting the `On` data.
 
-        :param name: A name of config that will get from loader.
-        :param extras: An extra parameter that will keep in extras.
+        :param name: (str) A name of config that will get from loader.
+        :param extras: (DictData) An extra parameter that use to override core
+            config values.
 
         :rtype: Self
         """
         extras: DictData = extras or {}
-        loader: Loader = Loader(name, externals=extras)
+        loader: FileLoad = FileLoad(name, extras=extras)
 
         # NOTE: Validate the config type match with current connection model
         if loader.type != cls.__name__:
@@ -155,17 +169,17 @@ class On(BaseModel):
         )
 
     @model_validator(mode="before")
-    def __prepare_values(cls, values: DictData) -> DictData:
+    def __prepare_values(cls, data: Any) -> Any:
         """Extract tz key from value and change name to timezone key.
 
-        :param values: (DictData) A data that want to pass for create an On
+        :param data: (DictData) A data that want to pass for create an On
             model.
 
         :rtype: DictData
         """
-        if tz := values.pop("tz", None):
-            values["timezone"] = tz
-        return values
+        if isinstance(data, dict) and (tz := data.pop("tz", None)):
+            data["timezone"] = tz
+        return data
 
     @field_validator("tz")
     def __validate_tz(cls, value: str) -> str:
@@ -238,6 +252,9 @@ class On(BaseModel):
         """Return a next datetime from Cron runner object that start with any
         date that given from input.
 
+        :param start: (str | datetime) A start datetime that use to generate
+            the CronRunner object.
+
         :rtype: CronRunner
         """
         runner: CronRunner = self.generate(start=start)