ddeutil-workflow 0.0.55__py3-none-any.whl → 0.0.56__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1 +1 @@
-__version__: str = "0.0.55"
+__version__: str = "0.0.56"

ddeutil/workflow/__init__.py

@@ -7,16 +7,18 @@ from .__cron import CronJob, CronRunner
 from .__types import DictData, DictStr, Matrix, Re, TupleStr
 from .conf import (
     Config,
-    Loader,
+    FileLoad,
     config,
     env,
 )
-from .cron import *
+from .event import *
 from .exceptions import *
 from .job import *
 from .logs import (
     Audit,
     AuditModel,
+    FileAudit,
+    FileTrace,
     Trace,
     TraceData,
     TraceMeta,

ddeutil/workflow/__main__.py

@@ -0,0 +1,30 @@
+import typer
+
+app = typer.Typer()
+
+
+@app.callback()
+def callback():
+    """
+    Awesome Portal Gun
+    """
+
+
+@app.command()
+def provision():
+    """
+    Shoot the portal gun
+    """
+    typer.echo("Shooting portal gun")
+
+
+@app.command()
+def job():
+    """
+    Load the portal gun
+    """
+    typer.echo("Loading portal gun")
+
+
+if __name__ == "__main__":
+    app()

ddeutil/workflow/conf.py

@@ -7,24 +7,26 @@ from __future__ import annotations
 
 import json
 import os
+from abc import ABC, abstractmethod
 from collections.abc import Iterator
 from datetime import timedelta
 from functools import cached_property
+from inspect import isclass
 from pathlib import Path
-from typing import Final, Optional, TypeVar
+from typing import Final, Optional, Protocol, TypeVar, Union
 from zoneinfo import ZoneInfo
 
 from ddeutil.core import str2bool
 from ddeutil.io import YamlFlResolve
 from ddeutil.io.paths import glob_files, is_ignored, read_ignore
 
-from .__types import DictData, TupleStr
+from .__types import DictData
 
 T = TypeVar("T")
 PREFIX: Final[str] = "WORKFLOW"
 
 
-def env(var: str, default: str | None = None) -> str | None:  # pragma: no cov
+def env(var: str, default: str | None = None) -> str | None:
     """Get environment variable with uppercase and adding prefix string.
 
     :param var: (str) A env variable name.
@@ -35,17 +37,6 @@ def env(var: str, default: str | None = None) -> str | None:  # pragma: no cov
     return os.getenv(f"{PREFIX}_{var.upper().replace(' ', '_')}", default)
 
 
-__all__: TupleStr = (
-    "api_config",
-    "env",
-    "Config",
-    "SimLoad",
-    "Loader",
-    "config",
-    "dynamic",
-)
-
-
 class Config:  # pragma: no cov
     """Config object for keeping core configurations on the current session
     without changing when if the application still running.
@@ -188,7 +179,7 @@ class Config:  # pragma: no cov
             return timedelta(**json.loads(stop_boundary_delta_str))
         except Exception as err:
             raise ValueError(
-                "Config ``WORKFLOW_APP_STOP_BOUNDARY_DELTA`` can not parsing to"
+                "Config `WORKFLOW_APP_STOP_BOUNDARY_DELTA` can not parsing to"
                 f"timedelta with {stop_boundary_delta_str}."
             ) from err
 
@@ -209,110 +200,194 @@ class APIConfig:
         return str2bool(env("API_ENABLE_ROUTE_SCHEDULE", "true"))
 
 
-class SimLoad:
-    """Simple Load Object that will search config data by given some identity
-    value like name of workflow or on.
+class BaseLoad(ABC):
+
+    @classmethod
+    @abstractmethod
+    def find(cls, name: str, *args, **kwargs) -> DictData: ...
+
+    @classmethod
+    @abstractmethod
+    def finds(
+        cls, obj: object, *args, **kwargs
+    ) -> Iterator[tuple[str, DictData]]: ...
 
-    :param name: A name of config data that will read by Yaml Loader object.
-    :param conf_path: A config path object.
-    :param externals: An external parameters
+
+class FileLoad(BaseLoad):
+    """Base Load object that use to search config data by given some identity
+    value like name of `Workflow` or `On` templates.
+
+    :param name: (str) A name of key of config data that read with YAML
+        Environment object.
+    :param path: (Path) A config path object.
+    :param externals: (DictData) An external config data that want to add to
+        loaded config data.
+    :param extras: (DictDdata) An extra parameters that use to override core
+        config values.
+
+    :raise ValueError: If the data does not find on the config path with the
+        name parameter.
 
     Noted:
-        The config data should have ``type`` key for modeling validation that
+        The config data should have `type` key for modeling validation that
     make this loader know what is config should to do pass to.
 
         ... <identity-key>:
         ...     type: <importable-object>
         ...     <key-data-1>: <value-data-1>
         ...     <key-data-2>: <value-data-2>
+
+        This object support multiple config paths if you pass the `conf_paths`
+    key to the `extras` parameter.
     """
 
     def __init__(
         self,
         name: str,
-        conf_path: Path,
+        *,
+        path: Optional[Union[str, Path]] = None,
         externals: DictData | None = None,
+        extras: DictData | None = None,
     ) -> None:
-        self.conf_path: Path = conf_path
+        self.path: Path = Path(dynamic("conf_path", f=path, extras=extras))
         self.externals: DictData = externals or {}
-
-        self.data: DictData = {}
-        for file in glob_files(conf_path):
-
-            if self.is_ignore(file, conf_path):
-                continue
-
-            if data := self.filter_yaml(file, name=name):
-                self.data = data
+        self.extras: DictData = extras or {}
+        self.data: DictData = self.find(
+            name,
+            path=path,
+            paths=self.extras.get("conf_paths"),
+            extras=extras,
+        )
 
         # VALIDATE: check the data that reading should not empty.
         if not self.data:
             raise ValueError(
-                f"Config {name!r} does not found on conf path: "
-                f"{self.conf_path}."
+                f"Config {name!r} does not found on the conf path: {self.path}."
             )
 
         self.data.update(self.externals)
 
+    @classmethod
+    def find(
+        cls,
+        name: str,
+        *,
+        path: Optional[Path] = None,
+        paths: Optional[list[Path]] = None,
+        extras: Optional[DictData] = None,
+    ) -> DictData:
+        """Find data with specific key and return the latest modify date data if
+        this key exists multiple files.
+
+        :param name: (str) A name of data that want to find.
+        :param path: (Path) A config path object.
+        :param paths: (list[Path]) A list of config path object.
+        :param extras: (DictData)  An extra parameter that use to override core
+            config values.
+
+        :rtype: DictData
+        """
+        path: Path = dynamic("conf_path", f=path, extras=extras)
+        if not paths:
+            paths: list[Path] = [path]
+        elif not isinstance(paths, list):
+            raise TypeError(
+                f"Multi-config paths does not support for type: {type(paths)}"
+            )
+        else:
+            paths.append(path)
+
+        all_data: list[tuple[float, DictData]] = []
+        for path in paths:
+            for file in glob_files(path):
+
+                if cls.is_ignore(file, path):
+                    continue
+
+                if data := cls.filter_yaml(file, name=name):
+                    all_data.append((file.lstat().st_mtime, data))
+
+        return {} if not all_data else max(all_data, key=lambda x: x[0])[1]
+
     @classmethod
     def finds(
         cls,
         obj: object,
-        conf_path: Path,
         *,
-        included: list[str] | None = None,
+        path: Optional[Path] = None,
+        paths: Optional[list[Path]] = None,
         excluded: list[str] | None = None,
+        extras: Optional[DictData] = None,
     ) -> Iterator[tuple[str, DictData]]:
         """Find all data that match with object type in config path. This class
         method can use include and exclude list of identity name for filter and
         adds-on.
 
         :param obj: An object that want to validate matching before return.
-        :param conf_path: A config object.
-        :param included: An excluded list of data key that want to reject this
-            data if any key exist.
+        :param path: A config path object.
+        :param paths: (list[Path]) A list of config path object.
         :param excluded: An included list of data key that want to filter from
             data.
+        :param extras: (DictData) An extra parameter that use to override core
+            config values.
 
         :rtype: Iterator[tuple[str, DictData]]
         """
-        exclude: list[str] = excluded or []
-        for file in glob_files(conf_path):
+        excluded: list[str] = excluded or []
+        path: Path = dynamic("conf_path", f=path, extras=extras)
+        if not paths:
+            paths: list[Path] = [path]
+        else:
+            paths.append(path)
+
+        all_data: dict[str, list[tuple[float, DictData]]] = {}
+        for path in paths:
+            for file in glob_files(path):
+
+                if cls.is_ignore(file, path):
+                    continue
 
-            if cls.is_ignore(file, conf_path):
-                continue
+                for key, data in cls.filter_yaml(file).items():
 
-            for key, data in cls.filter_yaml(file).items():
+                    if key in excluded:
+                        continue
 
-                if key in exclude:
-                    continue
+                    if (
+                        data.get("type", "")
+                        == (obj if isclass(obj) else obj.__class__).__name__
+                    ):
+                        marking: tuple[float, DictData] = (
+                            file.lstat().st_mtime,
+                            data,
+                        )
+                        if key in all_data:
+                            all_data[key].append(marking)
+                        else:
+                            all_data[key] = [marking]
 
-                if data.get("type", "") == obj.__name__:
-                    yield key, (
-                        {k: data[k] for k in data if k in included}
-                        if included
-                        else data
-                    )
+        for key in all_data:
+            yield key, max(all_data[key], key=lambda x: x[0])[1]
 
     @classmethod
     def is_ignore(
         cls,
         file: Path,
-        conf_path: Path,
+        path: Path,
         *,
         ignore_filename: Optional[str] = None,
     ) -> bool:
         """Check this file was ignored.
 
         :param file: (Path) A file path that want to check.
-        :param conf_path: (Path) A config path that want to read the config
+        :param path: (Path) A config path that want to read the config
             ignore file.
-        :param ignore_filename: (str) An ignore filename.
+        :param ignore_filename: (str) An ignore filename. Default is
+            `.confignore` filename.
 
         :rtype: bool
         """
         ignore_filename: str = ignore_filename or ".confignore"
-        return is_ignored(file, read_ignore(conf_path / ignore_filename))
+        return is_ignored(file, read_ignore(path / ignore_filename))
 
     @classmethod
     def filter_yaml(cls, file: Path, name: str | None = None) -> DictData:
@@ -369,44 +444,19 @@ def dynamic(
     return rsx if rsx is not None else rs
 
 
-class Loader(SimLoad):
-    """Loader Object that get the config `yaml` file from current path.
+class Loader(Protocol):  # pragma: no cov
+    type: str
+    path: Path
+    data: DictData
+    extras: DictData
+    externals: DictData
 
-    :param name: (str) A name of config data that will read by Yaml Loader object.
-    :param externals: (DictData) An external parameters
-    """
+    def __init__(self, *args, **kwargs) -> None: ...
 
     @classmethod
-    def finds(
-        cls,
-        obj: object,
-        *,
-        path: Path | None = None,
-        included: list[str] | None = None,
-        excluded: list[str] | None = None,
-        **kwargs,
-    ) -> Iterator[tuple[str, DictData]]:
-        """Override the find class method from the Simple Loader object.
+    def find(cls, name: str, *args, **kwargs) -> DictData: ...
 
-        :param obj: An object that want to validate matching before return.
-        :param path: (Path) A override config path.
-        :param included: An excluded list of data key that want to reject this
-            data if any key exist.
-        :param excluded: An included list of data key that want to filter from
-            data.
-
-        :rtype: Iterator[tuple[str, DictData]]
-        """
-        return super().finds(
-            obj=obj,
-            conf_path=(path or config.conf_path),
-            included=included,
-            excluded=excluded,
-        )
-
-    def __init__(self, name: str, externals: DictData) -> None:
-        super().__init__(
-            name,
-            conf_path=dynamic("conf_path", extras=externals),
-            externals=externals,
-        )
+    @classmethod
+    def finds(
+        cls, obj: object, *args, **kwargs
+    ) -> Iterator[tuple[str, DictData]]: ...

ddeutil/workflow/{cron.py → event.py}

@@ -3,11 +3,14 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
+"""Event module that store all event object. Now, it has only `On` and `OnYear`
+model these are schedule with crontab event.
+"""
 from __future__ import annotations
 
 from dataclasses import fields
 from datetime import datetime
-from typing import Annotated, Literal, Union
+from typing import Annotated, Any, Literal, Union
 from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
 
 from pydantic import BaseModel, ConfigDict, Field, ValidationInfo
@@ -17,11 +20,13 @@ from typing_extensions import Self
 
 from .__cron import WEEKDAYS, CronJob, CronJobYear, CronRunner, Options
 from .__types import DictData, DictStr
-from .conf import Loader
+from .conf import FileLoad
+
+Interval = Literal["daily", "weekly", "monthly"]
 
 
 def interval2crontab(
-    interval: Literal["daily", "weekly", "monthly"],
+    interval: Interval,
     *,
     day: str | None = None,
     time: str = "00:00",
@@ -59,10 +64,11 @@ def interval2crontab(
 
 
 class On(BaseModel):
-    """On Pydantic model (Warped crontab object by model).
+    """On model (Warped crontab object by Pydantic model) to keep crontab value
+    and generate CronRunner object from this crontab value.
 
-    See Also:
-        * `generate()` is the main use-case of this schedule object.
+    Methods:
+        - generate: is the main use-case of this schedule object.
     """
 
     model_config = ConfigDict(arbitrary_types_allowed=True)
@@ -90,19 +96,24 @@ class On(BaseModel):
             description="A timezone string value",
             alias="timezone",
         ),
-    ] = "Etc/UTC"
+    ] = "UTC"
 
     @classmethod
     def from_value(cls, value: DictStr, extras: DictData) -> Self:
         """Constructor from values that will generate crontab by function.
 
-        :param value: A mapping value that will generate crontab before create
-            schedule model.
-        :param extras: An extras parameter that will keep in extras.
+        :param value: (DictStr) A mapping value that will generate crontab
+            before create schedule model.
+        :param extras: (DictData) An extra parameter that use to override core
+            config value.
         """
         passing: DictStr = {}
+
         if "timezone" in value:
             passing["tz"] = value.pop("timezone")
+        elif "tz" in value:
+            passing["tz"] = value.pop("tz")
+
         passing["cronjob"] = interval2crontab(
             **{v: value[v] for v in value if v in ("interval", "day", "time")}
         )
@@ -112,18 +123,20 @@ class On(BaseModel):
     def from_conf(
         cls,
         name: str,
+        *,
         extras: DictData | None = None,
     ) -> Self:
-        """Constructor from the name of config that will use loader object for
-        getting the data.
+        """Constructor from the name of config loader that will use loader
+        object for getting the `On` data.
 
-        :param name: A name of config that will get from loader.
-        :param extras: An extra parameter that will keep in extras.
+        :param name: (str) A name of config that will get from loader.
+        :param extras: (DictData) An extra parameter that use to override core
+            config values.
 
         :rtype: Self
         """
         extras: DictData = extras or {}
-        loader: Loader = Loader(name, externals=extras)
+        loader: FileLoad = FileLoad(name, extras=extras)
 
         # NOTE: Validate the config type match with current connection model
         if loader.type != cls.__name__:
@@ -155,17 +168,17 @@ class On(BaseModel):
         )
 
     @model_validator(mode="before")
-    def __prepare_values(cls, values: DictData) -> DictData:
+    def __prepare_values(cls, data: Any) -> Any:
         """Extract tz key from value and change name to timezone key.
 
-        :param values: (DictData) A data that want to pass for create an On
+        :param data: (DictData) A data that want to pass for create an On
             model.
 
         :rtype: DictData
         """
-        if tz := values.pop("tz", None):
-            values["timezone"] = tz
-        return values
+        if isinstance(data, dict) and (tz := data.pop("tz", None)):
+            data["timezone"] = tz
+        return data
 
     @field_validator("tz")
     def __validate_tz(cls, value: str) -> str:
@@ -238,6 +251,9 @@ class On(BaseModel):
         """Return a next datetime from Cron runner object that start with any
         date that given from input.
 
+        :param start: (str | datetime) A start datetime that use to generate
+            the CronRunner object.
+
         :rtype: CronRunner
         """
         runner: CronRunner = self.generate(start=start)

ddeutil/workflow/exceptions.py

@@ -22,7 +22,12 @@ ErrorData = TypedDict(
 
 
 def to_dict(exception: Exception) -> ErrorData:  # pragma: no cov
-    """Create dict data from exception instance."""
+    """Create dict data from exception instance.
+
+    :param exception: An exception object.
+
+    :rtype: ErrorData
+    """
     return {
         "class": exception,
         "name": exception.__class__.__name__,
@@ -33,6 +38,10 @@ def to_dict(exception: Exception) -> ErrorData:  # pragma: no cov
 class BaseWorkflowException(Exception):
 
     def to_dict(self) -> ErrorData:
+        """Return ErrorData data from the current exception object.
+
+        :rtype: ErrorData
+        """
         return to_dict(self)
 
 

ddeutil/workflow/job.py

@@ -140,14 +140,19 @@ class Strategy(BaseModel):
 
     fail_fast: bool = Field(
         default=False,
+        description=(
+            "A fail-fast flag that use to cancel strategy execution when it "
+            "has some execution was failed."
+        ),
         alias="fail-fast",
     )
     max_parallel: int = Field(
         default=1,
         gt=0,
+        lt=10,
         description=(
             "The maximum number of executor thread pool that want to run "
-            "parallel"
+            "parallel. This value should gather than 0 and less than 10."
         ),
         alias="max-parallel",
     )

ddeutil/workflow/scheduler.py

@@ -34,7 +34,7 @@ from heapq import heappop, heappush
 from pathlib import Path
 from textwrap import dedent
 from threading import Thread
-from typing import Callable, Optional, TypedDict, Union
+from typing import Any, Callable, Optional, TypedDict, Union
 
 from pydantic import BaseModel, Field, ValidationInfo
 from pydantic.functional_validators import field_validator, model_validator
@@ -52,8 +52,8 @@ except ImportError:  # pragma: no cov
 
 from .__cron import CronRunner
 from .__types import DictData, TupleStr
-from .conf import Loader, SimLoad, dynamic
-from .cron import On
+from .conf import FileLoad, Loader, dynamic
+from .event import On
 from .exceptions import ScheduleException, WorkflowException
 from .logs import Audit, get_audit
 from .result import SUCCESS, Result
@@ -113,21 +113,15 @@ class ScheduleWorkflow(BaseModel):
     )
 
     @model_validator(mode="before")
-    def __prepare_before__(cls, values: DictData) -> DictData:
-        """Prepare incoming values before validating with model fields.
-
-        :rtype: DictData
-        """
-        # VALIDATE: Prepare a workflow name that should not include space.
-        if name := values.get("name"):
-            values["name"] = name.replace(" ", "_")
-
-        # VALIDATE: Add default the alias field with the name.
-        if not values.get("alias"):
-            values["alias"] = values.get("name")
-
-        cls.__bypass_on(values, extras=values.get("extras"))
-        return values
+    def __prepare_before__(cls, data: Any) -> Any:
+        """Prepare incoming values before validating with model fields."""
+        if isinstance(data, dict):
+            # VALIDATE: Add default the alias field with the name.
+            if "alias" not in data:
+                data["alias"] = data.get("name")
+
+            cls.__bypass_on(data, extras=data.get("extras"))
+        return data
 
     @classmethod
     def __bypass_on(
@@ -135,8 +129,10 @@ class ScheduleWorkflow(BaseModel):
     ) -> DictData:
         """Bypass and prepare the on data to loaded config data.
 
-        :param data: A data that want to validate for model initialization.
-        :param extras: An extra parameter that want to override core config.
+        :param data: (DictData) A data that want to validate for the model
+            initialization.
+        :param extras: (DictData) An extra parameter that want to override core
+            config values.
 
         :rtype: DictData
         """
@@ -151,7 +147,7 @@ class ScheduleWorkflow(BaseModel):
             # NOTE: Pass on value to Loader and keep on model object to on
             #   field.
             data["on"] = [
-                Loader(n, externals=extras).data if isinstance(n, str) else n
+                FileLoad(n, externals=extras).data if isinstance(n, str) else n
                 for n in on
             ]
 
@@ -162,6 +158,10 @@ class ScheduleWorkflow(BaseModel):
         """Validate the on fields should not contain duplicate values and if it
         contains every minute value, it should have only one on value.
 
+        :param value: (list[On]) A list of `On` object.
+        :param info: (ValidationInfo) An validation info object for getting an
+            extra parameter.
+
         :rtype: list[On]
         """
         set_ons: set[str] = {str(on.cronjob) for on in value}
@@ -191,23 +191,22 @@ class ScheduleWorkflow(BaseModel):
             This task creation need queue to tracking release date already
         mapped or not.
 
-        :param start_date: A start date that get from the workflow schedule.
-        :param queue: A mapping of name and list of datetime for queue.
+        :param start_date: (datetime) A start datetime that get from the
+            workflow schedule.
+        :param queue: (dict[str, ReleaseQueue]) A mapping of name and list of
+            datetime for queue.
 
         :rtype: list[WorkflowTask]
         :return: Return the list of WorkflowTask object from the specific
             input datetime that mapping with the on field.
         """
-        workflow_tasks: list[WorkflowTask] = []
-
-        # NOTE: Loading workflow model from the name of workflow.
         wf: Workflow = Workflow.from_conf(self.name, extras=self.extras)
         wf_queue: ReleaseQueue = queue[self.alias]
 
         # IMPORTANT: Create the default 'on' value if it does not pass the `on`
         #   field to the Schedule object.
         ons: list[On] = self.on or wf.on.copy()
-
+        workflow_tasks: list[WorkflowTask] = []
         for on in ons:
 
             # NOTE: Create CronRunner instance from the start_date param.
@@ -250,7 +249,7 @@ class Schedule(BaseModel):
     )
     workflows: list[ScheduleWorkflow] = Field(
         default_factory=list,
-        description="A list of ScheduleWorkflow models.",
+        description="A list of ScheduleWorkflow model.",
     )
 
     @field_validator("desc", mode="after")
@@ -267,6 +266,8 @@ class Schedule(BaseModel):
     def from_conf(
         cls,
         name: str,
+        *,
+        path: Optional[Path] = None,
         extras: DictData | None = None,
     ) -> Self:
         """Create Schedule instance from the Loader object that only receive
@@ -274,6 +275,7 @@ class Schedule(BaseModel):
         searching configuration data of this schedule model in conf path.
 
         :param name: (str) A schedule name that want to pass to Loader object.
+        :param path: (Path) An override config path.
         :param extras: An extra parameters that want to pass to Loader
             object.
 
@@ -281,55 +283,14 @@ class Schedule(BaseModel):
 
         :rtype: Self
         """
-        loader: Loader = Loader(name, externals=(extras or {}))
+        loader: Loader = FileLoad(name, path=path, extras=extras)
 
         # NOTE: Validate the config type match with current connection model
         if loader.type != cls.__name__:
             raise ValueError(f"Type {loader.type} does not match with {cls}")
 
         loader_data: DictData = copy.deepcopy(loader.data)
-
-        # NOTE: Add name to loader data
-        loader_data["name"] = name.replace(" ", "_")
-
-        if extras:
-            loader_data["extras"] = extras
-
-        return cls.model_validate(obj=loader_data)
-
-    @classmethod
-    def from_path(
-        cls,
-        name: str,
-        path: Path,
-        extras: DictData | None = None,
-    ) -> Self:
-        """Create Schedule instance from the SimLoad object that receive an
-        input schedule name and conf path. The loader object will use this
-        schedule name to searching configuration data of this schedule model
-        in conf path.
-
-        :param name: (str) A schedule name that want to pass to Loader object.
-        :param path: (Path) A config path that want to search.
-        :param extras: An external parameters that want to pass to Loader
-            object.
-
-        :raise ValueError: If the type does not match with current object.
-
-        :rtype: Self
-        """
-        loader: SimLoad = SimLoad(
-            name, conf_path=path, externals=(extras or {})
-        )
-
-        # NOTE: Validate the config type match with current connection model
-        if loader.type != cls.__name__:
-            raise ValueError(f"Type {loader.type} does not match with {cls}")
-
-        loader_data: DictData = copy.deepcopy(loader.data)
-
-        # NOTE: Add name to loader data
-        loader_data["name"] = name.replace(" ", "_")
+        loader_data["name"] = name
 
         if extras:
             loader_data["extras"] = extras

ddeutil/workflow/workflow.py

@@ -28,7 +28,7 @@ from pathlib import Path
 from queue import Queue
 from textwrap import dedent
 from threading import Event
-from typing import Optional
+from typing import Any, Optional, Union
 
 from pydantic import BaseModel, ConfigDict, Field, ValidationInfo
 from pydantic.dataclasses import dataclass
@@ -37,8 +37,8 @@ from typing_extensions import Self
 
 from .__cron import CronJob, CronRunner
 from .__types import DictData, TupleStr
-from .conf import Loader, SimLoad, dynamic
-from .cron import On
+from .conf import FileLoad, Loader, dynamic
+from .event import On
 from .exceptions import JobException, UtilException, WorkflowException
 from .job import Job
 from .logs import Audit, get_audit
@@ -144,7 +144,7 @@ class Release:
         return NotImplemented
 
     def __lt__(self, other: Release | datetime) -> bool:
-        """Override equal property that will compare only the same type or
+        """Override less-than property that will compare only the same type or
         datetime.
 
         :rtype: bool
@@ -172,7 +172,7 @@ class ReleaseQueue:
     @classmethod
     def from_list(
         cls,
-        queue: list[datetime] | list[Release] | None = None,
+        queue: Optional[Union[list[datetime], list[Release]]] = None,
         extras: Optional[DictData] = None,
     ) -> Self:
         """Construct ReleaseQueue object from an input queue value that passing
@@ -190,7 +190,6 @@ class ReleaseQueue:
             return cls()
 
         if isinstance(queue, list):
-
             if all(isinstance(q, datetime) for q in queue):
                 return cls(
                     queue=[
@@ -233,7 +232,11 @@ class ReleaseQueue:
         )
 
     def mark_complete(self, value: Release) -> Self:
-        """Push Release to the complete queue.
+        """Push Release to the complete queue. After push the release, it will
+        delete old release base on the `CORE_MAX_QUEUE_COMPLETE_HIST` value.
+
+        :param value: (Release) A Release value that want to push to the
+            complete field.
 
         :rtype: Self
         """
@@ -262,7 +265,7 @@ class ReleaseQueue:
         force_run: bool = False,
         extras: Optional[DictData] = None,
     ) -> Self:
-        """Generate Release model to queue.
+        """Generate a Release model to the queue field with an input CronRunner.
 
         Steps:
             - Create Release object from the current date that not reach the end
@@ -277,9 +280,10 @@ class ReleaseQueue:
         :param runner: (CronRunner) A CronRunner object.
         :param name: (str) A target name that want to check at pointer of audit.
         :param offset: (float) An offset in second unit for time travel.
-        :param force_run: A flag that allow to release workflow if the audit
-            with that release was pointed.
-        :param extras: An extra parameter that want to override core config.
+        :param force_run: (bool) A flag that allow to release workflow if the
+            audit with that release was pointed. (Default is False).
+        :param extras: (DictDatA) An extra parameter that want to override core
+            config values.
 
         :rtype: ReleaseQueue
 
@@ -287,7 +291,7 @@ class ReleaseQueue:
         if runner.date > end_date:
             return self
 
-        workflow_release = Release(
+        release = Release(
             date=runner.date,
             offset=offset,
             end_date=end_date,
@@ -295,13 +299,11 @@ class ReleaseQueue:
             type=ReleaseType.POKE,
         )
 
-        while self.check_queue(workflow_release) or (
-            audit.is_pointed(
-                name=name, release=workflow_release.date, extras=extras
-            )
+        while self.check_queue(release) or (
+            audit.is_pointed(name=name, release=release.date, extras=extras)
             and not force_run
         ):
-            workflow_release = Release(
+            release = Release(
                 date=runner.next,
                 offset=offset,
                 end_date=end_date,
@@ -312,12 +314,12 @@ class ReleaseQueue:
         if runner.date > end_date:
             return self
 
-        heappush(self.queue, workflow_release)
+        heappush(self.queue, release)
         return self
 
 
 class Workflow(BaseModel):
-    """Workflow Pydantic model.
+    """Workflow model that use to keep the `Job` and `On` models.
 
         This is the main future of this project because it uses to be workflow
     data for running everywhere that you want or using it to scheduler task in
@@ -355,6 +357,7 @@ class Workflow(BaseModel):
         cls,
         name: str,
         *,
+        path: Optional[Path] = None,
         extras: DictData | None = None,
         loader: type[Loader] = None,
     ) -> Self:
@@ -362,45 +365,8 @@ class Workflow(BaseModel):
         an input workflow name. The loader object will use this workflow name to
         searching configuration data of this workflow model in conf path.
 
-        :param name: A workflow name that want to pass to Loader object.
-        :param extras: An extra parameters that want to pass to Loader
-            object.
-        :param loader: A loader class for override default loader object.
-
-        :raise ValueError: If the type does not match with current object.
-
-        :rtype: Self
-        """
-        loader: Loader = (loader or Loader)(name, externals=(extras or {}))
-
-        # NOTE: Validate the config type match with current connection model
-        if loader.type != cls.__name__:
-            raise ValueError(f"Type {loader.type} does not match with {cls}")
-
-        loader_data: DictData = copy.deepcopy(loader.data)
-        loader_data["name"] = name.replace(" ", "_")
-
-        if extras:
-            loader_data["extras"] = extras
-
-        cls.__bypass_on__(loader_data, path=loader.conf_path, extras=extras)
-        return cls.model_validate(obj=loader_data)
-
-    @classmethod
-    def from_path(
-        cls,
-        name: str,
-        path: Path,
-        *,
-        extras: DictData | None = None,
-        loader: type[Loader] = None,
-    ) -> Self:
-        """Create Workflow instance from the specific path. The loader object
-        will use this workflow name and path to searching configuration data of
-        this workflow model.
-
         :param name: (str) A workflow name that want to pass to Loader object.
-        :param path: (Path) A config path that want to search.
+        :param path: (Path) An override config path.
         :param extras: (DictData) An extra parameters that want to override core
             config values.
         :param loader: A loader class for override default loader object.
@@ -409,21 +375,21 @@ class Workflow(BaseModel):
 
         :rtype: Self
         """
-        loader: SimLoad = (loader or SimLoad)(
-            name, conf_path=path, externals=(extras or {})
-        )
+        loader: type[Loader] = loader or FileLoad
+        load: Loader = loader(name, path=path, extras=extras)
+
         # NOTE: Validate the config type match with current connection model
-        if loader.type != cls.__name__:
-            raise ValueError(f"Type {loader.type} does not match with {cls}")
+        if load.type != cls.__name__:
+            raise ValueError(f"Type {load.type} does not match with {cls}")
 
-        loader_data: DictData = copy.deepcopy(loader.data)
-        loader_data["name"] = name.replace(" ", "_")
+        data: DictData = copy.deepcopy(load.data)
+        data["name"] = name
 
         if extras:
-            loader_data["extras"] = extras
+            data["extras"] = extras
 
-        cls.__bypass_on__(loader_data, path=path, extras=extras)
-        return cls.model_validate(obj=loader_data)
+        cls.__bypass_on__(data, path=load.path, extras=extras, loader=loader)
+        return cls.model_validate(obj=data)
 
     @classmethod
     def __bypass_on__(
@@ -431,6 +397,7 @@ class Workflow(BaseModel):
         data: DictData,
         path: Path,
         extras: DictData | None = None,
+        loader: type[Loader] = None,
     ) -> DictData:
         """Bypass the on data to loaded config data.
 
@@ -451,7 +418,7 @@ class Workflow(BaseModel):
             #   field.
             data["on"] = [
                 (
-                    SimLoad(n, conf_path=path, externals=(extras or {})).data
+                    (loader or FileLoad)(n, path=path, extras=extras).data
                     if isinstance(n, str)
                     else n
                 )
@@ -460,11 +427,10 @@ class Workflow(BaseModel):
         return data
 
     @model_validator(mode="before")
-    def __prepare_model_before__(cls, values: DictData) -> DictData:
+    def __prepare_model_before__(cls, data: Any) -> Any:
         """Prepare the params key in the data model before validating."""
-        # NOTE: Prepare params type if it is passing with only type value.
-        if params := values.pop("params", {}):
-            values["params"] = {
+        if isinstance(data, dict) and (params := data.pop("params", {})):
+            data["params"] = {
                 p: (
                     {"type": params[p]}
                     if isinstance(params[p], str)
@@ -472,7 +438,7 @@ class Workflow(BaseModel):
                 )
                 for p in params
             }
-        return values
+        return data
 
     @field_validator("desc", mode="after")
     def __dedent_desc__(cls, value: str) -> str:

{ddeutil_workflow-0.0.55.dist-info → ddeutil_workflow-0.0.56.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddeutil-workflow
-Version: 0.0.55
+Version: 0.0.56
 Summary: Lightweight workflow orchestration
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT

{ddeutil_workflow-0.0.55.dist-info → ddeutil_workflow-0.0.56.dist-info}/RECORD

@@ -1,20 +1,20 @@
-ddeutil/workflow/__about__.py,sha256=kfTS6Gqr5xkMNDVKm-NTS4vmeIl2Zj4oMBQGmq19LHA,28
+ddeutil/workflow/__about__.py,sha256=EXJHOyenQmmnoVgcw7W0m9HU1cg9EJV61611jGr4NDY,28
 ddeutil/workflow/__cron.py,sha256=h8rLeIUAAEB2SdZ4Jhch7LU1Yl3bbJ-iNNJ3tQ0eYVM,28095
-ddeutil/workflow/__init__.py,sha256=noE8LNRcgq32m9OnIFcQqh0P7PXWdp-SGmvBCYIXgf4,1338
-ddeutil/workflow/__main__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ddeutil/workflow/__init__.py,sha256=NXEhjzKFdIGa-jtIq9HXChLCjSXNPd8VJ8ltggxbBO8,1371
+ddeutil/workflow/__main__.py,sha256=x-sYedl4T8p6054aySk-EQX6vhytvPR0HvaBNYxMzp0,364
 ddeutil/workflow/__types.py,sha256=8jBdbfb3aZSetjz0mvNrpGHwwxJff7mK8_4v41cLqlc,4316
-ddeutil/workflow/conf.py,sha256=80rgmJKFU7BlH5xTLnghGzGhE8C6LFAQykd9mjHSjo8,12528
-ddeutil/workflow/cron.py,sha256=WS2MInn0Sp5DKlZDZH5VFZ5AA0Q3_AnBnYEU4lZSv4I,9779
-ddeutil/workflow/exceptions.py,sha256=r4Jrf9qtVPALU4wh4bnb_OYqC-StqSQJEmFC-_QK934,1408
-ddeutil/workflow/job.py,sha256=aVRWLMLv5vYFbckT6AKYrMu29FzXYESOEzDHhFIpUyo,34159
+ddeutil/workflow/conf.py,sha256=JaTfglfdgGe7M-nb2NeRska95MDmYapwKuAlZfzVdr4,14425
+ddeutil/workflow/event.py,sha256=qiUrkkVxOvYEFfxTWBsLCgYTyOWMY125abOPieY5Xqc,10319
+ddeutil/workflow/exceptions.py,sha256=0MvjCRBUsHfOm1kzMiC4Y22vb1_sfvTU0wAW7xZwtAo,1587
+ddeutil/workflow/job.py,sha256=T0zxzK682kYaNmVHTzNWeBrZu8QJ6J2eYCFlKto0vA4,34381
 ddeutil/workflow/logs.py,sha256=rsoBrUGQrooou18fg2yvPsB8NOaXnUA5ThQpBr_WVMg,26598
 ddeutil/workflow/params.py,sha256=FKY4Oo1Ze4QZKRfAk7rqKsi44YaJQAbqAtXM6vlO2hI,11392
 ddeutil/workflow/result.py,sha256=rI0S8-HanFDk1l6_BsYRRamzSfzKUy7bkKJUae1w_aQ,5708
 ddeutil/workflow/reusables.py,sha256=iXcS7Gg-71qVX4ln0ILTDx03cTtUnj_rNoXHTVdVrxc,17636
-ddeutil/workflow/scheduler.py,sha256=hk3-9R63DZH9J0PWbtBzaD8rqHyiOf03vTvAJDgsxTA,28279
+ddeutil/workflow/scheduler.py,sha256=oVSNwZ-iyXFOGXhsltzaDy7GDQejI9GalMHxa8JRcro,27063
 ddeutil/workflow/stages.py,sha256=E5XoMVijjcvm_YK8AbiA8xGAQUphCPTtGazW-oLAdeI,82543
 ddeutil/workflow/utils.py,sha256=NZPvPPP_5g4cigFcD7tHjIKLtKMeYAcb3oUhNyhTpJ0,7947
-ddeutil/workflow/workflow.py,sha256=F3I_c0LHMoJFZIFt1a92sxFMlQVkE-cH--TLyX2rwuo,46799
+ddeutil/workflow/workflow.py,sha256=vgVOwa79ZXWcOv2k6FG4I_FVuyswGwbglidWQJXSrsY,45739
 ddeutil/workflow/api/__init__.py,sha256=kY30dL8HPY8tY_GBmm7y_3OdoXzB1-EA2a96PLU0AQw,5278
 ddeutil/workflow/api/logs.py,sha256=NMTnOnsBrDB5129329xF2myLdrb-z9k1MQrmrP7qXJw,1818
 ddeutil/workflow/api/utils.py,sha256=uTtUFVLpiYYahXvCVx8sueRQ03K2Xw1id_gW3IMmX1U,5295
@@ -23,8 +23,9 @@ ddeutil/workflow/api/routes/job.py,sha256=8X5VLDJH6PumyNIY6JGRNBsf2gWN0eG9DzxRPS
 ddeutil/workflow/api/routes/logs.py,sha256=U6vOni3wd-ZTOwd3yVdSOpgyRmNdcgfngU5KlLM3Cww,5383
 ddeutil/workflow/api/routes/schedules.py,sha256=14RnaJKEGMSJtncI1H_QQVZNBe_jDS40PPRO6qFc3i0,4805
 ddeutil/workflow/api/routes/workflows.py,sha256=GJu5PiXEylswrXylEImpncySjeU9chrvrtjhiMCw2RQ,4529
-ddeutil_workflow-0.0.55.dist-info/licenses/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
-ddeutil_workflow-0.0.55.dist-info/METADATA,sha256=sTjAMsv4yIgw-CKKHyr-sg_yrgnnDT_0UMF3E2TP-Js,19008
-ddeutil_workflow-0.0.55.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-ddeutil_workflow-0.0.55.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
-ddeutil_workflow-0.0.55.dist-info/RECORD,,
+ddeutil_workflow-0.0.56.dist-info/licenses/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
+ddeutil_workflow-0.0.56.dist-info/METADATA,sha256=JYyoDdlPBgJukz3yFPseqjqTOwK7i2LejPgLhqkBXwo,19008
+ddeutil_workflow-0.0.56.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+ddeutil_workflow-0.0.56.dist-info/entry_points.txt,sha256=qDTpPSauL0ciO6T4iSVt8bJeYrVEkkoEEw_RlGx6Kgk,63
+ddeutil_workflow-0.0.56.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
+ddeutil_workflow-0.0.56.dist-info/RECORD,,

ddeutil_workflow-0.0.56.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+workflow-cli = ddeutil.workflow.__main__:app