ddeutil-workflow 0.0.54__tar.gz → 0.0.56__tar.gz

{ddeutil_workflow-0.0.54/src/ddeutil_workflow.egg-info → ddeutil_workflow-0.0.56}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddeutil-workflow
-Version: 0.0.54
+Version: 0.0.56
 Summary: Lightweight workflow orchestration
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT
@@ -121,12 +121,10 @@ flowchart LR
 
 > [!WARNING]
 > _**Disclaimer**_: I inspire the dynamic YAML statement from the [**GitHub Action**](https://github.com/features/actions),
-> and all configs pattern from several data orchestration framework tools from
-> my data engineering experience. :grimacing:
-
-> [!NOTE]
-> Other workflow orchestration tools that I interest and pick them to be inspiration
-> some for this package:
+> and my experience of data framework configs pattern. :grimacing:
+>
+> Other workflow orchestration services that I interest and pick them to be
+> this project inspiration:
 >
 > - [Google **Workflows**](https://cloud.google.com/workflows)
 > - [AWS **Step Functions**](https://aws.amazon.com/step-functions/)

{ddeutil_workflow-0.0.54 → ddeutil_workflow-0.0.56}/README.md

@@ -76,12 +76,10 @@ flowchart LR
 
 > [!WARNING]
 > _**Disclaimer**_: I inspire the dynamic YAML statement from the [**GitHub Action**](https://github.com/features/actions),
-> and all configs pattern from several data orchestration framework tools from
-> my data engineering experience. :grimacing:
-
-> [!NOTE]
-> Other workflow orchestration tools that I interest and pick them to be inspiration
-> some for this package:
+> and my experience of data framework configs pattern. :grimacing:
+>
+> Other workflow orchestration services that I interest and pick them to be
+> this project inspiration:
 >
 > - [Google **Workflows**](https://cloud.google.com/workflows)
 > - [AWS **Step Functions**](https://aws.amazon.com/step-functions/)

{ddeutil_workflow-0.0.54 → ddeutil_workflow-0.0.56}/pyproject.toml

@@ -57,6 +57,9 @@ docker = [
 Homepage = "https://github.com/ddeutils/ddeutil-workflow/"
 "Source Code" = "https://github.com/ddeutils/ddeutil-workflow/"
 
+[project.scripts]
+workflow-cli = "ddeutil.workflow.__main__:app"
+
 [tool.setuptools.dynamic]
 version = {attr = "ddeutil.workflow.__about__.__version__"}
 
@@ -79,11 +82,9 @@ source = ["src.ddeutil.workflow"]
 omit = [
     "src/ddeutil/workflow/__about__.py",
     "src/ddeutil/workflow/__cron.py",
-    "src/ddeutil/workflow/context.py",
     "src/ddeutil/workflow/api/__init__.py",
-    "src/ddeutil/workflow/api/api.py",
     "src/ddeutil/workflow/api/log.py",
-    "src/ddeutil/workflow/api/repeat.py",
+    "src/ddeutil/workflow/api/utils.py",
     "src/ddeutil/workflow/api/routes/__init__.py",
     "src/ddeutil/workflow/api/routes/job.py",
     "src/ddeutil/workflow/api/routes/logs.py",

ddeutil_workflow-0.0.56/src/ddeutil/workflow/__about__.py

@@ -0,0 +1 @@
+__version__: str = "0.0.56"

{ddeutil_workflow-0.0.54 → ddeutil_workflow-0.0.56}/src/ddeutil/workflow/__init__.py

@@ -7,16 +7,18 @@ from .__cron import CronJob, CronRunner
 from .__types import DictData, DictStr, Matrix, Re, TupleStr
 from .conf import (
     Config,
-    Loader,
+    FileLoad,
     config,
     env,
 )
-from .cron import *
+from .event import *
 from .exceptions import *
 from .job import *
 from .logs import (
     Audit,
     AuditModel,
+    FileAudit,
+    FileTrace,
     Trace,
     TraceData,
     TraceMeta,

ddeutil_workflow-0.0.56/src/ddeutil/workflow/__main__.py

@@ -0,0 +1,30 @@
+import typer
+
+app = typer.Typer()
+
+
+@app.callback()
+def callback():
+    """
+    Awesome Portal Gun
+    """
+
+
+@app.command()
+def provision():
+    """
+    Shoot the portal gun
+    """
+    typer.echo("Shooting portal gun")
+
+
+@app.command()
+def job():
+    """
+    Load the portal gun
+    """
+    typer.echo("Loading portal gun")
+
+
+if __name__ == "__main__":
+    app()

ddeutil_workflow-0.0.54/src/ddeutil/workflow/api/api.py → ddeutil_workflow-0.0.56/src/ddeutil/workflow/api/__init__.py

@@ -90,7 +90,7 @@ app.add_middleware(
 )
 
 
-@app.get("/")
+@app.get(path="/", response_class=UJSONResponse)
 async def health():
     """Index view that not return any template without json status."""
     return {"message": "Workflow already start up with healthy status."}

{ddeutil_workflow-0.0.54 → ddeutil_workflow-0.0.56}/src/ddeutil/workflow/api/routes/job.py

@@ -9,7 +9,7 @@ from typing import Any, Optional
 
 from fastapi import APIRouter
 from fastapi.responses import UJSONResponse
-from pydantic import BaseModel
+from pydantic import BaseModel, Field
 
 from ...__types import DictData
 from ...exceptions import JobException
@@ -18,33 +18,37 @@ from ...logs import get_logger
 from ...result import Result
 
 logger = get_logger("uvicorn.error")
+job_route = APIRouter(prefix="/job", tags=["job"])
 
 
-job_route = APIRouter(
-    prefix="/job",
-    tags=["job"],
-    default_response_class=UJSONResponse,
-)
+class ResultCreate(BaseModel):
+    """Create Result model for receive running IDs to create the Result
+    dataclass.
+    """
 
-
-class ResultPost(BaseModel):
-    context: DictData
-    run_id: str
-    parent_run_id: Optional[str] = None
+    run_id: str = Field(description="A running ID.")
+    parent_run_id: Optional[str] = Field(
+        default=None, description="A parent running ID."
+    )
 
 
-@job_route.post(path="/execute/")
+@job_route.post(path="/execute/", response_class=UJSONResponse)
 async def job_execute(
-    result: ResultPost,
+    result: ResultCreate,
     job: Job,
     params: dict[str, Any],
+    extras: Optional[dict[str, Any]] = None,
 ):
-    """Execute job via RestAPI."""
+    """Execute job via RestAPI with execute route path."""
     rs: Result = Result(
-        context=result.context,
         run_id=result.run_id,
         parent_run_id=result.parent_run_id,
+        extras=extras or {},
     )
+
+    if extras:
+        job.extras = extras
+
     context: DictData = {}
     try:
         job.set_outputs(
@@ -59,14 +63,11 @@ async def job_execute(
         rs.trace.error(f"[JOB]: {err.__class__.__name__}: {err}")
 
     return {
-        "message": "Start execute job via API.",
-        "result": {
-            "run_id": rs.run_id,
-            "parent_run_id": rs.parent_run_id,
-        },
+        "message": "Execute job via RestAPI.",
+        "result": {"run_id": rs.run_id, "parent_run_id": rs.parent_run_id},
         "job": job.model_dump(
             by_alias=True,
-            exclude_none=True,
+            exclude_none=False,
             exclude_unset=True,
             exclude_defaults=True,
         ),

{ddeutil_workflow-0.0.54 → ddeutil_workflow-0.0.56}/src/ddeutil/workflow/api/routes/schedules.py

@@ -17,7 +17,6 @@ from ...logs import get_logger
 from ...scheduler import Schedule
 
 logger = get_logger("uvicorn.error")
-
 schedule_route = APIRouter(
     prefix="/schedules",
     tags=["schedules"],
@@ -108,7 +107,6 @@ async def add_deploy_scheduler(request: Request, name: str):
         schedule.tasks(
             start_date_waiting,
             queue=request.state.workflow_queue,
-            extras={},
         ),
     )
     return {

{ddeutil_workflow-0.0.54 → ddeutil_workflow-0.0.56}/src/ddeutil/workflow/api/routes/workflows.py

@@ -21,7 +21,6 @@ from ...result import Result
 from ...workflow import Workflow
 
 logger = get_logger("uvicorn.error")
-
 workflow_route = APIRouter(
     prefix="/workflows",
     tags=["workflows"],
@@ -55,7 +54,7 @@ async def get_workflow_by_name(name: str) -> DictData:
         ) from None
     return workflow.model_dump(
         by_alias=True,
-        exclude_none=True,
+        exclude_none=False,
         exclude_unset=True,
         exclude_defaults=True,
     )
@@ -98,7 +97,7 @@ async def get_workflow_audits(name: str):
             "audits": [
                 audit.model_dump(
                     by_alias=True,
-                    exclude_none=True,
+                    exclude_none=False,
                     exclude_unset=True,
                     exclude_defaults=True,
                 )
@@ -132,7 +131,7 @@ async def get_workflow_release_audit(name: str, release: str):
         "message": f"Getting workflow {name!r} audit in release {release}",
         "audit": audit.model_dump(
             by_alias=True,
-            exclude_none=True,
+            exclude_none=False,
             exclude_unset=True,
             exclude_defaults=True,
         ),

{ddeutil_workflow-0.0.54 → ddeutil_workflow-0.0.56}/src/ddeutil/workflow/conf.py

@@ -7,24 +7,26 @@ from __future__ import annotations
 
 import json
 import os
+from abc import ABC, abstractmethod
 from collections.abc import Iterator
 from datetime import timedelta
 from functools import cached_property
+from inspect import isclass
 from pathlib import Path
-from typing import Final, Optional, TypeVar
+from typing import Final, Optional, Protocol, TypeVar, Union
 from zoneinfo import ZoneInfo
 
 from ddeutil.core import str2bool
 from ddeutil.io import YamlFlResolve
 from ddeutil.io.paths import glob_files, is_ignored, read_ignore
 
-from .__types import DictData, TupleStr
+from .__types import DictData
 
 T = TypeVar("T")
 PREFIX: Final[str] = "WORKFLOW"
 
 
-def env(var: str, default: str | None = None) -> str | None:  # pragma: no cov
+def env(var: str, default: str | None = None) -> str | None:
     """Get environment variable with uppercase and adding prefix string.
 
     :param var: (str) A env variable name.
@@ -35,17 +37,6 @@ def env(var: str, default: str | None = None) -> str | None:  # pragma: no cov
     return os.getenv(f"{PREFIX}_{var.upper().replace(' ', '_')}", default)
 
 
-__all__: TupleStr = (
-    "api_config",
-    "env",
-    "Config",
-    "SimLoad",
-    "Loader",
-    "config",
-    "dynamic",
-)
-
-
 class Config:  # pragma: no cov
     """Config object for keeping core configurations on the current session
     without changing when if the application still running.
@@ -188,7 +179,7 @@ class Config:  # pragma: no cov
             return timedelta(**json.loads(stop_boundary_delta_str))
         except Exception as err:
             raise ValueError(
-                "Config ``WORKFLOW_APP_STOP_BOUNDARY_DELTA`` can not parsing to"
+                "Config `WORKFLOW_APP_STOP_BOUNDARY_DELTA` can not parsing to"
                 f"timedelta with {stop_boundary_delta_str}."
             ) from err
 
@@ -209,110 +200,194 @@ class APIConfig:
         return str2bool(env("API_ENABLE_ROUTE_SCHEDULE", "true"))
 
 
-class SimLoad:
-    """Simple Load Object that will search config data by given some identity
-    value like name of workflow or on.
+class BaseLoad(ABC):
+
+    @classmethod
+    @abstractmethod
+    def find(cls, name: str, *args, **kwargs) -> DictData: ...
+
+    @classmethod
+    @abstractmethod
+    def finds(
+        cls, obj: object, *args, **kwargs
+    ) -> Iterator[tuple[str, DictData]]: ...
 
-    :param name: A name of config data that will read by Yaml Loader object.
-    :param conf_path: A config path object.
-    :param externals: An external parameters
+
+class FileLoad(BaseLoad):
+    """Base Load object that use to search config data by given some identity
+    value like name of `Workflow` or `On` templates.
+
+    :param name: (str) A name of key of config data that read with YAML
+        Environment object.
+    :param path: (Path) A config path object.
+    :param externals: (DictData) An external config data that want to add to
+        loaded config data.
+    :param extras: (DictDdata) An extra parameters that use to override core
+        config values.
+
+    :raise ValueError: If the data does not find on the config path with the
+        name parameter.
 
     Noted:
-        The config data should have ``type`` key for modeling validation that
+        The config data should have `type` key for modeling validation that
     make this loader know what is config should to do pass to.
 
         ... <identity-key>:
         ...     type: <importable-object>
         ...     <key-data-1>: <value-data-1>
         ...     <key-data-2>: <value-data-2>
+
+        This object support multiple config paths if you pass the `conf_paths`
+    key to the `extras` parameter.
     """
 
     def __init__(
         self,
         name: str,
-        conf_path: Path,
+        *,
+        path: Optional[Union[str, Path]] = None,
         externals: DictData | None = None,
+        extras: DictData | None = None,
     ) -> None:
-        self.conf_path: Path = conf_path
+        self.path: Path = Path(dynamic("conf_path", f=path, extras=extras))
         self.externals: DictData = externals or {}
-
-        self.data: DictData = {}
-        for file in glob_files(conf_path):
-
-            if self.is_ignore(file, conf_path):
-                continue
-
-            if data := self.filter_yaml(file, name=name):
-                self.data = data
+        self.extras: DictData = extras or {}
+        self.data: DictData = self.find(
+            name,
+            path=path,
+            paths=self.extras.get("conf_paths"),
+            extras=extras,
+        )
 
         # VALIDATE: check the data that reading should not empty.
         if not self.data:
             raise ValueError(
-                f"Config {name!r} does not found on conf path: "
-                f"{self.conf_path}."
+                f"Config {name!r} does not found on the conf path: {self.path}."
             )
 
         self.data.update(self.externals)
 
+    @classmethod
+    def find(
+        cls,
+        name: str,
+        *,
+        path: Optional[Path] = None,
+        paths: Optional[list[Path]] = None,
+        extras: Optional[DictData] = None,
+    ) -> DictData:
+        """Find data with specific key and return the latest modify date data if
+        this key exists multiple files.
+
+        :param name: (str) A name of data that want to find.
+        :param path: (Path) A config path object.
+        :param paths: (list[Path]) A list of config path object.
+        :param extras: (DictData)  An extra parameter that use to override core
+            config values.
+
+        :rtype: DictData
+        """
+        path: Path = dynamic("conf_path", f=path, extras=extras)
+        if not paths:
+            paths: list[Path] = [path]
+        elif not isinstance(paths, list):
+            raise TypeError(
+                f"Multi-config paths does not support for type: {type(paths)}"
+            )
+        else:
+            paths.append(path)
+
+        all_data: list[tuple[float, DictData]] = []
+        for path in paths:
+            for file in glob_files(path):
+
+                if cls.is_ignore(file, path):
+                    continue
+
+                if data := cls.filter_yaml(file, name=name):
+                    all_data.append((file.lstat().st_mtime, data))
+
+        return {} if not all_data else max(all_data, key=lambda x: x[0])[1]
+
     @classmethod
     def finds(
         cls,
         obj: object,
-        conf_path: Path,
         *,
-        included: list[str] | None = None,
+        path: Optional[Path] = None,
+        paths: Optional[list[Path]] = None,
         excluded: list[str] | None = None,
+        extras: Optional[DictData] = None,
     ) -> Iterator[tuple[str, DictData]]:
         """Find all data that match with object type in config path. This class
         method can use include and exclude list of identity name for filter and
         adds-on.
 
         :param obj: An object that want to validate matching before return.
-        :param conf_path: A config object.
-        :param included: An excluded list of data key that want to reject this
-            data if any key exist.
+        :param path: A config path object.
+        :param paths: (list[Path]) A list of config path object.
         :param excluded: An included list of data key that want to filter from
             data.
+        :param extras: (DictData) An extra parameter that use to override core
+            config values.
 
         :rtype: Iterator[tuple[str, DictData]]
         """
-        exclude: list[str] = excluded or []
-        for file in glob_files(conf_path):
+        excluded: list[str] = excluded or []
+        path: Path = dynamic("conf_path", f=path, extras=extras)
+        if not paths:
+            paths: list[Path] = [path]
+        else:
+            paths.append(path)
+
+        all_data: dict[str, list[tuple[float, DictData]]] = {}
+        for path in paths:
+            for file in glob_files(path):
+
+                if cls.is_ignore(file, path):
+                    continue
 
-            if cls.is_ignore(file, conf_path):
-                continue
+                for key, data in cls.filter_yaml(file).items():
 
-            for key, data in cls.filter_yaml(file).items():
+                    if key in excluded:
+                        continue
 
-                if key in exclude:
-                    continue
+                    if (
+                        data.get("type", "")
+                        == (obj if isclass(obj) else obj.__class__).__name__
+                    ):
+                        marking: tuple[float, DictData] = (
+                            file.lstat().st_mtime,
+                            data,
+                        )
+                        if key in all_data:
+                            all_data[key].append(marking)
+                        else:
+                            all_data[key] = [marking]
 
-                if data.get("type", "") == obj.__name__:
-                    yield key, (
-                        {k: data[k] for k in data if k in included}
-                        if included
-                        else data
-                    )
+        for key in all_data:
+            yield key, max(all_data[key], key=lambda x: x[0])[1]
 
     @classmethod
     def is_ignore(
         cls,
         file: Path,
-        conf_path: Path,
+        path: Path,
         *,
         ignore_filename: Optional[str] = None,
     ) -> bool:
         """Check this file was ignored.
 
         :param file: (Path) A file path that want to check.
-        :param conf_path: (Path) A config path that want to read the config
+        :param path: (Path) A config path that want to read the config
             ignore file.
-        :param ignore_filename: (str) An ignore filename.
+        :param ignore_filename: (str) An ignore filename. Default is
+            `.confignore` filename.
 
         :rtype: bool
         """
         ignore_filename: str = ignore_filename or ".confignore"
-        return is_ignored(file, read_ignore(conf_path / ignore_filename))
+        return is_ignored(file, read_ignore(path / ignore_filename))
 
     @classmethod
     def filter_yaml(cls, file: Path, name: str | None = None) -> DictData:
@@ -369,44 +444,19 @@ def dynamic(
     return rsx if rsx is not None else rs
 
 
-class Loader(SimLoad):
-    """Loader Object that get the config `yaml` file from current path.
+class Loader(Protocol):  # pragma: no cov
+    type: str
+    path: Path
+    data: DictData
+    extras: DictData
+    externals: DictData
 
-    :param name: (str) A name of config data that will read by Yaml Loader object.
-    :param externals: (DictData) An external parameters
-    """
+    def __init__(self, *args, **kwargs) -> None: ...
 
     @classmethod
-    def finds(
-        cls,
-        obj: object,
-        *,
-        path: Path | None = None,
-        included: list[str] | None = None,
-        excluded: list[str] | None = None,
-        **kwargs,
-    ) -> Iterator[tuple[str, DictData]]:
-        """Override the find class method from the Simple Loader object.
+    def find(cls, name: str, *args, **kwargs) -> DictData: ...
 
-        :param obj: An object that want to validate matching before return.
-        :param path: (Path) A override config path.
-        :param included: An excluded list of data key that want to reject this
-            data if any key exist.
-        :param excluded: An included list of data key that want to filter from
-            data.
-
-        :rtype: Iterator[tuple[str, DictData]]
-        """
-        return super().finds(
-            obj=obj,
-            conf_path=(path or config.conf_path),
-            included=included,
-            excluded=excluded,
-        )
-
-    def __init__(self, name: str, externals: DictData) -> None:
-        super().__init__(
-            name,
-            conf_path=dynamic("conf_path", extras=externals),
-            externals=externals,
-        )
+    @classmethod
+    def finds(
+        cls, obj: object, *args, **kwargs
+    ) -> Iterator[tuple[str, DictData]]: ...