ddeutil-workflow 0.0.39__tar.gz → 0.0.41__tar.gz

{ddeutil_workflow-0.0.39/src/ddeutil_workflow.egg-info → ddeutil_workflow-0.0.41}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddeutil-workflow
-Version: 0.0.39
+Version: 0.0.41
 Summary: Lightweight workflow orchestration
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT
@@ -23,10 +23,16 @@ Requires-Python: >=3.9.13
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: ddeutil>=0.4.6
-Requires-Dist: ddeutil-io[toml,yaml]>=0.2.3
-Requires-Dist: pydantic==2.10.6
-Requires-Dist: python-dotenv==1.0.1
+Requires-Dist: ddeutil-io[toml,yaml]>=0.2.10
+Requires-Dist: pydantic==2.11.1
+Requires-Dist: python-dotenv==1.1.0
 Requires-Dist: schedule<2.0.0,==1.2.2
+Provides-Extra: all
+Requires-Dist: fastapi<1.0.0,>=0.115.0; extra == "all"
+Requires-Dist: httpx; extra == "all"
+Requires-Dist: ujson; extra == "all"
+Requires-Dist: aiofiles; extra == "all"
+Requires-Dist: aiohttp; extra == "all"
 Provides-Extra: api
 Requires-Dist: fastapi<1.0.0,>=0.115.0; extra == "api"
 Requires-Dist: httpx; extra == "api"

{ddeutil_workflow-0.0.39 → ddeutil_workflow-0.0.41}/pyproject.toml

@@ -27,14 +27,21 @@ classifiers = [
 requires-python = ">=3.9.13"
 dependencies = [
     "ddeutil>=0.4.6",
-    "ddeutil-io[yaml,toml]>=0.2.3",
-    "pydantic==2.10.6",
-    "python-dotenv==1.0.1",
+    "ddeutil-io[yaml,toml]>=0.2.10",
+    "pydantic==2.11.1",
+    "python-dotenv==1.1.0",
     "schedule==1.2.2,<2.0.0",
 ]
 dynamic = ["version"]
 
 [project.optional-dependencies]
+all = [
+    "fastapi>=0.115.0,<1.0.0",
+    "httpx",
+    "ujson",
+    "aiofiles",
+    "aiohttp",
+]
 api = [
     "fastapi>=0.115.0,<1.0.0",
     "httpx",

ddeutil_workflow-0.0.41/src/ddeutil/workflow/__about__.py

@@ -0,0 +1 @@
+__version__: str = "0.0.41"

{ddeutil_workflow-0.0.39 → ddeutil_workflow-0.0.41}/src/ddeutil/workflow/__cron.py

@@ -37,7 +37,7 @@ class CronYearLimit(Exception): ...
 def str2cron(value: str) -> str:  # pragma: no cov
     """Convert Special String with the @ prefix to Crontab value.
 
-    :param value: A string value that want to convert to cron value.
+    :param value: (str) A string value that want to convert to cron value.
     :rtype: str
 
     Table:
@@ -82,6 +82,9 @@ class Unit:
         )
 
 
+Units = tuple[Unit, ...]
+
+
 @dataclass
 class Options:
     """Options dataclass for config CronPart object."""
@@ -91,7 +94,7 @@ class Options:
     output_hashes: bool = False
 
 
-CRON_UNITS: tuple[Unit, ...] = (
+CRON_UNITS: Units = (
     Unit(
         name="minute",
         range=partial(range, 0, 60),
@@ -147,7 +150,7 @@ CRON_UNITS: tuple[Unit, ...] = (
     ),
 )
 
-CRON_UNITS_YEAR: tuple[Unit, ...] = CRON_UNITS + (
+CRON_UNITS_YEAR: Units = CRON_UNITS + (
     Unit(
         name="year",
         range=partial(range, 1990, 2101),
@@ -220,18 +223,21 @@ class CronPart:
         return ",".join(cron_range_strings) if cron_range_strings else "?"
 
     def __repr__(self) -> str:
+        """Override __repr__ method."""
         return (
             f"{self.__class__.__name__}"
             f"(unit={self.unit}, values={self.__str__()!r})"
         )
 
     def __lt__(self, other) -> bool:
+        """Override __lt__ method."""
         if isinstance(other, CronPart):
             return self.values < other.values
         elif isinstance(other, list):
             return self.values < other
 
     def __eq__(self, other) -> bool:
+        """Override __eq__ method."""
         if isinstance(other, CronPart):
             return self.values == other.values
         elif isinstance(other, list):
@@ -239,18 +245,26 @@ class CronPart:
 
     @property
     def min(self) -> int:
-        """Returns the smallest value in the range."""
+        """Returns the smallest value in the range.
+
+        :rtype: int
+        """
         return self.values[0]
 
     @property
     def max(self) -> int:
-        """Returns the largest value in the range."""
+        """Returns the largest value in the range.
+
+        :rtype: int
+        """
         return self.values[-1]
 
     @property
     def step(self) -> Optional[int]:
         """Returns the difference between first and second elements in the
         range.
+
+        :rtype: Optional[int]
         """
         if (
             len(self.values) > 2
@@ -260,15 +274,17 @@ class CronPart:
 
     @property
     def is_full(self) -> bool:
-        """Returns true if range has all the values of the unit."""
+        """Returns true if range has all the values of the unit.
+
+        :rtype: bool
+        """
         return len(self.values) == (self.unit.max - self.unit.min + 1)
 
     def from_str(self, value: str) -> tuple[int, ...]:
         """Parses a string as a range of positive integers. The string should
         include only `-` and `,` special strings.
 
-        :param value: A string value that want to parse
-        :type value: str
+        :param value: (str) A string value that want to parse
 
         TODO: support for `L`, `W`, and `#`
         ---
@@ -351,6 +367,8 @@ class CronPart:
             For example if value == 'JAN,AUG' it will replace to '1,8'.
 
         :param value: A string value that want to replace alternative to int.
+
+        :rtype: str
         """
         for i, alt in enumerate(self.unit.alt):
             if alt in value:
@@ -361,6 +379,7 @@ class CronPart:
         """Replaces all 7 with 0 as Sunday can be represented by both.
 
         :param values: list or iter of int that want to mode by 7
+
         :rtype: list[int]
         """
         if self.unit.name == "weekday":
@@ -388,7 +407,13 @@ class CronPart:
         return values
 
     def _parse_range(self, value: str) -> list[int]:
-        """Parses a range string."""
+        """Parses a range string from a cron-part.
+
+        :param value: (str) A cron-part string value that want to parse.
+
+        :rtype: list[int]
+        :return: A list of parse range.
+        """
         if value == "*":
             return list(self.unit.range())
         elif value.count("-") > 1:
@@ -410,7 +435,13 @@ class CronPart:
         values: list[int],
         step: int | None = None,
     ) -> list[int]:
-        """Applies an interval step to a collection of values."""
+        """Applies an interval step to a collection of values.
+
+        :param values:
+        :param step:
+
+        :rtype: list[int]
+        """
         if not step:
             return values
         elif (_step := int(step)) < 1:
@@ -427,7 +458,10 @@ class CronPart:
 
     @property
     def is_interval(self) -> bool:
-        """Returns true if the range can be represented as an interval."""
+        """Returns true if the range can be represented as an interval.
+
+        :rtype: bool
+        """
         if not (step := self.step):
             return False
         for idx, value in enumerate(self.values):
@@ -439,7 +473,10 @@ class CronPart:
 
     @property
     def is_full_interval(self) -> bool:
-        """Returns true if the range contains all the interval values."""
+        """Returns true if the range contains all the interval values.
+
+        :rtype: bool
+        """
         if step := self.step:
             return (
                 self.min == self.unit.min
@@ -482,8 +519,7 @@ class CronPart:
         """Formats weekday and month names as string when the relevant options
         are set.
 
-        :param value: a int value
-        :type value: int
+        :param value: (int) An int value that want to get from the unit.
 
         :rtype: int | str
         """
@@ -538,8 +574,8 @@ class CronJob:
             monitor-data-warehouse-schedule.html
     """
 
-    cron_length: int = 5
-    cron_units: tuple[Unit, ...] = CRON_UNITS
+    cron_length: ClassVar[int] = 5
+    cron_units: ClassVar[Units] = CRON_UNITS
 
     def __init__(
         self,
@@ -600,38 +636,64 @@ class CronJob:
 
     @property
     def parts_order(self) -> Iterator[CronPart]:
+        """Return iterator of CronPart instance.
+
+        :rtype: Iterator[CronPart]
+        """
         return reversed(self.parts[:3] + [self.parts[4], self.parts[3]])
 
     @property
     def minute(self) -> CronPart:
-        """Return part of minute."""
+        """Return part of minute with the CronPart instance.
+
+        :rtype: CronPart
+        """
         return self.parts[0]
 
     @property
     def hour(self) -> CronPart:
-        """Return part of hour."""
+        """Return part of hour with the CronPart instance.
+
+        :rtype: CronPart
+        """
         return self.parts[1]
 
     @property
     def day(self) -> CronPart:
-        """Return part of day."""
+        """Return part of day with the CronPart instance.
+
+        :rtype: CronPart
+        """
         return self.parts[2]
 
     @property
     def month(self) -> CronPart:
-        """Return part of month."""
+        """Return part of month with the CronPart instance.
+
+        :rtype: CronPart
+        """
         return self.parts[3]
 
     @property
     def dow(self) -> CronPart:
-        """Return part of day of month."""
+        """Return part of day of month with the CronPart instance.
+
+        :rtype: CronPart
+        """
         return self.parts[4]
 
     def to_list(self) -> list[list[int]]:
-        """Returns the cron schedule as a 2-dimensional list of integers."""
+        """Returns the cron schedule as a 2-dimensional list of integers.
+
+        :rtype: list[list[int]]
+        """
         return [part.values for part in self.parts]
 
     def check(self, date: datetime, mode: str) -> bool:
+        """Check the date value with the mode.
+
+        :rtype: bool
+        """
         assert mode in ("year", "month", "day", "hour", "minute")
         return getattr(date, mode) in getattr(self, mode).values
 
@@ -667,12 +729,14 @@ class CronJobYear(CronJob):
         (vi)    year (1990 - 2100)
     """
 
-    cron_length = 6
-    cron_units = CRON_UNITS_YEAR
+    cron_length: ClassVar[int] = 6
+    cron_units: ClassVar[Units] = CRON_UNITS_YEAR
 
     @property
     def year(self) -> CronPart:
-        """Return part of year."""
+        """Return part of year with the CronPart instance.
+
+        :rtype: CronPart"""
         return self.parts[5]
 
 

{ddeutil_workflow-0.0.39 → ddeutil_workflow-0.0.41}/src/ddeutil/workflow/__init__.py

@@ -5,17 +5,6 @@
 # ------------------------------------------------------------------------------
 from .__cron import CronJob, CronRunner
 from .__types import Re
-from .audit import (
-    Audit,
-    get_audit,
-)
-from .caller import (
-    ReturnTagFunc,
-    TagFunc,
-    extract_call,
-    make_registry,
-    tag,
-)
 from .conf import (
     Config,
     Loader,
@@ -43,8 +32,10 @@ from .job import (
     local_execute_strategy,
 )
 from .logs import (
+    Audit,
     TraceData,
     TraceLog,
+    get_audit,
     get_dt_tznow,
     get_trace,
 )
@@ -58,7 +49,24 @@ from .params import (
 from .result import (
     Result,
     Status,
-    default_gen_id,
+)
+from .reusables import (
+    FILTERS,
+    FilterFunc,
+    FilterRegistry,
+    ReturnTagFunc,
+    TagFunc,
+    custom_filter,
+    extract_call,
+    get_args_const,
+    has_template,
+    make_filter_registry,
+    make_registry,
+    map_post_filter,
+    not_in_template,
+    param2template,
+    str2template,
+    tag,
 )
 from .scheduler import (
     Schedule,
@@ -77,22 +85,10 @@ from .stages import (
     Stage,
     TriggerStage,
 )
-from .templates import (
-    FILTERS,
-    FilterFunc,
-    FilterRegistry,
-    custom_filter,
-    get_args_const,
-    has_template,
-    make_filter_registry,
-    map_post_filter,
-    not_in_template,
-    param2template,
-    str2template,
-)
 from .utils import (
     batch,
     cross_product,
+    default_gen_id,
     delay,
     filter_func,
     gen_id,

{ddeutil_workflow-0.0.39 → ddeutil_workflow-0.0.41}/src/ddeutil/workflow/api/api.py

@@ -109,7 +109,7 @@ if config.enable_route_workflow:
 
 # NOTE: Enable the schedules route.
 if config.enable_route_schedule:
-    from ..audit import get_audit
+    from ..logs import get_audit
     from ..scheduler import schedule_task
     from .routes import schedule
 

{ddeutil_workflow-0.0.39 → ddeutil_workflow-0.0.41}/src/ddeutil/workflow/api/routes/logs.py

@@ -10,8 +10,7 @@ from fastapi import APIRouter, Path, Query
 from fastapi import status as st
 from fastapi.responses import UJSONResponse
 
-from ...audit import get_audit
-from ...logs import get_trace_obj
+from ...logs import get_audit, get_trace_obj
 
 log_route = APIRouter(
     prefix="/logs",

{ddeutil_workflow-0.0.39 → ddeutil_workflow-0.0.41}/src/ddeutil/workflow/api/routes/workflows.py

@@ -15,8 +15,8 @@ from fastapi.responses import UJSONResponse
 from pydantic import BaseModel
 
 from ...__types import DictData
-from ...audit import Audit, get_audit
 from ...conf import Loader, get_logger
+from ...logs import Audit, get_audit
 from ...result import Result
 from ...workflow import Workflow
 

{ddeutil_workflow-0.0.39 → ddeutil_workflow-0.0.41}/src/ddeutil/workflow/conf.py

@@ -16,6 +16,7 @@ from zoneinfo import ZoneInfo
 
 from ddeutil.core import str2bool
 from ddeutil.io import YamlFlResolve
+from ddeutil.io.paths import glob_files, is_ignored, read_ignore
 
 from .__types import DictData, TupleStr
 
@@ -23,13 +24,10 @@ PREFIX: str = "WORKFLOW"
 
 
 def env(var: str, default: str | None = None) -> str | None:  # pragma: no cov
+    """Get environment variable with uppercase and adding prefix string."""
     return os.getenv(f"{PREFIX}_{var.upper().replace(' ', '_')}", default)
 
 
-def glob_files(path: Path) -> Iterator[Path]:  # pragma: no cov
-    yield from (file for file in path.rglob("*") if file.is_file())
-
-
 __all__: TupleStr = (
     "env",
     "get_logger",
@@ -37,7 +35,6 @@ __all__: TupleStr = (
     "SimLoad",
     "Loader",
     "config",
-    "glob_files",
 )
 
 
@@ -98,11 +95,19 @@ class Config(BaseConfig):  # pragma: no cov
     # NOTE: Register
     @property
     def regis_call(self) -> list[str]:
+        """Register Caller module importer str.
+
+        :rtype: list[str]
+        """
         regis_call_str: str = env("CORE_REGISTRY", ".")
         return [r.strip() for r in regis_call_str.split(",")]
 
     @property
     def regis_filter(self) -> list[str]:
+        """Register Filter module.
+
+        :rtype: list[str]
+        """
         regis_filter_str: str = env(
             "CORE_REGISTRY_FILTER", "ddeutil.workflow.templates"
         )
@@ -273,7 +278,7 @@ class SimLoad:
             if self.is_ignore(file, conf_path):
                 continue
 
-            if data := self.filter_suffix(file, name=name):
+            if data := self.filter_yaml(file, name=name):
                 self.data = data
 
         # VALIDATE: check the data that reading should not empty.
@@ -307,15 +312,15 @@ class SimLoad:
         exclude: list[str] = excluded or []
         for file in glob_files(conf_path):
 
-            for key, data in cls.filter_suffix(file).items():
+            if cls.is_ignore(file, conf_path):
+                continue
 
-                if cls.is_ignore(file, conf_path):
-                    continue
+            for key, data in cls.filter_yaml(file).items():
 
                 if key in exclude:
                     continue
 
-                if data["type"] == obj.__name__:
+                if data.get("type", "") == obj.__name__:
                     yield key, (
                         {k: data[k] for k in data if k in included}
                         if included
@@ -324,24 +329,14 @@ class SimLoad:
 
     @classmethod
     def is_ignore(cls, file: Path, conf_path: Path) -> bool:
-        ignore_file: Path = conf_path / ".confignore"
-        ignore: list[str] = []
-        if ignore_file.exists():
-            ignore = ignore_file.read_text(encoding="utf-8").splitlines()
-
-        if any(
-            (file.match(f"**/{pattern}/*") or file.match(f"**/{pattern}*"))
-            for pattern in ignore
-        ):
-            return True
-        return False
+        return is_ignored(file, read_ignore(conf_path / ".confignore"))
 
     @classmethod
-    def filter_suffix(cls, file: Path, name: str | None = None) -> DictData:
+    def filter_yaml(cls, file: Path, name: str | None = None) -> DictData:
         if any(file.suffix.endswith(s) for s in (".yml", ".yaml")):
             values: DictData = YamlFlResolve(file).read()
-            return values.get(name, {}) if name else values
-
+            if values is not None:
+                return values.get(name, {}) if name else values
         return {}
 
     @cached_property

{ddeutil_workflow-0.0.39 → ddeutil_workflow-0.0.41}/src/ddeutil/workflow/job.py

@@ -39,8 +39,8 @@ from .exceptions import (
     UtilException,
 )
 from .result import Result, Status
+from .reusables import has_template, param2template
 from .stages import Stage
-from .templates import has_template, param2template
 from .utils import cross_product, filter_func, gen_id
 
 MatrixFilter = list[dict[str, Union[str, int]]]
@@ -344,6 +344,10 @@ class Job(BaseModel):
         default_factory=Strategy,
         description="A strategy matrix that want to generate.",
     )
+    extras: DictData = Field(
+        default_factory=dict,
+        description="An extra override values.",
+    )
 
     @field_validator("desc", mode="after")
     def ___prepare_desc__(cls, value: str) -> str:
@@ -394,6 +398,8 @@ class Job(BaseModel):
         """
         for stage in self.stages:
             if stage_id == (stage.id or ""):
+                if self.extras:
+                    stage.extras = self.extras
                 return stage
         raise ValueError(f"Stage ID {stage_id} does not exists")
 
@@ -596,7 +602,7 @@ def local_execute_strategy(
     *,
     result: Result | None = None,
     event: Event | None = None,
-    raise_error: bool = False,
+    raise_error: bool | None = None,
 ) -> Result:
     """Local job strategy execution with passing dynamic parameters from the
     workflow execution to strategy matrix.
@@ -694,12 +700,16 @@ def local_execute_strategy(
                     params=context,
                     run_id=result.run_id,
                     parent_run_id=result.parent_run_id,
+                    event=event,
                 ).context,
                 to=context,
             )
         except (StageException, UtilException) as err:
             result.trace.error(f"[JOB]: {err.__class__.__name__}: {err}")
-            if raise_error or config.job_raise_error:
+            do_raise: bool = (
+                config.job_raise_error if raise_error is None else raise_error
+            )
+            if do_raise:
                 raise JobException(
                     f"Stage execution error: {err.__class__.__name__}: "
                     f"{err}"
@@ -735,7 +745,7 @@ def local_execute(
     parent_run_id: str | None = None,
     result: Result | None = None,
     event: Event | None = None,
-    raise_error: bool = False,
+    raise_error: bool | None = None,
 ) -> Result:
     """Local job execution with passing dynamic parameters from the workflow
     execution. It will generate matrix values at the first step and run
@@ -865,3 +875,44 @@ def local_execute(
                 context.update({"errors": err.to_dict()})
 
     return result.catch(status=status, context=context)
+
+
+def self_hosted_execute(
+    job: Job,
+    params: DictData,
+    *,
+    run_id: str | None = None,
+    parent_run_id: str | None = None,
+    result: Result | None = None,
+    event: Event | None = None,
+    raise_error: bool | None = None,
+) -> Result:  # pragma: no cov
+    result: Result = Result.construct_with_rs_or_id(
+        result,
+        run_id=run_id,
+        parent_run_id=parent_run_id,
+        id_logic=(job.id or "not-set"),
+    )
+
+    if event and event.is_set():
+        return result.catch(status=Status.FAILED)
+
+    import requests
+
+    resp = requests.post(
+        job.runs_on.args.host,
+        data={"job": job.model_dump(), "params": params},
+    )
+
+    if resp.status_code != 200:
+        do_raise: bool = (
+            config.job_raise_error if raise_error is None else raise_error
+        )
+        if do_raise:
+            raise JobException(
+                f"Job execution error from request to self-hosted: "
+                f"{job.runs_on.args.host!r}"
+            )
+
+        return result.catch(status=Status.FAILED)
+    return result.catch(status=Status.SUCCESS)