ddeutil-workflow 0.0.38__py3-none-any.whl → 0.0.40__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1 +1 @@
-__version__: str = "0.0.38"
+__version__: str = "0.0.40"

ddeutil/workflow/__cron.py

@@ -37,7 +37,7 @@ class CronYearLimit(Exception): ...
 def str2cron(value: str) -> str:  # pragma: no cov
     """Convert Special String with the @ prefix to Crontab value.
 
-    :param value: A string value that want to convert to cron value.
+    :param value: (str) A string value that want to convert to cron value.
     :rtype: str
 
     Table:
@@ -82,6 +82,9 @@ class Unit:
         )
 
 
+Units = tuple[Unit, ...]
+
+
 @dataclass
 class Options:
     """Options dataclass for config CronPart object."""
@@ -91,7 +94,7 @@ class Options:
     output_hashes: bool = False
 
 
-CRON_UNITS: tuple[Unit, ...] = (
+CRON_UNITS: Units = (
     Unit(
         name="minute",
         range=partial(range, 0, 60),
@@ -147,7 +150,7 @@ CRON_UNITS: tuple[Unit, ...] = (
     ),
 )
 
-CRON_UNITS_YEAR: tuple[Unit, ...] = CRON_UNITS + (
+CRON_UNITS_YEAR: Units = CRON_UNITS + (
     Unit(
         name="year",
         range=partial(range, 1990, 2101),
@@ -220,18 +223,21 @@ class CronPart:
         return ",".join(cron_range_strings) if cron_range_strings else "?"
 
     def __repr__(self) -> str:
+        """Override __repr__ method."""
         return (
             f"{self.__class__.__name__}"
             f"(unit={self.unit}, values={self.__str__()!r})"
         )
 
     def __lt__(self, other) -> bool:
+        """Override __lt__ method."""
         if isinstance(other, CronPart):
             return self.values < other.values
         elif isinstance(other, list):
             return self.values < other
 
     def __eq__(self, other) -> bool:
+        """Override __eq__ method."""
         if isinstance(other, CronPart):
             return self.values == other.values
         elif isinstance(other, list):
@@ -239,18 +245,26 @@ class CronPart:
 
     @property
     def min(self) -> int:
-        """Returns the smallest value in the range."""
+        """Returns the smallest value in the range.
+
+        :rtype: int
+        """
         return self.values[0]
 
     @property
     def max(self) -> int:
-        """Returns the largest value in the range."""
+        """Returns the largest value in the range.
+
+        :rtype: int
+        """
         return self.values[-1]
 
     @property
     def step(self) -> Optional[int]:
         """Returns the difference between first and second elements in the
         range.
+
+        :rtype: Optional[int]
         """
         if (
             len(self.values) > 2
@@ -260,15 +274,17 @@ class CronPart:
 
     @property
     def is_full(self) -> bool:
-        """Returns true if range has all the values of the unit."""
+        """Returns true if range has all the values of the unit.
+
+        :rtype: bool
+        """
         return len(self.values) == (self.unit.max - self.unit.min + 1)
 
     def from_str(self, value: str) -> tuple[int, ...]:
         """Parses a string as a range of positive integers. The string should
         include only `-` and `,` special strings.
 
-        :param value: A string value that want to parse
-        :type value: str
+        :param value: (str) A string value that want to parse
 
         TODO: support for `L`, `W`, and `#`
         ---
@@ -351,6 +367,8 @@ class CronPart:
             For example if value == 'JAN,AUG' it will replace to '1,8'.
 
         :param value: A string value that want to replace alternative to int.
+
+        :rtype: str
         """
         for i, alt in enumerate(self.unit.alt):
             if alt in value:
@@ -361,6 +379,7 @@ class CronPart:
         """Replaces all 7 with 0 as Sunday can be represented by both.
 
         :param values: list or iter of int that want to mode by 7
+
         :rtype: list[int]
         """
         if self.unit.name == "weekday":
@@ -388,7 +407,13 @@ class CronPart:
         return values
 
     def _parse_range(self, value: str) -> list[int]:
-        """Parses a range string."""
+        """Parses a range string from a cron-part.
+
+        :param value: (str) A cron-part string value that want to parse.
+
+        :rtype: list[int]
+        :return: A list of parse range.
+        """
         if value == "*":
             return list(self.unit.range())
         elif value.count("-") > 1:
@@ -410,7 +435,13 @@ class CronPart:
         values: list[int],
         step: int | None = None,
     ) -> list[int]:
-        """Applies an interval step to a collection of values."""
+        """Applies an interval step to a collection of values.
+
+        :param values:
+        :param step:
+
+        :rtype: list[int]
+        """
         if not step:
             return values
         elif (_step := int(step)) < 1:
@@ -427,7 +458,10 @@ class CronPart:
 
     @property
     def is_interval(self) -> bool:
-        """Returns true if the range can be represented as an interval."""
+        """Returns true if the range can be represented as an interval.
+
+        :rtype: bool
+        """
         if not (step := self.step):
             return False
         for idx, value in enumerate(self.values):
@@ -439,7 +473,10 @@ class CronPart:
 
     @property
     def is_full_interval(self) -> bool:
-        """Returns true if the range contains all the interval values."""
+        """Returns true if the range contains all the interval values.
+
+        :rtype: bool
+        """
         if step := self.step:
             return (
                 self.min == self.unit.min
@@ -482,8 +519,7 @@ class CronPart:
         """Formats weekday and month names as string when the relevant options
         are set.
 
-        :param value: a int value
-        :type value: int
+        :param value: (int) An int value that want to get from the unit.
 
         :rtype: int | str
         """
@@ -538,8 +574,8 @@ class CronJob:
             monitor-data-warehouse-schedule.html
     """
 
-    cron_length: int = 5
-    cron_units: tuple[Unit, ...] = CRON_UNITS
+    cron_length: ClassVar[int] = 5
+    cron_units: ClassVar[Units] = CRON_UNITS
 
     def __init__(
         self,
@@ -600,38 +636,64 @@ class CronJob:
 
     @property
     def parts_order(self) -> Iterator[CronPart]:
+        """Return iterator of CronPart instance.
+
+        :rtype: Iterator[CronPart]
+        """
         return reversed(self.parts[:3] + [self.parts[4], self.parts[3]])
 
     @property
     def minute(self) -> CronPart:
-        """Return part of minute."""
+        """Return part of minute with the CronPart instance.
+
+        :rtype: CronPart
+        """
         return self.parts[0]
 
     @property
     def hour(self) -> CronPart:
-        """Return part of hour."""
+        """Return part of hour with the CronPart instance.
+
+        :rtype: CronPart
+        """
         return self.parts[1]
 
     @property
     def day(self) -> CronPart:
-        """Return part of day."""
+        """Return part of day with the CronPart instance.
+
+        :rtype: CronPart
+        """
         return self.parts[2]
 
     @property
     def month(self) -> CronPart:
-        """Return part of month."""
+        """Return part of month with the CronPart instance.
+
+        :rtype: CronPart
+        """
         return self.parts[3]
 
     @property
     def dow(self) -> CronPart:
-        """Return part of day of month."""
+        """Return part of day of month with the CronPart instance.
+
+        :rtype: CronPart
+        """
         return self.parts[4]
 
     def to_list(self) -> list[list[int]]:
-        """Returns the cron schedule as a 2-dimensional list of integers."""
+        """Returns the cron schedule as a 2-dimensional list of integers.
+
+        :rtype: list[list[int]]
+        """
         return [part.values for part in self.parts]
 
     def check(self, date: datetime, mode: str) -> bool:
+        """Check the date value with the mode.
+
+        :rtype: bool
+        """
         assert mode in ("year", "month", "day", "hour", "minute")
         return getattr(date, mode) in getattr(self, mode).values
 
@@ -667,12 +729,14 @@ class CronJobYear(CronJob):
         (vi)    year (1990 - 2100)
     """
 
-    cron_length = 6
-    cron_units = CRON_UNITS_YEAR
+    cron_length: ClassVar[int] = 6
+    cron_units: ClassVar[Units] = CRON_UNITS_YEAR
 
     @property
     def year(self) -> CronPart:
-        """Return part of year."""
+        """Return part of year with the CronPart instance.
+
+        :rtype: CronPart"""
         return self.parts[5]
 
 

ddeutil/workflow/__types.py

@@ -61,8 +61,10 @@ class Re:
     #   Regular expression:
     #       - Version 1:
     #         \${{\s*(?P<caller>[a-zA-Z0-9_.\s'\"\[\]\(\)\-\{}]+?)\s*(?P<post_filters>(?:\|\s*(?:[a-zA-Z0-9_]{3,}[a-zA-Z0-9_.,-\\%\s'\"[\]()\{}]+)\s*)*)}}
+    #
     #       - Version 2: (2024-09-30):
     #         \${{\s*(?P<caller>(?P<caller_prefix>(?:[a-zA-Z_-]+\.)*)(?P<caller_last>[a-zA-Z0-9_\-.'\"(\)[\]{}]+))\s*(?P<post_filters>(?:\|\s*(?:[a-zA-Z0-9_]{3,}[a-zA-Z0-9_.,-\\%\s'\"[\]()\{}]+)\s*)*)}}
+    #
     #       - Version 3: (2024-10-05):
     #         \${{\s*(?P<caller>(?P<caller_prefix>(?:[a-zA-Z_-]+\??\.)*)(?P<caller_last>[a-zA-Z0-9_\-.'\"(\)[\]{}]+\??))\s*(?P<post_filters>(?:\|\s*(?:[a-zA-Z0-9_]{3,}[a-zA-Z0-9_.,-\\%\s'\"[\]()\{}]+)\s*)*)}}
     #

ddeutil/workflow/audit.py

@@ -20,7 +20,7 @@ from typing_extensions import Self
 
 from .__types import DictData, TupleStr
 from .conf import config
-from .logs import TraceLog, get_trace
+from .logs import TraceLog, get_dt_tznow, get_trace
 
 __all__: TupleStr = (
     "get_audit",
@@ -43,10 +43,12 @@ class BaseAudit(BaseModel, ABC):
         default_factory=dict,
         description="A context that receive from a workflow execution result.",
     )
-    parent_run_id: Optional[str] = Field(default=None)
-    run_id: str
-    update: datetime = Field(default_factory=datetime.now)
-    execution_time: float = Field(default=0)
+    parent_run_id: Optional[str] = Field(
+        default=None, description="A parent running ID."
+    )
+    run_id: str = Field(description="A running ID")
+    update: datetime = Field(default_factory=get_dt_tznow)
+    execution_time: float = Field(default=0, description="An execution time.")
 
     @model_validator(mode="after")
     def __model_action(self) -> Self:

ddeutil/workflow/caller.py

@@ -48,9 +48,10 @@ def tag(
     """Tag decorator function that set function attributes, ``tag`` and ``name``
     for making registries variable.
 
-    :param: name: A tag name for make different use-case of a function.
-    :param: alias: A alias function name that keeping in registries. If this
-        value does not supply, it will use original function name from __name__.
+    :param: name: (str) A tag name for make different use-case of a function.
+    :param: alias: (str) A alias function name that keeping in registries.
+        If this value does not supply, it will use original function name
+        from `__name__` argument.
 
     :rtype: Callable[P, TagFunc]
     """
@@ -78,7 +79,7 @@ Registry = dict[str, Callable[[], TagFunc]]
 def make_registry(submodule: str) -> dict[str, Registry]:
     """Return registries of all functions that able to called with task.
 
-    :param submodule: A module prefix that want to import registry.
+    :param submodule: (str) A module prefix that want to import registry.
 
     :rtype: dict[str, Registry]
     """
@@ -134,12 +135,7 @@ def extract_call(call: str) -> Callable[[], TagFunc]:
     """Extract Call function from string value to call partial function that
     does run it at runtime.
 
-    :raise NotImplementedError: When the searching call's function result does
-        not exist in the registry.
-    :raise NotImplementedError: When the searching call's tag result does not
-        exist in the registry with its function key.
-
-    :param call: A call value that able to match with Task regex.
+    :param call: (str) A call value that able to match with Task regex.
 
         The format of call value should contain 3 regular expression groups
     which match with the below config format:
@@ -152,6 +148,11 @@ def extract_call(call: str) -> Callable[[], TagFunc]:
         >>> extract_call("tasks/return-type-not-valid@raise")
         ...
 
+    :raise NotImplementedError: When the searching call's function result does
+        not exist in the registry.
+    :raise NotImplementedError: When the searching call's tag result does not
+        exist in the registry with its function key.
+
     :rtype: Callable[[], TagFunc]
     """
     if not (found := Re.RE_TASK_FMT.search(call)):

ddeutil/workflow/conf.py

@@ -16,6 +16,7 @@ from zoneinfo import ZoneInfo
 
 from ddeutil.core import str2bool
 from ddeutil.io import YamlFlResolve
+from ddeutil.io.paths import glob_files, is_ignored, read_ignore
 
 from .__types import DictData, TupleStr
 
@@ -26,10 +27,6 @@ def env(var: str, default: str | None = None) -> str | None:  # pragma: no cov
     return os.getenv(f"{PREFIX}_{var.upper().replace(' ', '_')}", default)
 
 
-def glob_files(path: Path) -> Iterator[Path]:  # pragma: no cov
-    yield from (file for file in path.rglob("*") if file.is_file())
-
-
 __all__: TupleStr = (
     "env",
     "get_logger",
@@ -37,7 +34,6 @@ __all__: TupleStr = (
     "SimLoad",
     "Loader",
     "config",
-    "glob_files",
 )
 
 
@@ -273,7 +269,7 @@ class SimLoad:
             if self.is_ignore(file, conf_path):
                 continue
 
-            if data := self.filter_suffix(file, name=name):
+            if data := self.filter_yaml(file, name=name):
                 self.data = data
 
         # VALIDATE: check the data that reading should not empty.
@@ -307,15 +303,15 @@ class SimLoad:
         exclude: list[str] = excluded or []
         for file in glob_files(conf_path):
 
-            for key, data in cls.filter_suffix(file).items():
+            if cls.is_ignore(file, conf_path):
+                continue
 
-                if cls.is_ignore(file, conf_path):
-                    continue
+            for key, data in cls.filter_yaml(file).items():
 
                 if key in exclude:
                     continue
 
-                if data["type"] == obj.__name__:
+                if data.get("type", "") == obj.__name__:
                     yield key, (
                         {k: data[k] for k in data if k in included}
                         if included
@@ -324,24 +320,13 @@ class SimLoad:
 
     @classmethod
     def is_ignore(cls, file: Path, conf_path: Path) -> bool:
-        ignore_file: Path = conf_path / ".confignore"
-        ignore: list[str] = []
-        if ignore_file.exists():
-            ignore = ignore_file.read_text(encoding="utf-8").splitlines()
-
-        if any(
-            (file.match(f"**/{pattern}/*") or file.match(f"**/{pattern}*"))
-            for pattern in ignore
-        ):
-            return True
-        return False
+        return is_ignored(file, read_ignore(conf_path / ".confignore"))
 
     @classmethod
-    def filter_suffix(cls, file: Path, name: str | None = None) -> DictData:
+    def filter_yaml(cls, file: Path, name: str | None = None) -> DictData:
         if any(file.suffix.endswith(s) for s in (".yml", ".yaml")):
             values: DictData = YamlFlResolve(file).read()
             return values.get(name, {}) if name else values
-
         return {}
 
     @cached_property

ddeutil/workflow/context.py

@@ -18,6 +18,7 @@ class ErrorContext(BaseModel):  # pragma: no cov
 class OutputContext(BaseModel):  # pragma: no cov
     outputs: DictData = Field(default_factory=dict)
     errors: Optional[ErrorContext] = Field(default=None)
+    skipped: bool = Field(default=False)
 
     def is_exception(self) -> bool:
         return self.errors is not None
@@ -57,3 +58,4 @@ class JobContext(BaseModel):  # pragma: no cov
     params: DictData = Field(description="A parameterize value")
     jobs: dict[str, StrategyMatrixContext]
     errors: Optional[ErrorContext] = Field(default=None)
+    skipped: bool = Field(default=False)