ddeutil-workflow 0.0.18__tar.gz → 0.0.19__tar.gz

{ddeutil_workflow-0.0.18/src/ddeutil_workflow.egg-info → ddeutil_workflow-0.0.19}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ddeutil-workflow
-Version: 0.0.18
+Version: 0.0.19
 Summary: Lightweight workflow orchestration with less dependencies
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT
@@ -46,7 +46,7 @@ for easy to make a simple metadata driven for data workflow orchestration.
 It can to use for data operator by a `.yaml` template.
 
 > [!WARNING]
-> This package provide only orchestration workload task. That mean you should not
+> This package provide only orchestration workload. That mean you should not
 > use the workflow stage to process any large volume data which use lot of compute
 > resource. :cold_sweat:
 
@@ -58,10 +58,10 @@ configuration. It called **Metadata Driven Data Workflow**.
 
 **:pushpin: <u>Rules of This Workflow engine</u>**:
 
-1. Minimum frequency unit of scheduling is **1 minute** :warning:
+1. The Minimum frequency unit of scheduling is **1 minute** :warning:
 2. Can not re-run only failed stage and its pending downstream :rotating_light:
 3. All parallel tasks inside workflow engine use Multi-Threading
-   (Because Python 3.13 unlock GIL :unlock:)
+   (Python 3.13 unlock GIL :unlock:)
 
 > [!NOTE]
 > _Disclaimer_: I inspire the dynamic statement from the [**GitHub Action**](https://github.com/features/actions)
@@ -183,7 +183,7 @@ application. If any configuration values do not set yet, it will use default val
 and do not raise any error to you.
 
 | Environment                             | Component | Default                          | Description                                                                                                        | Remark |
-|:----------------------------------------|-----------|----------------------------------|--------------------------------------------------------------------------------------------------------------------|--------|
+|:----------------------------------------|:----------|:---------------------------------|--------------------------------------------------------------------------------------------------------------------|--------|
 | `WORKFLOW_ROOT_PATH`                    | Core      | .                                | The root path of the workflow application.                                                                         |        |
 | `WORKFLOW_CORE_REGISTRY`                | Core      | src.ddeutil.workflow,tests.utils | List of importable string for the hook stage.                                                                      |        |
 | `WORKFLOW_CORE_REGISTRY_FILTER`         | Core      | ddeutil.workflow.utils           | List of importable string for the filter template.                                                                 |        |

{ddeutil_workflow-0.0.18 → ddeutil_workflow-0.0.19}/README.md

@@ -13,7 +13,7 @@ for easy to make a simple metadata driven for data workflow orchestration.
 It can to use for data operator by a `.yaml` template.
 
 > [!WARNING]
-> This package provide only orchestration workload task. That mean you should not
+> This package provide only orchestration workload. That mean you should not
 > use the workflow stage to process any large volume data which use lot of compute
 > resource. :cold_sweat:
 
@@ -25,10 +25,10 @@ configuration. It called **Metadata Driven Data Workflow**.
 
 **:pushpin: <u>Rules of This Workflow engine</u>**:
 
-1. Minimum frequency unit of scheduling is **1 minute** :warning:
+1. The Minimum frequency unit of scheduling is **1 minute** :warning:
 2. Can not re-run only failed stage and its pending downstream :rotating_light:
 3. All parallel tasks inside workflow engine use Multi-Threading
-   (Because Python 3.13 unlock GIL :unlock:)
+   (Python 3.13 unlock GIL :unlock:)
 
 > [!NOTE]
 > _Disclaimer_: I inspire the dynamic statement from the [**GitHub Action**](https://github.com/features/actions)
@@ -150,7 +150,7 @@ application. If any configuration values do not set yet, it will use default val
 and do not raise any error to you.
 
 | Environment                             | Component | Default                          | Description                                                                                                        | Remark |
-|:----------------------------------------|-----------|----------------------------------|--------------------------------------------------------------------------------------------------------------------|--------|
+|:----------------------------------------|:----------|:---------------------------------|--------------------------------------------------------------------------------------------------------------------|--------|
 | `WORKFLOW_ROOT_PATH`                    | Core      | .                                | The root path of the workflow application.                                                                         |        |
 | `WORKFLOW_CORE_REGISTRY`                | Core      | src.ddeutil.workflow,tests.utils | List of importable string for the hook stage.                                                                      |        |
 | `WORKFLOW_CORE_REGISTRY_FILTER`         | Core      | ddeutil.workflow.utils           | List of importable string for the filter template.                                                                 |        |

ddeutil_workflow-0.0.19/src/ddeutil/workflow/__about__.py

@@ -0,0 +1 @@
+__version__: str = "0.0.19"

{ddeutil_workflow-0.0.18 → ddeutil_workflow-0.0.19}/src/ddeutil/workflow/__cron.py

@@ -646,6 +646,7 @@ class CronJob:
 
         :param date: An initial date that want to mark as the start point.
         :param tz: A string timezone that want to change on runner.
+
         :rtype: CronRunner
         """
         return CronRunner(self, date, tz=tz)

{ddeutil_workflow-0.0.18 → ddeutil_workflow-0.0.19}/src/ddeutil/workflow/conf.py

@@ -195,6 +195,7 @@ class SimLoad:
         :param conf: A config object.
         :param include:
         :param exclude:
+
         :rtype: Iterator[tuple[str, DictData]]
         """
         exclude: list[str] = exclude or []
@@ -247,12 +248,14 @@ class Loader(SimLoad):
         include: list[str] | None = None,
         exclude: list[str] | None = None,
         **kwargs,
-    ) -> DictData:
+    ) -> Iterator[tuple[str, DictData]]:
         """Override the find class method from the Simple Loader object.
 
         :param obj: A object that want to validate matching before return.
         :param include:
         :param exclude:
+
+        :rtype: Iterator[tuple[str, DictData]]
         """
         return super().finds(
             obj=obj, conf=Config(), include=include, exclude=exclude
@@ -268,6 +271,7 @@ def get_type(t: str, params: Config) -> AnyModelType:
     :param t: A importable type string.
     :param params: A config parameters that use registry to search this
         type.
+
     :rtype: AnyModelType
     """
     try:
@@ -366,6 +370,8 @@ class FileLog(BaseLog):
         workflow name.
 
         :param name: A workflow name that want to search release logging data.
+
+        :rtype: Iterator[Self]
         """
         pointer: Path = config.root_path / f"./logs/workflow={name}"
         if not pointer.exists():
@@ -387,6 +393,9 @@ class FileLog(BaseLog):
         workflow name and release values. If a release does not pass to an input
         argument, it will return the latest release from the current log path.
 
+        :param name:
+        :param release:
+
         :raise FileNotFoundError:
         :raise NotImplementedError:
 
@@ -411,21 +420,17 @@ class FileLog(BaseLog):
             return cls.model_validate(obj=json.load(f))
 
     @classmethod
-    def is_pointed(
-        cls,
-        name: str,
-        release: datetime,
-        *,
-        queue: list[datetime] | None = None,
-    ) -> bool:
-        """Check this log already point in the destination.
+    def is_pointed(cls, name: str, release: datetime) -> bool:
+        """Check the release log already pointed or created at the destination
+        log path.
 
         :param name: A workflow name.
         :param release: A release datetime.
-        :param queue: A list of queue of datetime that already run in the
-            future.
+
+        :rtype: bool
+        :return: Return False if the release log was not pointed or created.
         """
-        # NOTE: Check environ variable was set for real writing.
+        # NOTE: Return False if enable writing log flag does not set.
         if not config.enable_write_log:
             return False
 
@@ -434,9 +439,7 @@ class FileLog(BaseLog):
             name=name, release=release
         )
 
-        if not queue:
-            return pointer.exists()
-        return pointer.exists() or (release in queue)
+        return pointer.exists()
 
     def pointer(self) -> Path:
         """Return release directory path that was generated from model data.

{ddeutil_workflow-0.0.18 → ddeutil_workflow-0.0.19}/src/ddeutil/workflow/job.py

@@ -19,6 +19,7 @@ from concurrent.futures import (
     as_completed,
     wait,
 )
+from enum import Enum
 from functools import lru_cache
 from textwrap import dedent
 from threading import Event
@@ -198,6 +199,11 @@ class Strategy(BaseModel):
         return make(self.matrix, self.include, self.exclude)
 
 
+class TriggerRules(str, Enum):
+    all_success: str = "all_success"
+    all_failed: str = "all_failed"
+
+
 class Job(BaseModel):
     """Job Pydantic model object (group of stages).
 
@@ -245,6 +251,11 @@ class Job(BaseModel):
         default_factory=list,
         description="A list of Stage of this job.",
     )
+    trigger_rule: TriggerRules = Field(
+        default=TriggerRules.all_success,
+        description="A trigger rule of tracking needed jobs.",
+        serialization_alias="trigger-rule",
+    )
     needs: list[str] = Field(
         default_factory=list,
         description="A list of the job ID that want to run before this job.",
@@ -269,6 +280,7 @@ class Job(BaseModel):
         :rtype: DictData
         """
         dash2underscore("runs-on", values)
+        dash2underscore("trigger-rule", values)
         return values
 
     @field_validator("desc", mode="after")

{ddeutil_workflow-0.0.18 → ddeutil_workflow-0.0.19}/src/ddeutil/workflow/on.py

@@ -190,6 +190,19 @@ class On(BaseModel):
         """
         return self.generate(start=start).next
 
+    # def pop(self, queue: list[datetime]) -> datetime:
+    #     """Pop the matching datetime value from list of datetime alias queue."""
+    #     for dt in queue:
+    #         if self.next(dt) == dt:
+    #             return dt
+    #
+    #     # NOTE: Add 1 second value to the current datetime for forcing crontab
+    #     #   runner generate the next datetime instead if current datetime be
+    #     #   valid because I already replaced second to zero before passing.
+    #     return datetime.now(tz=config.tz).replace(
+    #         second=0, microsecond=0
+    #     ) + timedelta(seconds=1)
+
 
 class YearOn(On):
     """Implement On Year Schedule Model for limit year matrix that use by some