ddeutil-workflow 0.0.17__py3-none-any.whl → 0.0.19__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1 +1 @@
-__version__: str = "0.0.17"
+__version__: str = "0.0.19"

ddeutil/workflow/__cron.py

@@ -646,6 +646,7 @@ class CronJob:
 
         :param date: An initial date that want to mark as the start point.
         :param tz: A string timezone that want to change on runner.
+
         :rtype: CronRunner
         """
         return CronRunner(self, date, tz=tz)

ddeutil/workflow/__init__.py

@@ -3,7 +3,11 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
-from .conf import Config, FileLog, Loader
+from .conf import (
+    Config,
+    FileLog,
+    Loader,
+)
 from .exceptions import (
     JobException,
     ParamValueException,
@@ -12,14 +16,57 @@ from .exceptions import (
     WorkflowException,
 )
 from .job import Job, Strategy
-from .on import On, interval2crontab
+from .on import (
+    On,
+    YearOn,
+    interval2crontab,
+)
 from .scheduler import (
     Schedule,
+    ScheduleWorkflow,
     Workflow,
+    WorkflowTaskData,
+)
+from .stage import (
+    BashStage,
+    EmptyStage,
+    HookStage,
+    PyStage,
+    Stage,
+    TriggerStage,
+    handler_result,
 )
-from .stage import Stage, handler_result
 from .utils import (
+    FILTERS,
+    ChoiceParam,
+    DatetimeParam,
+    DefaultParam,
+    FilterFunc,
+    FilterRegistry,
+    IntParam,
     Param,
+    Result,
+    ReturnTagFunc,
+    StrParam,
+    TagFunc,
+    batch,
+    cross_product,
+    custom_filter,
     dash2underscore,
+    delay,
+    filter_func,
+    gen_id,
+    get_args_const,
+    get_diff_sec,
+    get_dt_now,
+    has_template,
+    make_exec,
+    make_filter_registry,
+    make_registry,
+    map_post_filter,
+    not_in_template,
     param2template,
+    queue2str,
+    str2template,
+    tag,
 )

ddeutil/workflow/conf.py

@@ -41,8 +41,8 @@ class Config:
     # NOTE: Core
     root_path: Path = Path(os.getenv("WORKFLOW_ROOT_PATH", "."))
     tz: ZoneInfo = ZoneInfo(env("WORKFLOW_CORE_TIMEZONE", "UTC"))
-    workflow_id_simple_mode: bool = str2bool(
-        os.getenv("WORKFLOW_CORE_WORKFLOW_ID_SIMPLE_MODE", "true")
+    gen_id_simple_mode: bool = str2bool(
+        os.getenv("WORKFLOW_CORE_GENERATE_ID_SIMPLE_MODE", "true")
     )
 
     # NOTE: Register
@@ -195,6 +195,7 @@ class SimLoad:
         :param conf: A config object.
         :param include:
         :param exclude:
+
         :rtype: Iterator[tuple[str, DictData]]
         """
         exclude: list[str] = exclude or []
@@ -247,12 +248,14 @@ class Loader(SimLoad):
         include: list[str] | None = None,
         exclude: list[str] | None = None,
         **kwargs,
-    ) -> DictData:
+    ) -> Iterator[tuple[str, DictData]]:
         """Override the find class method from the Simple Loader object.
 
         :param obj: A object that want to validate matching before return.
         :param include:
         :param exclude:
+
+        :rtype: Iterator[tuple[str, DictData]]
         """
         return super().finds(
             obj=obj, conf=Config(), include=include, exclude=exclude
@@ -268,6 +271,7 @@ def get_type(t: str, params: Config) -> AnyModelType:
     :param t: A importable type string.
     :param params: A config parameters that use registry to search this
         type.
+
     :rtype: AnyModelType
     """
     try:
@@ -366,6 +370,8 @@ class FileLog(BaseLog):
         workflow name.
 
         :param name: A workflow name that want to search release logging data.
+
+        :rtype: Iterator[Self]
         """
         pointer: Path = config.root_path / f"./logs/workflow={name}"
         if not pointer.exists():
@@ -387,6 +393,9 @@ class FileLog(BaseLog):
         workflow name and release values. If a release does not pass to an input
         argument, it will return the latest release from the current log path.
 
+        :param name:
+        :param release:
+
         :raise FileNotFoundError:
         :raise NotImplementedError:
 
@@ -411,21 +420,17 @@ class FileLog(BaseLog):
             return cls.model_validate(obj=json.load(f))
 
     @classmethod
-    def is_pointed(
-        cls,
-        name: str,
-        release: datetime,
-        *,
-        queue: list[datetime] | None = None,
-    ) -> bool:
-        """Check this log already point in the destination.
+    def is_pointed(cls, name: str, release: datetime) -> bool:
+        """Check the release log already pointed or created at the destination
+        log path.
 
         :param name: A workflow name.
         :param release: A release datetime.
-        :param queue: A list of queue of datetime that already run in the
-            future.
+
+        :rtype: bool
+        :return: Return False if the release log was not pointed or created.
         """
-        # NOTE: Check environ variable was set for real writing.
+        # NOTE: Return False if enable writing log flag does not set.
         if not config.enable_write_log:
             return False
 
@@ -434,9 +439,7 @@ class FileLog(BaseLog):
             name=name, release=release
         )
 
-        if not queue:
-            return pointer.exists()
-        return pointer.exists() or (release in queue)
+        return pointer.exists()
 
     def pointer(self) -> Path:
         """Return release directory path that was generated from model data.

ddeutil/workflow/job.py

@@ -19,6 +19,7 @@ from concurrent.futures import (
     as_completed,
     wait,
 )
+from enum import Enum
 from functools import lru_cache
 from textwrap import dedent
 from threading import Event
@@ -198,6 +199,11 @@ class Strategy(BaseModel):
         return make(self.matrix, self.include, self.exclude)
 
 
+class TriggerRules(str, Enum):
+    all_success: str = "all_success"
+    all_failed: str = "all_failed"
+
+
 class Job(BaseModel):
     """Job Pydantic model object (group of stages).
 
@@ -245,6 +251,11 @@ class Job(BaseModel):
         default_factory=list,
         description="A list of Stage of this job.",
     )
+    trigger_rule: TriggerRules = Field(
+        default=TriggerRules.all_success,
+        description="A trigger rule of tracking needed jobs.",
+        serialization_alias="trigger-rule",
+    )
     needs: list[str] = Field(
         default_factory=list,
         description="A list of the job ID that want to run before this job.",
@@ -269,6 +280,7 @@ class Job(BaseModel):
         :rtype: DictData
         """
         dash2underscore("runs-on", values)
+        dash2underscore("trigger-rule", values)
         return values
 
     @field_validator("desc", mode="after")
@@ -553,8 +565,6 @@ class Job(BaseModel):
         # NOTE: Create event for cancel executor by trigger stop running event.
         event: Event = Event()
 
-        print("Job Run Fail-Fast:", self.strategy.fail_fast)
-
         # IMPORTANT: Start running strategy execution by multithreading because
         #   it will running by strategy values without waiting previous
         #   execution.

ddeutil/workflow/on.py

@@ -190,6 +190,19 @@ class On(BaseModel):
         """
         return self.generate(start=start).next
 
+    # def pop(self, queue: list[datetime]) -> datetime:
+    #     """Pop the matching datetime value from list of datetime alias queue."""
+    #     for dt in queue:
+    #         if self.next(dt) == dt:
+    #             return dt
+    #
+    #     # NOTE: Add 1 second value to the current datetime for forcing crontab
+    #     #   runner generate the next datetime instead if current datetime be
+    #     #   valid because I already replaced second to zero before passing.
+    #     return datetime.now(tz=config.tz).replace(
+    #         second=0, microsecond=0
+    #     ) + timedelta(seconds=1)
+
 
 class YearOn(On):
     """Implement On Year Schedule Model for limit year matrix that use by some

ddeutil/workflow/repeat.py

@@ -12,8 +12,8 @@ from functools import wraps
 
 from starlette.concurrency import run_in_threadpool
 
+from .__cron import CronJob
 from .conf import config, get_logger
-from .cron import CronJob
 
 logger = get_logger("ddeutil.workflow")
 

ddeutil/workflow/scheduler.py

@@ -218,6 +218,24 @@ class Workflow(BaseModel):
         """
         return dedent(value)
 
+    @field_validator("on", mode="after")
+    def __on_no_dup__(cls, value: list[On]) -> list[On]:
+        """Validate the on fields should not contain duplicate values and if it
+        contain every minute value, it should has only one on value."""
+        set_ons: set[str] = {str(on.cronjob) for on in value}
+        if len(set_ons) != len(value):
+            raise ValueError(
+                "The on fields should not contain duplicate on value."
+            )
+
+        # WARNING:
+        # if '* * * * *' in set_ons and len(set_ons) > 1:
+        #     raise ValueError(
+        #         "If it has every minute cronjob on value, it should has only "
+        #         "one value in the on field."
+        #     )
+        return value
+
     @model_validator(mode="after")
     def __validate_jobs_need_and_prepare_running_id(self) -> Self:
         """Validate each need job in any jobs should exists.
@@ -265,7 +283,7 @@ class Workflow(BaseModel):
         :type name: str
 
         :rtype: Job
-        :returns: A job model that exists on this workflow by input name.
+        :return: A job model that exists on this workflow by input name.
         """
         if name not in self.jobs:
             raise ValueError(
@@ -287,7 +305,13 @@ class Workflow(BaseModel):
 
         :param params: A parameter mapping that receive from workflow execution.
         :type params: DictData
+
+        :raise WorkflowException: If parameter value that want to validate does
+            not include the necessary parameter that had required flag.
+
         :rtype: DictData
+        :return: The parameter value that validate with its parameter fields and
+            adding jobs key to this parameter.
         """
         # VALIDATE: Incoming params should have keys that set on this workflow.
         if check_key := tuple(
@@ -315,7 +339,7 @@ class Workflow(BaseModel):
 
     def release(
         self,
-        on: On,
+        runner: CronRunner,
         params: DictData,
         queue: list[datetime],
         *,
@@ -330,41 +354,42 @@ class Workflow(BaseModel):
             This method allow workflow use log object to save the execution
         result to log destination like file log to local `/logs` directory.
 
-        :param on: An on schedule value.
+            I will add sleep with 0.15 seconds on every step that interact with
+        the queue object.
+
+        :param runner: A CronRunner instance.
         :param params: A workflow parameter that pass to execute method.
         :param queue: A list of release time that already running.
         :param waiting_sec: A second period value that allow workflow execute.
         :param sleep_interval: A second value that want to waiting until time
             to execute.
         :param log: A log object that want to save execution result.
+
         :rtype: Result
         """
         logger.debug(
-            f"({self.run_id}) [CORE]: {self.name!r}: {on.cronjob} : run with "
+            f"({self.run_id}) [CORE]: {self.name!r}: {runner.cron} : run with "
             f"queue id: {id(queue)}"
         )
         log: Log = log or FileLog
-        gen: CronRunner = on.generate(
-            datetime.now(tz=config.tz).replace(second=0, microsecond=0)
-            + timedelta(seconds=1)
-        )
-        cron_tz: ZoneInfo = gen.tz
+        cron_tz: ZoneInfo = runner.tz
 
         # NOTE: get next schedule time that generate from now.
-        next_time: datetime = gen.next
+        next_time: datetime = runner.next
 
         # NOTE: While-loop to getting next until it does not logger.
-        while log.is_pointed(self.name, next_time, queue=queue):
-            next_time: datetime = gen.next
+        while log.is_pointed(self.name, next_time) or (next_time in queue):
+            next_time: datetime = runner.next
 
         # NOTE: Heap-push this next running time to log queue list.
         heappush(queue, next_time)
+        time.sleep(0.15)
 
         # VALIDATE: Check the different time between the next schedule time and
         #   now that less than waiting period (second unit).
         if get_diff_sec(next_time, tz=cron_tz) > waiting_sec:
             logger.debug(
-                f"({self.run_id}) [CORE]: {self.name!r} : {on.cronjob} : "
+                f"({self.run_id}) [CORE]: {self.name!r} : {runner.cron} : "
                 f"Does not closely >> {next_time:%Y-%m-%d %H:%M:%S}"
             )
 
@@ -376,54 +401,51 @@ class Workflow(BaseModel):
                 status=0,
                 context={
                     "params": params,
-                    "poking": {"skipped": [str(on.cronjob)], "run": []},
+                    "release": {
+                        "status": "skipped",
+                        "cron": [str(runner.cron)],
+                    },
                 },
             )
 
         logger.debug(
-            f"({self.run_id}) [CORE]: {self.name!r} : {on.cronjob} : "
+            f"({self.run_id}) [CORE]: {self.name!r} : {runner.cron} : "
             f"Closely to run >> {next_time:%Y-%m-%d %H:%M:%S}"
         )
 
         # NOTE: Release when the time is nearly to schedule time.
         while (duration := get_diff_sec(next_time, tz=cron_tz)) > (
             sleep_interval + 5
-        ):
+        ):  # pragma: no cov
             logger.debug(
-                f"({self.run_id}) [CORE]: {self.name!r} : {on.cronjob} : "
+                f"({self.run_id}) [CORE]: {self.name!r} : {runner.cron} : "
                 f"Sleep until: {duration}"
             )
             time.sleep(sleep_interval)
 
-        time.sleep(0.5)
+        time.sleep(0.15)
 
-        # NOTE: Release parameter that use to change if params has
-        #   templating.
-        release_params: DictData = {
-            "release": {
-                "logical_date": next_time,
-            },
-        }
+        # NOTE: Release parameter that use to change if params has templating.
+        release_params: DictData = {"release": {"logical_date": next_time}}
 
-        # WARNING: Re-create workflow object that use new running workflow
-        #   ID.
-        runner: Self = self.get_running_id(run_id=self.new_run_id)
-        rs: Result = runner.execute(
+        # WARNING: Re-create workflow object that use new running workflow ID.
+        workflow: Self = self.get_running_id(run_id=self.new_run_id)
+        rs: Result = workflow.execute(
             params=param2template(params, release_params),
         )
         logger.debug(
-            f"({runner.run_id}) [CORE]: {self.name!r} : {on.cronjob} : "
+            f"({workflow.run_id}) [CORE]: {self.name!r} : {runner.cron} : "
             f"End release {next_time:%Y-%m-%d %H:%M:%S}"
         )
 
         # NOTE: Delete a copied workflow instance for saving memory.
-        del runner
+        del workflow
 
         rs.set_parent_run_id(self.run_id)
         rs_log: Log = log.model_validate(
             {
                 "name": self.name,
-                "on": str(on.cronjob),
+                "on": str(runner.cron),
                 "release": next_time,
                 "context": rs.context,
                 "parent_run_id": rs.run_id,
@@ -434,27 +456,31 @@ class Workflow(BaseModel):
         rs_log.save(excluded=None)
 
         queue.remove(next_time)
-        time.sleep(0.05)
+        time.sleep(0.15)
         return Result(
             status=0,
             context={
                 "params": params,
-                "poking": {"skipped": [], "run": [str(on.cronjob)]},
+                "release": {"status": "run", "cron": [str(runner.cron)]},
             },
         )
 
     def poke(
         self,
+        start_date: datetime | None = None,
         params: DictData | None = None,
         *,
         log: Log | None = None,
     ) -> list[Result]:
-        """Poke workflow with threading executor pool for executing with all its
-        schedules that was set on the `on` value. This method will observe its
-        schedule that nearing to run with the ``self.release()`` method.
+        """Poke workflow with the ``on`` field with threading executor pool for
+        executing with all its schedules that was set on the `on` value.
+        This method will observe its schedule that nearing to run with the
+        ``self.release()`` method.
 
+        :param start_date: A start datetime object.
         :param params: A parameters that want to pass to the release method.
         :param log: A log object that want to use on this poking process.
+
         :rtype: list[Result]
         """
         logger.info(
@@ -470,21 +496,32 @@ class Workflow(BaseModel):
         queue: list[datetime] = []
         results: list[Result] = []
 
+        start_date: datetime = start_date or datetime.now(tz=config.tz).replace(
+            second=0, microsecond=0
+        ) + timedelta(seconds=1)
+
         with ThreadPoolExecutor(
             max_workers=config.max_poking_pool_worker,
             thread_name_prefix="wf_poking_",
         ) as executor:
+
             futures: list[Future] = []
+
+            # NOTE: For-loop the on values that exists in this workflow object.
             for on in self.on:
                 futures.append(
                     executor.submit(
                         self.release,
-                        on,
+                        on.generate(start_date),
                         params=params,
                         log=log,
                         queue=queue,
                     )
                 )
+
+                # NOTE: Delay release date because it run so fast and making
+                #   queue object can not handle release date that will duplicate
+                #   by the cron runner object.
                 delay(second=0.15)
 
             # WARNING: This poking method does not allow to use fail-fast logic
@@ -492,7 +529,7 @@ class Workflow(BaseModel):
             for future in as_completed(futures):
                 results.append(future.result(timeout=60))
 
-        if len(queue) > 0:
+        if len(queue) > 0:  # pragma: no cov
             logger.error(
                 f"({self.run_id}) [POKING]: Log Queue does empty when poking "
                 f"process was finishing."
@@ -717,11 +754,11 @@ class Workflow(BaseModel):
             return context
 
         # NOTE: Raise timeout error.
-        logger.warning(
+        logger.warning(  # pragma: no cov
             f"({self.run_id}) [WORKFLOW]: Execution of workflow, {self.name!r} "
             f", was timeout"
         )
-        raise WorkflowException(
+        raise WorkflowException(  # pragma: no cov
             f"Execution of workflow: {self.name} was timeout"
         )
 
@@ -765,7 +802,8 @@ class Workflow(BaseModel):
                 continue
 
             # NOTE: Start workflow job execution with deep copy context data
-            #   before release.
+            #   before release. This job execution process will running until
+            #   done before checking all execution timeout or not.
             #
             #   {
             #       'params': <input-params>,
@@ -783,10 +821,10 @@ class Workflow(BaseModel):
             return context
 
         # NOTE: Raise timeout error.
-        logger.warning(
+        logger.warning(  # pragma: no cov
             f"({self.run_id}) [WORKFLOW]: Execution of workflow was timeout"
         )
-        raise WorkflowException(
+        raise WorkflowException(  # pragma: no cov
             f"Execution of workflow: {self.name} was timeout"
         )
 
@@ -805,7 +843,7 @@ class ScheduleWorkflow(BaseModel):
     )
     params: DictData = Field(
         default_factory=dict,
-        description="A parameters that want to use to workflow execution.",
+        description="A parameters that want to use in workflow execution.",
     )
 
     @model_validator(mode="before")
@@ -825,19 +863,23 @@ class ScheduleWorkflow(BaseModel):
         data: DictData,
         externals: DictData | None = None,
     ) -> DictData:
-        """Bypass the on data to loaded config data.
+        """Bypass and prepare the on data to loaded config data.
+
+        :param data:
+        :param externals:
 
         :rtype: DictData
         """
         if on := data.pop("on", []):
 
             if isinstance(on, str):
-                on = [on]
+                on: list[str] = [on]
 
             if any(not isinstance(n, (dict, str)) for n in on):
                 raise TypeError("The ``on`` key should be list of str or dict")
 
-            # NOTE: Pass on value to Loader and keep on model object to on field
+            # NOTE: Pass on value to Loader and keep on model object to on
+            #   field.
             data["on"] = [
                 (
                     Loader(n, externals=(externals or {})).data
@@ -848,6 +890,24 @@ class ScheduleWorkflow(BaseModel):
             ]
         return data
 
+    @field_validator("on", mode="after")
+    def __on_no_dup__(cls, value: list[On]) -> list[On]:
+        """Validate the on fields should not contain duplicate values and if it
+        contain every minute value, it should has only one on value."""
+        set_ons: set[str] = {str(on.cronjob) for on in value}
+        if len(set_ons) != len(value):
+            raise ValueError(
+                "The on fields should not contain duplicate on value."
+            )
+
+        # WARNING:
+        # if '* * * * *' in set_ons and len(set_ons) > 1:
+        #     raise ValueError(
+        #         "If it has every minute cronjob on value, it should has only "
+        #         "one value in the on field."
+        #     )
+        return value
+
 
 class Schedule(BaseModel):
     """Schedule Pydantic Model that use to run with scheduler package. It does
@@ -866,6 +926,15 @@ class Schedule(BaseModel):
         description="A list of ScheduleWorkflow models.",
     )
 
+    @field_validator("desc", mode="after")
+    def __dedent_desc__(cls, value: str) -> str:
+        """Prepare description string that was created on a template.
+
+        :param value: A description string value that want to dedent.
+        :rtype: str
+        """
+        return dedent(value)
+
     @classmethod
     def from_loader(
         cls,
@@ -902,45 +971,51 @@ class Schedule(BaseModel):
         *,
         externals: DictData | None = None,
     ) -> list[WorkflowTaskData]:
-        """Generate Task from the current datetime.
+        """Return the list of WorkflowTaskData object from the specific input
+        datetime that mapping with the on field.
 
         :param start_date: A start date that get from the workflow schedule.
         :param queue: A mapping of name and list of datetime for queue.
         :param running: A mapping of name and list of datetime for running.
         :param externals: An external parameters that pass to the Loader object.
+
         :rtype: list[WorkflowTaskData]
+        :return: Return the list of WorkflowTaskData object from the specific
+            input datetime that mapping with the on field.
         """
 
         # NOTE: Create pair of workflow and on.
         workflow_tasks: list[WorkflowTaskData] = []
-        externals: DictData = externals or {}
+        extras: DictData = externals or {}
 
-        for wfs in self.workflows:
-            wf: Workflow = Workflow.from_loader(wfs.name, externals=externals)
+        for sch_wf in self.workflows:
+            wf: Workflow = Workflow.from_loader(sch_wf.name, externals=extras)
 
             # NOTE: Create default list of release datetime.
-            queue[wfs.name]: list[datetime] = []
-            running[wfs.name]: list[datetime] = []
+            queue[sch_wf.name]: list[datetime] = []
+            running[sch_wf.name]: list[datetime] = []
+
+            # IMPORTANT: Create the default 'on' value if it does not passing
+            #   the on field to the Schedule object.
+            ons: list[On] = wf.on.copy() if len(sch_wf.on) == 0 else sch_wf.on
 
-            # NOTE: Create default on if it does not passing on the Schedule.
-            _ons: list[On] = wf.on.copy() if len(wfs.on) == 0 else wfs.on
+            for on in ons:
+                gen: CronRunner = on.generate(start_date)
+                next_running_date = gen.next
 
-            for on in _ons:
-                on_gen = on.generate(start_date)
-                next_running_date = on_gen.next
-                while next_running_date in queue[wfs.name]:
-                    next_running_date = on_gen.next
+                while next_running_date in queue[sch_wf.name]:
+                    next_running_date = gen.next
 
                 # NOTE: Push the next running date to queue list.
-                heappush(queue[wfs.name], next_running_date)
+                heappush(queue[sch_wf.name], next_running_date)
 
                 workflow_tasks.append(
                     WorkflowTaskData(
                         workflow=wf,
                         on=on,
-                        params=wfs.params,
-                        queue=queue,
-                        running=running,
+                        params=sch_wf.params,
+                        queue=queue[sch_wf.name],
+                        running=running[sch_wf.name],
                     ),
                 )
 
@@ -957,13 +1032,14 @@ def catch_exceptions(cancel_on_failure: bool = False) -> DecoratorCancelJob:
 
     :param cancel_on_failure: A flag that allow to return the CancelJob or not
         it will raise.
-    :rtype: Callable[P, Optional[CancelJob]]
+
+    :rtype: DecoratorCancelJob
     """
 
     def decorator(func: ReturnCancelJob) -> ReturnCancelJob:
         try:
             # NOTE: Check the function that want to handle is method or not.
-            if inspect.ismethod(func):
+            if inspect.ismethod(func):  # pragma: no cov
 
                 @wraps(func)
                 def wrapper(self, *args, **kwargs):
@@ -977,7 +1053,7 @@ def catch_exceptions(cancel_on_failure: bool = False) -> DecoratorCancelJob:
 
             return wrapper
 
-        except Exception as err:
+        except Exception as err:  # pragma: no cov
             logger.exception(err)
             if cancel_on_failure:
                 return CancelJob
@@ -990,13 +1066,15 @@ def catch_exceptions(cancel_on_failure: bool = False) -> DecoratorCancelJob:
 class WorkflowTaskData:
     """Workflow task dataclass that use to keep mapping data and objects for
     passing in multithreading task.
+
+        This dataclass will be 1-1 mapping with workflow and on objects.
     """
 
     workflow: Workflow
     on: On
     params: DictData = field(compare=False, hash=False)
-    queue: dict[str, list[datetime]] = field(compare=False, hash=False)
-    running: dict[str, list[datetime]] = field(compare=False, hash=False)
+    queue: list[datetime] = field(compare=False, hash=False)
+    running: list[datetime] = field(compare=False, hash=False)
 
     @catch_exceptions(cancel_on_failure=True)
     def release(
@@ -1005,9 +1083,9 @@ class WorkflowTaskData:
         *,
         waiting_sec: int = 60,
         sleep_interval: int = 15,
-    ) -> None:
-        """Workflow release, it will use with the same logic of
-        `workflow.release` method.
+    ) -> None:  # pragma: no cov
+        """Workflow task release that use the same logic of `workflow.release`
+        method.
 
         :param log: A log object for saving result logging from workflow
             execution process.
@@ -1028,14 +1106,14 @@ class WorkflowTaskData:
         next_time: datetime = gen.next
 
         # NOTE: get next utils it does not running.
-        while log.is_pointed(wf.name, next_time, queue=self.running[wf.name]):
+        while log.is_pointed(wf.name, next_time) or (next_time in self.running):
             next_time: datetime = gen.next
 
         logger.debug(
             f"({wf.run_id}) [CORE]: {wf.name!r} : {on.cronjob} : "
             f"{next_time:%Y-%m-%d %H:%M:%S}"
         )
-        heappush(self.running[wf.name], next_time)
+        heappush(self.running, next_time)
 
         if get_diff_sec(next_time, tz=cron_tz) > waiting_sec:
             logger.debug(
@@ -1045,8 +1123,8 @@ class WorkflowTaskData:
 
             # NOTE: Add this next running datetime that not in period to queue
             #   and remove it to running.
-            self.running[wf.name].remove(next_time)
-            heappush(self.queue[wf.name], next_time)
+            self.running.remove(next_time)
+            heappush(self.queue, next_time)
 
             time.sleep(0.2)
             return
@@ -1107,7 +1185,7 @@ class WorkflowTaskData:
         rs_log.save(excluded=None)
 
         # NOTE: remove this release date from running
-        self.running[wf.name].remove(next_time)
+        self.running.remove(next_time)
 
         # IMPORTANT:
         #   Add the next running datetime to workflow queue
@@ -1116,13 +1194,13 @@ class WorkflowTaskData:
         )
         future_running_time: datetime = gen.next
         while (
-            future_running_time in self.running[wf.name]
-            or future_running_time in self.queue[wf.name]
+            future_running_time in self.running
+            or future_running_time in self.queue
             or future_running_time < finish_time
-        ):
+        ):  # pragma: no cov
             future_running_time: datetime = gen.next
 
-        heappush(self.queue[wf.name], future_running_time)
+        heappush(self.queue, future_running_time)
         logger.debug(f"[CORE]: {'-' * 100}")
 
     def __eq__(self, other) -> bool:
@@ -1134,7 +1212,7 @@ class WorkflowTaskData:
         return NotImplemented
 
 
-@catch_exceptions(cancel_on_failure=True)
+@catch_exceptions(cancel_on_failure=True)  # pragma: no cov
 def workflow_task(
     workflow_tasks: list[WorkflowTaskData],
     stop: datetime,
@@ -1233,7 +1311,7 @@ def workflow_task(
     logger.debug(f"[WORKFLOW]: {'=' * 100}")
 
 
-def workflow_monitor(threads: dict[str, Thread]) -> None:
+def workflow_monitor(threads: dict[str, Thread]) -> None:  # pragma: no cov
     """Workflow schedule for monitoring long running thread from the schedule
     control.
 
@@ -1255,7 +1333,7 @@ def workflow_control(
     schedules: list[str],
     stop: datetime | None = None,
     externals: DictData | None = None,
-) -> list[str]:
+) -> list[str]:  # pragma: no cov
     """Workflow scheduler control.
 
     :param schedules: A list of workflow names that want to schedule running.
@@ -1270,7 +1348,7 @@ def workflow_control(
             "Should install schedule package before use this module."
         ) from None
 
-    schedule: Scheduler = Scheduler()
+    scheduler: Scheduler = Scheduler()
     start_date: datetime = datetime.now(tz=config.tz)
 
     # NOTE: Design workflow queue caching.
@@ -1288,9 +1366,11 @@ def workflow_control(
     # NOTE: Create pair of workflow and on from schedule model.
     workflow_tasks: list[WorkflowTaskData] = []
     for name in schedules:
-        sch: Schedule = Schedule.from_loader(name, externals=externals)
+        schedule: Schedule = Schedule.from_loader(name, externals=externals)
+
+        # NOTE: Create a workflow task data instance from schedule object.
         workflow_tasks.extend(
-            sch.tasks(
+            schedule.tasks(
                 start_date_waiting,
                 queue=wf_queue,
                 running=wf_running,
@@ -1300,7 +1380,7 @@ def workflow_control(
 
     # NOTE: This schedule job will start every minute at :02 seconds.
     (
-        schedule.every(1)
+        scheduler.every(1)
         .minutes.at(":02")
         .do(
             workflow_task,
@@ -1312,7 +1392,7 @@ def workflow_control(
     )
 
     # NOTE: Checking zombie task with schedule job will start every 5 minute.
-    schedule.every(5).minutes.at(":10").do(
+    scheduler.every(5).minutes.at(":10").do(
         workflow_monitor,
         threads=thread_releases,
     ).tag("monitor")
@@ -1320,10 +1400,10 @@ def workflow_control(
     # NOTE: Start running schedule
     logger.info(f"[WORKFLOW]: Start schedule: {schedules}")
     while True:
-        schedule.run_pending()
+        scheduler.run_pending()
         time.sleep(1)
-        if not schedule.get_jobs("control"):
-            schedule.clear("monitor")
+        if not scheduler.get_jobs("control"):
+            scheduler.clear("monitor")
             logger.warning(
                 f"[WORKFLOW]: Workflow release thread: {thread_releases}"
             )
@@ -1343,7 +1423,7 @@ def workflow_runner(
     stop: datetime | None = None,
     externals: DictData | None = None,
     excluded: list[str] | None = None,
-) -> list[str]:
+) -> list[str]:  # pragma: no cov
     """Workflow application that running multiprocessing schedule with chunk of
     workflows that exists in config path.
 
@@ -1360,14 +1440,14 @@ def workflow_runner(
 
         The current workflow logic that split to process will be below diagram:
 
-        PIPELINES ==> process 01 ==> schedule 1 minute --> thread of release
-                                                           workflow task 01 01
-                                                       --> thread of release
-                                                           workflow task 01 02
-                  ==> process 02 ==> schedule 1 minute --> thread of release
-                                                           workflow task 02 01
-                                                       --> thread of release
-                                                           workflow task 02 02
+        PIPELINES ==> process 01 ==> schedule --> thread of release
+                                                  workflow task 01 01
+                                              --> thread of release
+                                                  workflow task 01 02
+                  ==> process 02 ==> schedule --> thread of release
+                                                  workflow task 02 01
+                                              --> thread of release
+                                                  workflow task 02 02
                   ==> ...
     """
     excluded: list[str] = excluded or []

ddeutil/workflow/stage.py

@@ -127,7 +127,6 @@ def handler_result(message: str | None = None) -> DecoratorResult:
                 logger.error(
                     f"({self.run_id}) [STAGE]: {err.__class__.__name__}: {err}"
                 )
-                print("Stage Raise error:", config.stage_raise_error)
                 if config.stage_raise_error:
                     # NOTE: If error that raise from stage execution course by
                     #   itself, it will return that error with previous

ddeutil/workflow/utils.py

@@ -100,7 +100,7 @@ def gen_id(
     if not isinstance(value, str):
         value: str = str(value)
 
-    if config.workflow_id_simple_mode:
+    if config.gen_id_simple_mode:
         return hash_str(f"{(value if sensitive else value.lower())}", n=10) + (
             f"{datetime.now(tz=config.tz):%Y%m%d%H%M%S%f}" if unique else ""
         )
@@ -445,6 +445,7 @@ FILTERS: dict[str, callable] = {  # pragma: no cov
     "abs": abs,
     "str": str,
     "int": int,
+    "title": lambda x: x.title(),
     "upper": lambda x: x.upper(),
     "lower": lambda x: x.lower(),
     "rstr": [str, repr],
@@ -549,6 +550,30 @@ def get_args_const(
     return name.id, args, keywords
 
 
+def get_args_from_filter(
+    ft: str,
+    filters: dict[str, FilterRegistry],
+) -> tuple[str, FilterRegistry, list[Any], dict[Any, Any]]:  # pragma: no cov
+    """Get arguments and keyword-arguments from filter function calling string.
+    and validate it with the filter functions mapping dict.
+    """
+    func_name, _args, _kwargs = get_args_const(ft)
+    args: list[Any] = [arg.value for arg in _args]
+    kwargs: dict[Any, Any] = {k: v.value for k, v in _kwargs.items()}
+
+    if func_name not in filters:
+        raise UtilException(
+            f"The post-filter: {func_name} does not support yet."
+        )
+
+    if isinstance((f_func := filters[func_name]), list) and (args or kwargs):
+        raise UtilException(
+            "Chain filter function does not support for passing arguments."
+        )
+
+    return func_name, f_func, args, kwargs
+
+
 @custom_filter("fmt")  # pragma: no cov
 def datetime_format(value: datetime, fmt: str = "%Y-%m-%d %H:%M:%S") -> str:
     """Format datetime object to string with the format."""
@@ -573,28 +598,16 @@ def map_post_filter(
 
     :rtype: T
     """
-    for _filter in post_filter:
-        func_name, _args, _kwargs = get_args_const(_filter)
-        args: list = [arg.value for arg in _args]
-        kwargs: dict = {k: v.value for k, v in _kwargs.items()}
-
-        if func_name not in filters:
-            raise UtilException(
-                f"The post-filter: {func_name} does not support yet."
-            )
-
+    for ft in post_filter:
+        func_name, f_func, args, kwargs = get_args_from_filter(ft, filters)
         try:
-            if isinstance((f_func := filters[func_name]), list):
-                if args or kwargs:
-                    raise UtilException(
-                        "Chain filter function does not support for passing "
-                        "arguments."
-                    )
+            if isinstance(f_func, list):
                 for func in f_func:
-                    value: Any = func(value)
+                    value: T = func(value)
             else:
-                value: Any = f_func(value, *args, **kwargs)
-        except UtilException:
+                value: T = f_func(value, *args, **kwargs)
+        except UtilException as err:
+            logger.warning(str(err))
             raise
         except Exception as err:
             logger.warning(str(err))

{ddeutil_workflow-0.0.17.dist-info → ddeutil_workflow-0.0.19.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ddeutil-workflow
-Version: 0.0.17
+Version: 0.0.19
 Summary: Lightweight workflow orchestration with less dependencies
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT
@@ -24,6 +24,7 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: ddeutil >=0.4.3
 Requires-Dist: ddeutil-io[toml,yaml] >=0.2.3
+Requires-Dist: pydantic ==2.9.2
 Requires-Dist: python-dotenv ==1.0.1
 Requires-Dist: typer <1.0.0,==0.12.5
 Requires-Dist: schedule <2.0.0,==1.2.2
@@ -45,7 +46,7 @@ for easy to make a simple metadata driven for data workflow orchestration.
 It can to use for data operator by a `.yaml` template.
 
 > [!WARNING]
-> This package provide only orchestration workload task. That mean you should not
+> This package provide only orchestration workload. That mean you should not
 > use the workflow stage to process any large volume data which use lot of compute
 > resource. :cold_sweat:
 
@@ -57,10 +58,10 @@ configuration. It called **Metadata Driven Data Workflow**.
 
 **:pushpin: <u>Rules of This Workflow engine</u>**:
 
-1. Minimum frequency unit of scheduling is **1 minute** :warning:
+1. The Minimum frequency unit of scheduling is **1 minute** :warning:
 2. Can not re-run only failed stage and its pending downstream :rotating_light:
 3. All parallel tasks inside workflow engine use Multi-Threading
-   (Because Python 3.13 unlock GIL :unlock:)
+   (Python 3.13 unlock GIL :unlock:)
 
 > [!NOTE]
 > _Disclaimer_: I inspire the dynamic statement from the [**GitHub Action**](https://github.com/features/actions)
@@ -182,7 +183,7 @@ application. If any configuration values do not set yet, it will use default val
 and do not raise any error to you.
 
 | Environment                             | Component | Default                          | Description                                                                                                        | Remark |
-|:----------------------------------------|-----------|----------------------------------|--------------------------------------------------------------------------------------------------------------------|--------|
+|:----------------------------------------|:----------|:---------------------------------|--------------------------------------------------------------------------------------------------------------------|--------|
 | `WORKFLOW_ROOT_PATH`                    | Core      | .                                | The root path of the workflow application.                                                                         |        |
 | `WORKFLOW_CORE_REGISTRY`                | Core      | src.ddeutil.workflow,tests.utils | List of importable string for the hook stage.                                                                      |        |
 | `WORKFLOW_CORE_REGISTRY_FILTER`         | Core      | ddeutil.workflow.utils           | List of importable string for the filter template.                                                                 |        |
@@ -194,7 +195,7 @@ and do not raise any error to you.
 | `WORKFLOW_CORE_JOB_RAISE_ERROR`         | Core      | true                             | A flag that all job raise JobException from job strategy execution.                                                |        |
 | `WORKFLOW_CORE_MAX_NUM_POKING`          | Core      | 4                                | .                                                                                                                  |        |
 | `WORKFLOW_CORE_MAX_JOB_PARALLEL`        | Core      | 2                                | The maximum job number that able to run parallel in workflow executor.                                             |        |
-| `WORKFLOW_CORE_WORKFLOW_ID_SIMPLE_MODE` | Core      | true                             | .                                                                                                                  |        |
+| `WORKFLOW_CORE_GENERATE_ID_SIMPLE_MODE` | Core      | true                             | A flog that enable generating ID with `md5` algorithm.                                                             |        |
 | `WORKFLOW_LOG_DEBUG_MODE`               | Log       | true                             | A flag that enable logging with debug level mode.                                                                  |        |
 | `WORKFLOW_LOG_ENABLE_WRITE`             | Log       | true                             | A flag that enable logging object saving log to its destination.                                                   |        |
 | `WORKFLOW_APP_MAX_PROCESS`              | Schedule  | 2                                | The maximum process worker number that run in scheduler app module.                                                |        |

ddeutil_workflow-0.0.19.dist-info/RECORD

@@ -0,0 +1,21 @@
+ddeutil/workflow/__about__.py,sha256=GGHkQYD3vWi0C2DiWlU64oocfEx-Bn3LsXBorKFxtlM,28
+ddeutil/workflow/__cron.py,sha256=KUCSdx30juyX6IE6Dal8T_qSudOiaD02r1SRHFJp7IM,25778
+ddeutil/workflow/__init__.py,sha256=HA0tjGBXJItNPsAqvhnFUXU0fP0K6iMMfMtJ37tRwcw,1385
+ddeutil/workflow/__types.py,sha256=yizLXzjQpBt_WPaof2pIyncitJvYeksw4Q1zYJeuCLA,3707
+ddeutil/workflow/api.py,sha256=vUT2RVS9sF3hvY-IrzAEnahxwq4ZFYP0G3xfctHbNsw,4701
+ddeutil/workflow/cli.py,sha256=baHhvtI8snbHYHeThoX401Cd6SMB2boyyCbCtTrIl3E,3278
+ddeutil/workflow/conf.py,sha256=3xJPHIQcY4Q7rJoe0V8CUVHiEt_kww_bmr1f6MhcyCM,15420
+ddeutil/workflow/exceptions.py,sha256=Uf1-Tn8rAzj0aiVHSqo4fBqO80W0za7UFZgKv24E-tg,706
+ddeutil/workflow/job.py,sha256=Ww1zjviDCfTVUC_q7e3HHJwk3KXEFZxzGROQXoi_JS8,24349
+ddeutil/workflow/on.py,sha256=slaNJr2RWBEmAmEUcW0S99qD45ENUUgAGka5XoZ6Yag,7937
+ddeutil/workflow/repeat.py,sha256=s0azh-f5JQeow7kpxM8GKlqgAmKL7oU6St3L4Ggx4cY,4925
+ddeutil/workflow/route.py,sha256=JALwOH6xKu5rnII7DgA1Lbp_E5ehCoBbOW_eKqB_Olk,6753
+ddeutil/workflow/scheduler.py,sha256=0xE3bjIMe4eguo24rotOt6JfTy78tgst_qe7csSlt4k,50477
+ddeutil/workflow/stage.py,sha256=6Ng3RiCSrnQ-FUsRRcuG2ClMD6ifiQlgyBFi6tohfxI,25455
+ddeutil/workflow/utils.py,sha256=ETzixrfrXhacAm06agnvI1E8UZKjKCkKempJnW9KKes,25581
+ddeutil_workflow-0.0.19.dist-info/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
+ddeutil_workflow-0.0.19.dist-info/METADATA,sha256=zTUO4MZi08V0upHMA8xmH8q7ms5M7Eot0lflQyGeaXw,13597
+ddeutil_workflow-0.0.19.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
+ddeutil_workflow-0.0.19.dist-info/entry_points.txt,sha256=0BVOgO3LdUdXVZ-CiHHDKxzEk2c8J30jEwHeKn2YCWI,62
+ddeutil_workflow-0.0.19.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
+ddeutil_workflow-0.0.19.dist-info/RECORD,,

{ddeutil_workflow-0.0.17.dist-info → ddeutil_workflow-0.0.19.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.2.0)
+Generator: setuptools (75.3.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

ddeutil_workflow-0.0.17.dist-info/RECORD

@@ -1,21 +0,0 @@
-ddeutil/workflow/__about__.py,sha256=z3f1GAF3VbZK1m4FWAXXMsWplP_jSe-X-wVlshvlDWU,28
-ddeutil/workflow/__cron.py,sha256=ZiuV4ASkXvAyFJYxEb9PKiAFNYnUt4AJozu_kH3pI4U,25777
-ddeutil/workflow/__init__.py,sha256=RNKME4FPMAjqtrBR-IBwQVEKeoY5yBAiHYcZw0k9cI4,729
-ddeutil/workflow/__types.py,sha256=yizLXzjQpBt_WPaof2pIyncitJvYeksw4Q1zYJeuCLA,3707
-ddeutil/workflow/api.py,sha256=vUT2RVS9sF3hvY-IrzAEnahxwq4ZFYP0G3xfctHbNsw,4701
-ddeutil/workflow/cli.py,sha256=baHhvtI8snbHYHeThoX401Cd6SMB2boyyCbCtTrIl3E,3278
-ddeutil/workflow/conf.py,sha256=SV4GMtjUc-Bor9BPi0yOtTIsiZ0FImsoRbuJysUIE9w,15395
-ddeutil/workflow/exceptions.py,sha256=Uf1-Tn8rAzj0aiVHSqo4fBqO80W0za7UFZgKv24E-tg,706
-ddeutil/workflow/job.py,sha256=dW9NXR_bttDGLwelVi7qXXlLd96KX-TKG8xnHejA6u0,24041
-ddeutil/workflow/on.py,sha256=rneZB5HyFWTBWriGef999bovA3glQIK6LTgC996q9Gc,7334
-ddeutil/workflow/repeat.py,sha256=9uKku5uMcQgzY5fWyaJMwJ0wPFX0oTwmu7vXKdgB_ec,4923
-ddeutil/workflow/route.py,sha256=JALwOH6xKu5rnII7DgA1Lbp_E5ehCoBbOW_eKqB_Olk,6753
-ddeutil/workflow/scheduler.py,sha256=Oa6bZpphjlGp0mXdBuLMk1m6G-dezaBNQxQX-SB3WJ0,47032
-ddeutil/workflow/stage.py,sha256=fMv_oFkoqpfoewzPUMdl3-BQcrJ8SE53cF7es8yGxfs,25525
-ddeutil/workflow/utils.py,sha256=lpnqGGd_Rw7eZo2wDbZ-NZNItBooFooPjwM4_40Csh8,25152
-ddeutil_workflow-0.0.17.dist-info/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
-ddeutil_workflow-0.0.17.dist-info/METADATA,sha256=btmCr-yjy4gzhnZppfXjANfPH-3tKUJFGon2aOMUK30,13574
-ddeutil_workflow-0.0.17.dist-info/WHEEL,sha256=OVMc5UfuAQiSplgO0_WdW7vXVGAt9Hdd6qtN4HotdyA,91
-ddeutil_workflow-0.0.17.dist-info/entry_points.txt,sha256=0BVOgO3LdUdXVZ-CiHHDKxzEk2c8J30jEwHeKn2YCWI,62
-ddeutil_workflow-0.0.17.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
-ddeutil_workflow-0.0.17.dist-info/RECORD,,