ddeutil-workflow 0.0.55__py3-none-any.whl → 0.0.57__py3-none-any.whl

ddeutil/workflow/scheduler.py

@@ -3,7 +3,6 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
-# [x] Use dynamic config
 """The main schedule running is `schedule_runner` function that trigger the
 multiprocess of `schedule_control` function for listing schedules on the
 config by `Loader.finds(Schedule)`.
@@ -17,6 +16,11 @@ functions; `workflow_task`, and `workflow_monitor`.
     The `schedule_task` will run `task.release` method in threading object
 for multithreading strategy. This `release` method will run only one crontab
 value with the on field.
+
+Steps:
+    - Extract all schedule config on the conf path.
+    - Slice schedules to multiprocess
+    - Start running task.
 """
 from __future__ import annotations
 
@@ -34,7 +38,7 @@ from heapq import heappop, heappush
 from pathlib import Path
 from textwrap import dedent
 from threading import Thread
-from typing import Callable, Optional, TypedDict, Union
+from typing import Any, Callable, Optional, TypedDict, Union
 
 from pydantic import BaseModel, Field, ValidationInfo
 from pydantic.functional_validators import field_validator, model_validator
@@ -52,8 +56,8 @@ except ImportError:  # pragma: no cov
 
 from .__cron import CronRunner
 from .__types import DictData, TupleStr
-from .conf import Loader, SimLoad, dynamic
-from .cron import On
+from .conf import FileLoad, Loader, dynamic
+from .event import On
 from .exceptions import ScheduleException, WorkflowException
 from .logs import Audit, get_audit
 from .result import SUCCESS, Result
@@ -113,21 +117,15 @@ class ScheduleWorkflow(BaseModel):
     )
 
     @model_validator(mode="before")
-    def __prepare_before__(cls, values: DictData) -> DictData:
-        """Prepare incoming values before validating with model fields.
-
-        :rtype: DictData
-        """
-        # VALIDATE: Prepare a workflow name that should not include space.
-        if name := values.get("name"):
-            values["name"] = name.replace(" ", "_")
-
-        # VALIDATE: Add default the alias field with the name.
-        if not values.get("alias"):
-            values["alias"] = values.get("name")
-
-        cls.__bypass_on(values, extras=values.get("extras"))
-        return values
+    def __prepare_before__(cls, data: Any) -> Any:
+        """Prepare incoming values before validating with model fields."""
+        if isinstance(data, dict):
+            # VALIDATE: Add default the alias field with the name.
+            if "alias" not in data:
+                data["alias"] = data.get("name")
+
+            cls.__bypass_on(data, extras=data.get("extras"))
+        return data
 
     @classmethod
     def __bypass_on(
@@ -135,8 +133,10 @@ class ScheduleWorkflow(BaseModel):
     ) -> DictData:
         """Bypass and prepare the on data to loaded config data.
 
-        :param data: A data that want to validate for model initialization.
-        :param extras: An extra parameter that want to override core config.
+        :param data: (DictData) A data that want to validate for the model
+            initialization.
+        :param extras: (DictData) An extra parameter that want to override core
+            config values.
 
         :rtype: DictData
         """
@@ -151,7 +151,7 @@ class ScheduleWorkflow(BaseModel):
             # NOTE: Pass on value to Loader and keep on model object to on
             #   field.
             data["on"] = [
-                Loader(n, externals=extras).data if isinstance(n, str) else n
+                FileLoad(n, externals=extras).data if isinstance(n, str) else n
                 for n in on
             ]
 
@@ -162,6 +162,10 @@ class ScheduleWorkflow(BaseModel):
         """Validate the on fields should not contain duplicate values and if it
         contains every minute value, it should have only one on value.
 
+        :param value: (list[On]) A list of `On` object.
+        :param info: (ValidationInfo) An validation info object for getting an
+            extra parameter.
+
         :rtype: list[On]
         """
         set_ons: set[str] = {str(on.cronjob) for on in value}
@@ -191,23 +195,22 @@ class ScheduleWorkflow(BaseModel):
             This task creation need queue to tracking release date already
         mapped or not.
 
-        :param start_date: A start date that get from the workflow schedule.
-        :param queue: A mapping of name and list of datetime for queue.
+        :param start_date: (datetime) A start datetime that get from the
+            workflow schedule.
+        :param queue: (dict[str, ReleaseQueue]) A mapping of name and list of
+            datetime for queue.
 
         :rtype: list[WorkflowTask]
         :return: Return the list of WorkflowTask object from the specific
             input datetime that mapping with the on field.
         """
-        workflow_tasks: list[WorkflowTask] = []
-
-        # NOTE: Loading workflow model from the name of workflow.
         wf: Workflow = Workflow.from_conf(self.name, extras=self.extras)
         wf_queue: ReleaseQueue = queue[self.alias]
 
         # IMPORTANT: Create the default 'on' value if it does not pass the `on`
         #   field to the Schedule object.
         ons: list[On] = self.on or wf.on.copy()
-
+        workflow_tasks: list[WorkflowTask] = []
         for on in ons:
 
             # NOTE: Create CronRunner instance from the start_date param.
@@ -250,7 +253,7 @@ class Schedule(BaseModel):
     )
     workflows: list[ScheduleWorkflow] = Field(
         default_factory=list,
-        description="A list of ScheduleWorkflow models.",
+        description="A list of ScheduleWorkflow model.",
     )
 
     @field_validator("desc", mode="after")
@@ -267,6 +270,8 @@ class Schedule(BaseModel):
     def from_conf(
         cls,
         name: str,
+        *,
+        path: Optional[Path] = None,
         extras: DictData | None = None,
     ) -> Self:
         """Create Schedule instance from the Loader object that only receive
@@ -274,6 +279,7 @@ class Schedule(BaseModel):
         searching configuration data of this schedule model in conf path.
 
         :param name: (str) A schedule name that want to pass to Loader object.
+        :param path: (Path) An override config path.
         :param extras: An extra parameters that want to pass to Loader
             object.
 
@@ -281,55 +287,14 @@ class Schedule(BaseModel):
 
         :rtype: Self
         """
-        loader: Loader = Loader(name, externals=(extras or {}))
+        loader: Loader = FileLoad(name, path=path, extras=extras)
 
         # NOTE: Validate the config type match with current connection model
         if loader.type != cls.__name__:
             raise ValueError(f"Type {loader.type} does not match with {cls}")
 
         loader_data: DictData = copy.deepcopy(loader.data)
-
-        # NOTE: Add name to loader data
-        loader_data["name"] = name.replace(" ", "_")
-
-        if extras:
-            loader_data["extras"] = extras
-
-        return cls.model_validate(obj=loader_data)
-
-    @classmethod
-    def from_path(
-        cls,
-        name: str,
-        path: Path,
-        extras: DictData | None = None,
-    ) -> Self:
-        """Create Schedule instance from the SimLoad object that receive an
-        input schedule name and conf path. The loader object will use this
-        schedule name to searching configuration data of this schedule model
-        in conf path.
-
-        :param name: (str) A schedule name that want to pass to Loader object.
-        :param path: (Path) A config path that want to search.
-        :param extras: An external parameters that want to pass to Loader
-            object.
-
-        :raise ValueError: If the type does not match with current object.
-
-        :rtype: Self
-        """
-        loader: SimLoad = SimLoad(
-            name, conf_path=path, externals=(extras or {})
-        )
-
-        # NOTE: Validate the config type match with current connection model
-        if loader.type != cls.__name__:
-            raise ValueError(f"Type {loader.type} does not match with {cls}")
-
-        loader_data: DictData = copy.deepcopy(loader.data)
-
-        # NOTE: Add name to loader data
-        loader_data["name"] = name.replace(" ", "_")
+        loader_data["name"] = name
 
         if extras:
             loader_data["extras"] = extras
@@ -738,7 +703,7 @@ def schedule_control(
     :rtype: Result
     """
     audit: type[Audit] = audit or get_audit(extras=extras)
-    result: Result = Result().set_parent_run_id(parent_run_id)
+    result: Result = Result.construct_with_rs_or_id(parent_run_id=parent_run_id)
 
     # NOTE: Create the start and stop datetime.
     start_date: datetime = datetime.now(tz=dynamic("tz", extras=extras))