ddeutil-workflow 0.0.48__py3-none-any.whl → 0.0.49__py3-none-any.whl

ddeutil/workflow/reusables.py

@@ -91,7 +91,7 @@ def make_filter_registry(
     :rtype: dict[str, FilterRegistry]
     """
     rs: dict[str, FilterRegistry] = {}
-    for module in dynamic("regis_filter", f=registers):
+    for module in dynamic("registry_filter", f=registers):
         # NOTE: try to sequential import task functions
         try:
             importer = import_module(module)
@@ -343,7 +343,7 @@ def param2template(
     :returns: An any getter value from the params input.
     """
     registers: Optional[list[str]] = (
-        extras.get("regis_filter") if extras else None
+        extras.get("registry_filter") if extras else None
     )
     filters: dict[str, FilterRegistry] = filters or make_filter_registry(
         registers=registers
@@ -449,7 +449,7 @@ def make_registry(
     """
     rs: dict[str, Registry] = {}
     regis_calls: list[str] = dynamic(
-        "regis_call", f=registries
+        "registry_caller", f=registries
     )  # pragma: no cov
     regis_calls.extend(["ddeutil.vendors"])
 

ddeutil/workflow/scheduler.py

@@ -3,7 +3,7 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
-# [x] Use fix config
+# [x] Use dynamic config
 """The main schedule running is `schedule_runner` function that trigger the
 multiprocess of `schedule_control` function for listing schedules on the
 config by `Loader.finds(Schedule)`.
@@ -36,7 +36,7 @@ from textwrap import dedent
 from threading import Thread
 from typing import Callable, Optional, TypedDict, Union
 
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, ValidationInfo
 from pydantic.functional_validators import field_validator, model_validator
 from typing_extensions import Self
 
@@ -52,7 +52,7 @@ except ImportError:  # pragma: no cov
 
 from .__cron import CronRunner
 from .__types import DictData, TupleStr
-from .conf import Loader, SimLoad, config
+from .conf import Loader, SimLoad, dynamic
 from .cron import On
 from .exceptions import ScheduleException, WorkflowException
 from .logs import Audit, get_audit
@@ -89,6 +89,11 @@ class ScheduleWorkflow(BaseModel):
     not want to change on the workflow model.
     """
 
+    extras: DictData = Field(
+        default_factory=dict,
+        description="An extra parameters that want to override config values.",
+    )
+
     alias: Optional[str] = Field(
         default=None,
         description="An alias name of workflow that use for schedule model.",
@@ -121,14 +126,17 @@ class ScheduleWorkflow(BaseModel):
         if not values.get("alias"):
             values["alias"] = values.get("name")
 
-        cls.__bypass_on(values)
+        cls.__bypass_on(values, extras=values.get("extras"))
         return values
 
     @classmethod
-    def __bypass_on(cls, data: DictData) -> DictData:
+    def __bypass_on(
+        cls, data: DictData, *, extras: Optional[DictData] = None
+    ) -> DictData:
         """Bypass and prepare the on data to loaded config data.
 
         :param data: A data that want to validate for model initialization.
+        :param extras: An extra parameter that want to override core config.
 
         :rtype: DictData
         """
@@ -143,14 +151,14 @@ class ScheduleWorkflow(BaseModel):
             # NOTE: Pass on value to Loader and keep on model object to on
             #   field.
             data["on"] = [
-                Loader(n, externals={}).data if isinstance(n, str) else n
+                Loader(n, externals=extras).data if isinstance(n, str) else n
                 for n in on
             ]
 
         return data
 
     @field_validator("on", mode="after")
-    def __on_no_dup__(cls, value: list[On]) -> list[On]:
+    def __on_no_dup__(cls, value: list[On], info: ValidationInfo) -> list[On]:
         """Validate the on fields should not contain duplicate values and if it
         contains every minute value, it should have only one on value.
 
@@ -162,10 +170,12 @@ class ScheduleWorkflow(BaseModel):
                 "The on fields should not contain duplicate on value."
             )
 
-        if len(set_ons) > config.max_on_per_workflow:
+        extras: Optional[DictData] = info.data.get("extras")
+        if len(set_ons) > (
+            conf := dynamic("max_cron_per_workflow", extras=extras)
+        ):
             raise ValueError(
-                f"The number of the on should not more than "
-                f"{config.max_on_per_workflow} crontab."
+                f"The number of the on should not more than {conf} crontabs."
             )
 
         return value
@@ -174,8 +184,6 @@ class ScheduleWorkflow(BaseModel):
         self,
         start_date: datetime,
         queue: dict[str, ReleaseQueue],
-        *,
-        extras: DictData | None = None,
     ) -> list[WorkflowTask]:
         """Return the list of WorkflowTask object from the specific input
         datetime that mapping with the on field.
@@ -185,17 +193,15 @@ class ScheduleWorkflow(BaseModel):
 
         :param start_date: A start date that get from the workflow schedule.
         :param queue: A mapping of name and list of datetime for queue.
-        :param extras: An extra parameters that pass to the Loader object.
 
         :rtype: list[WorkflowTask]
         :return: Return the list of WorkflowTask object from the specific
             input datetime that mapping with the on field.
         """
         workflow_tasks: list[WorkflowTask] = []
-        extras: DictData = extras or {}
 
         # NOTE: Loading workflow model from the name of workflow.
-        wf: Workflow = Workflow.from_conf(self.name, extras=extras)
+        wf: Workflow = Workflow.from_conf(self.name, extras=self.extras)
         wf_queue: ReleaseQueue = queue[self.alias]
 
         # IMPORTANT: Create the default 'on' value if it does not pass the `on`
@@ -217,6 +223,7 @@ class ScheduleWorkflow(BaseModel):
                     workflow=wf,
                     runner=runner,
                     values=self.values,
+                    extras=self.extras,
                 ),
             )
 
@@ -232,7 +239,7 @@ class Schedule(BaseModel):
 
     extras: DictData = Field(
         default_factory=dict,
-        description="An extra override config values.",
+        description="An extra parameters that want to override config values.",
     )
 
     desc: Optional[str] = Field(
@@ -333,8 +340,6 @@ class Schedule(BaseModel):
         self,
         start_date: datetime,
         queue: dict[str, ReleaseQueue],
-        *,
-        extras: DictData | None = None,
     ) -> list[WorkflowTask]:
         """Return the list of WorkflowTask object from the specific input
         datetime that mapping with the on field from workflow schedule model.
@@ -342,8 +347,6 @@ class Schedule(BaseModel):
         :param start_date: A start date that get from the workflow schedule.
         :param queue: (dict[str, ReleaseQueue]) A mapping of name and list of
             datetime for queue.
-        :param extras: (DictData) An extra parameters that pass to the Loader
-            object.
 
         :rtype: list[WorkflowTask]
         :return: Return the list of WorkflowTask object from the specific
@@ -352,13 +355,13 @@ class Schedule(BaseModel):
         workflow_tasks: list[WorkflowTask] = []
 
         for workflow in self.workflows:
+            if self.extras:
+                workflow.extras = self.extras
 
             if workflow.alias not in queue:
                 queue[workflow.alias] = ReleaseQueue()
 
-            workflow_tasks.extend(
-                workflow.tasks(start_date, queue=queue, extras=extras)
-            )
+            workflow_tasks.extend(workflow.tasks(start_date, queue=queue))
 
         return workflow_tasks
 
@@ -366,24 +369,26 @@ class Schedule(BaseModel):
         self,
         *,
         stop: datetime | None = None,
-        extras: DictData | None = None,
         audit: type[Audit] | None = None,
         parent_run_id: str | None = None,
     ) -> Result:  # pragma: no cov
         """Pending this schedule tasks with the schedule package.
 
         :param stop: A datetime value that use to stop running schedule.
-        :param extras: An extra parameters that pass to Loader.
         :param audit: An audit class that use on the workflow task release for
             writing its release audit context.
         :param parent_run_id: A parent workflow running ID for this release.
         """
-        audit: type[Audit] = audit or get_audit()
+        audit: type[Audit] = audit or get_audit(extras=self.extras)
         result: Result = Result().set_parent_run_id(parent_run_id)
 
         # NOTE: Create the start and stop datetime.
-        start_date: datetime = datetime.now(tz=config.tz)
-        stop_date: datetime = stop or (start_date + config.stop_boundary_delta)
+        start_date: datetime = datetime.now(
+            tz=dynamic("tz", extras=self.extras)
+        )
+        stop_date: datetime = stop or (
+            start_date + dynamic("stop_boundary_delta", extras=self.extras)
+        )
 
         # IMPORTANT: Create main mapping of queue and thread object.
         queue: dict[str, ReleaseQueue] = {}
@@ -394,7 +399,7 @@ class Schedule(BaseModel):
         ) + timedelta(minutes=1)
 
         scheduler_pending(
-            tasks=self.tasks(start_date_waiting, queue=queue, extras=extras),
+            tasks=self.tasks(start_date_waiting, queue=queue),
             stop=stop_date,
             queue=queue,
             threads=threads,
@@ -469,6 +474,7 @@ def schedule_task(
     audit: type[Audit],
     *,
     parent_run_id: str | None = None,
+    extras: Optional[DictData] = None,
 ) -> ResultOrCancel:
     """Schedule task function that generate thread of workflow task release
     method in background. This function do the same logic as the workflow poke
@@ -483,12 +489,13 @@ def schedule_task(
     :param threads: A mapping of alias name and Thread object.
     :param audit: An audit class that want to make audit object.
     :param parent_run_id: A parent workflow running ID for this release.
+    :param extras: An extra parameter that want to override the core config.
 
     :rtype: ResultOrCancel
     """
     result: Result = Result().set_parent_run_id(parent_run_id)
-    current_date: datetime = datetime.now(tz=config.tz)
-    if current_date > stop.replace(tzinfo=config.tz):
+    current_date: datetime = datetime.now(tz=dynamic("tz", extras=extras))
+    if current_date > stop.replace(tzinfo=dynamic("tz", extras=extras)):
         return CancelJob
 
     # IMPORTANT:
@@ -569,7 +576,7 @@ def schedule_task(
 
         threads[thread_name] = {
             "thread": thread,
-            "start_date": datetime.now(tz=config.tz),
+            "start_date": datetime.now(tz=dynamic("tz", extras=extras)),
             "release_date": release.date,
         }
 
@@ -714,8 +721,8 @@ def scheduler_pending(
 def schedule_control(
     schedules: list[str],
     stop: datetime | None = None,
-    extras: DictData | None = None,
     *,
+    extras: DictData | None = None,
     audit: type[Audit] | None = None,
     parent_run_id: str | None = None,
 ) -> Result:  # pragma: no cov
@@ -732,12 +739,14 @@ def schedule_control(
 
     :rtype: Result
     """
-    audit: type[Audit] = audit or get_audit()
+    audit: type[Audit] = audit or get_audit(extras=extras)
     result: Result = Result().set_parent_run_id(parent_run_id)
 
     # NOTE: Create the start and stop datetime.
-    start_date: datetime = datetime.now(tz=config.tz)
-    stop_date: datetime = stop or (start_date + config.stop_boundary_delta)
+    start_date: datetime = datetime.now(tz=dynamic("tz", extras=extras))
+    stop_date: datetime = stop or (
+        start_date + dynamic("stop_boundary_delta", extras=extras)
+    )
 
     # IMPORTANT: Create main mapping of queue and thread object.
     queue: dict[str, ReleaseQueue] = {}
@@ -750,10 +759,10 @@ def schedule_control(
     tasks: list[WorkflowTask] = []
     for name in schedules:
         tasks.extend(
-            Schedule.from_conf(name, extras=extras).tasks(
-                start_date_waiting,
-                queue=queue,
-                extras=extras,
+            (
+                Schedule.from_conf(name, extras=extras).tasks(
+                    start_date_waiting, queue=queue
+                )
             ),
         )
 
@@ -771,6 +780,7 @@ def schedule_control(
 
 def schedule_runner(
     stop: datetime | None = None,
+    *,
     extras: DictData | None = None,
     excluded: list[str] | None = None,
 ) -> Result:  # pragma: no cov
@@ -803,7 +813,7 @@ def schedule_runner(
     context: DictData = {"schedules": [], "threads": []}
 
     with ProcessPoolExecutor(
-        max_workers=config.max_schedule_process,
+        max_workers=dynamic("max_schedule_process", extras=extras),
     ) as executor:
 
         futures: list[Future] = [
@@ -811,12 +821,12 @@ def schedule_runner(
                 schedule_control,
                 schedules=[load[0] for load in loader],
                 stop=stop,
-                extras=(extras or {}),
+                extras=extras,
                 parent_run_id=result.parent_run_id,
             )
             for loader in batch(
                 Loader.finds(Schedule, excluded=excluded),
-                n=config.max_schedule_per_process,
+                n=dynamic("max_schedule_per_process", extras=extras),
             )
         ]
 
@@ -831,4 +841,4 @@ def schedule_runner(
             context["schedule"].extend(rs.context.get("schedules", []))
             context["threads"].extend(rs.context.get("threads", []))
 
-    return result.catch(status=0, context=context)
+    return result.catch(status=SUCCESS, context=context)