ddeutil-workflow 0.0.47__py3-none-any.whl → 0.0.49__py3-none-any.whl

ddeutil/workflow/result.py

@@ -3,6 +3,7 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
+# [x] Use dynamic config
 """Result module. It is the data context transfer objects that use by all object
 in this package. This module provide Status enum object and Result dataclass.
 """
@@ -19,8 +20,9 @@ from pydantic.functional_validators import model_validator
 from typing_extensions import Self
 
 from .__types import DictData
-from .logs import TraceLog, get_dt_tznow, get_trace
-from .utils import default_gen_id, gen_id
+from .conf import dynamic
+from .logs import Trace, get_dt_tznow, get_trace
+from .utils import default_gen_id, gen_id, get_dt_now
 
 
 class Status(IntEnum):
@@ -66,8 +68,8 @@ class Result:
     parent_run_id: Optional[str] = field(default=None, compare=False)
     ts: datetime = field(default_factory=get_dt_tznow, compare=False)
 
-    trace: Optional[TraceLog] = field(default=None, compare=False, repr=False)
-    extras: DictData = field(default_factory=dict)
+    trace: Optional[Trace] = field(default=None, compare=False, repr=False)
+    extras: DictData = field(default_factory=dict, compare=False, repr=False)
 
     @classmethod
     def construct_with_rs_or_id(
@@ -111,17 +113,24 @@ class Result:
         :rtype: Self
         """
         if self.trace is None:  # pragma: no cov
-            self.trace: TraceLog = get_trace(self.run_id, self.parent_run_id)
+            self.trace: Trace = get_trace(
+                self.run_id,
+                parent_run_id=self.parent_run_id,
+                extras=self.extras,
+            )
         return self
 
     def set_parent_run_id(self, running_id: str) -> Self:
         """Set a parent running ID.
 
-        :param running_id: A running ID that want to update on this model.
+        :param running_id: (str) A running ID that want to update on this model.
+
         :rtype: Self
         """
         self.parent_run_id: str = running_id
-        self.trace: TraceLog = get_trace(self.run_id, running_id)
+        self.trace: Trace = get_trace(
+            self.run_id, parent_run_id=running_id, extras=self.extras
+        )
         return self
 
     def catch(
@@ -152,4 +161,6 @@ class Result:
 
         :rtype: float
         """
-        return (get_dt_tznow() - self.ts).total_seconds()
+        return (
+            get_dt_now(tz=dynamic("tz", extras=self.extras)) - self.ts
+        ).total_seconds()

ddeutil/workflow/reusables.py

@@ -91,7 +91,7 @@ def make_filter_registry(
     :rtype: dict[str, FilterRegistry]
     """
     rs: dict[str, FilterRegistry] = {}
-    for module in dynamic("regis_filter", f=registers):
+    for module in dynamic("registry_filter", f=registers):
         # NOTE: try to sequential import task functions
         try:
             importer = import_module(module)
@@ -343,7 +343,7 @@ def param2template(
     :returns: An any getter value from the params input.
     """
     registers: Optional[list[str]] = (
-        extras.get("regis_filter") if extras else None
+        extras.get("registry_filter") if extras else None
     )
     filters: dict[str, FilterRegistry] = filters or make_filter_registry(
         registers=registers
@@ -449,7 +449,7 @@ def make_registry(
     """
     rs: dict[str, Registry] = {}
     regis_calls: list[str] = dynamic(
-        "regis_call", f=registries
+        "registry_caller", f=registries
     )  # pragma: no cov
     regis_calls.extend(["ddeutil.vendors"])
 
@@ -534,8 +534,7 @@ def extract_call(
 
     call: CallSearchData = CallSearchData(**found.groupdict())
     rgt: dict[str, Registry] = make_registry(
-        submodule=f"{call.path}",
-        registries=registries,
+        submodule=f"{call.path}", registries=registries
     )
 
     if call.func not in rgt:

ddeutil/workflow/scheduler.py

@@ -3,7 +3,7 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
-# [x] Use fix config
+# [x] Use dynamic config
 """The main schedule running is `schedule_runner` function that trigger the
 multiprocess of `schedule_control` function for listing schedules on the
 config by `Loader.finds(Schedule)`.
@@ -36,7 +36,7 @@ from textwrap import dedent
 from threading import Thread
 from typing import Callable, Optional, TypedDict, Union
 
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, ValidationInfo
 from pydantic.functional_validators import field_validator, model_validator
 from typing_extensions import Self
 
@@ -52,7 +52,7 @@ except ImportError:  # pragma: no cov
 
 from .__cron import CronRunner
 from .__types import DictData, TupleStr
-from .conf import Loader, SimLoad, config, get_logger
+from .conf import Loader, SimLoad, dynamic
 from .cron import On
 from .exceptions import ScheduleException, WorkflowException
 from .logs import Audit, get_audit
@@ -62,7 +62,6 @@ from .workflow import Release, ReleaseQueue, Workflow, WorkflowTask
 
 P = ParamSpec("P")
 
-logger = get_logger("ddeutil.workflow")
 logging.getLogger("schedule").setLevel(logging.INFO)
 
 
@@ -90,6 +89,11 @@ class ScheduleWorkflow(BaseModel):
     not want to change on the workflow model.
     """
 
+    extras: DictData = Field(
+        default_factory=dict,
+        description="An extra parameters that want to override config values.",
+    )
+
     alias: Optional[str] = Field(
         default=None,
         description="An alias name of workflow that use for schedule model.",
@@ -122,14 +126,17 @@ class ScheduleWorkflow(BaseModel):
         if not values.get("alias"):
             values["alias"] = values.get("name")
 
-        cls.__bypass_on(values)
+        cls.__bypass_on(values, extras=values.get("extras"))
         return values
 
     @classmethod
-    def __bypass_on(cls, data: DictData) -> DictData:
+    def __bypass_on(
+        cls, data: DictData, *, extras: Optional[DictData] = None
+    ) -> DictData:
         """Bypass and prepare the on data to loaded config data.
 
         :param data: A data that want to validate for model initialization.
+        :param extras: An extra parameter that want to override core config.
 
         :rtype: DictData
         """
@@ -144,14 +151,14 @@ class ScheduleWorkflow(BaseModel):
             # NOTE: Pass on value to Loader and keep on model object to on
             #   field.
             data["on"] = [
-                Loader(n, externals={}).data if isinstance(n, str) else n
+                Loader(n, externals=extras).data if isinstance(n, str) else n
                 for n in on
             ]
 
         return data
 
     @field_validator("on", mode="after")
-    def __on_no_dup__(cls, value: list[On]) -> list[On]:
+    def __on_no_dup__(cls, value: list[On], info: ValidationInfo) -> list[On]:
         """Validate the on fields should not contain duplicate values and if it
         contains every minute value, it should have only one on value.
 
@@ -163,10 +170,12 @@ class ScheduleWorkflow(BaseModel):
                 "The on fields should not contain duplicate on value."
             )
 
-        if len(set_ons) > config.max_on_per_workflow:
+        extras: Optional[DictData] = info.data.get("extras")
+        if len(set_ons) > (
+            conf := dynamic("max_cron_per_workflow", extras=extras)
+        ):
             raise ValueError(
-                f"The number of the on should not more than "
-                f"{config.max_on_per_workflow} crontab."
+                f"The number of the on should not more than {conf} crontabs."
             )
 
         return value
@@ -175,8 +184,6 @@ class ScheduleWorkflow(BaseModel):
         self,
         start_date: datetime,
         queue: dict[str, ReleaseQueue],
-        *,
-        extras: DictData | None = None,
     ) -> list[WorkflowTask]:
         """Return the list of WorkflowTask object from the specific input
         datetime that mapping with the on field.
@@ -186,17 +193,15 @@ class ScheduleWorkflow(BaseModel):
 
         :param start_date: A start date that get from the workflow schedule.
         :param queue: A mapping of name and list of datetime for queue.
-        :param extras: An extra parameters that pass to the Loader object.
 
         :rtype: list[WorkflowTask]
         :return: Return the list of WorkflowTask object from the specific
             input datetime that mapping with the on field.
         """
         workflow_tasks: list[WorkflowTask] = []
-        extras: DictData = extras or {}
 
         # NOTE: Loading workflow model from the name of workflow.
-        wf: Workflow = Workflow.from_conf(self.name, extras=extras)
+        wf: Workflow = Workflow.from_conf(self.name, extras=self.extras)
         wf_queue: ReleaseQueue = queue[self.alias]
 
         # IMPORTANT: Create the default 'on' value if it does not pass the `on`
@@ -218,6 +223,7 @@ class ScheduleWorkflow(BaseModel):
                     workflow=wf,
                     runner=runner,
                     values=self.values,
+                    extras=self.extras,
                 ),
             )
 
@@ -231,6 +237,11 @@ class Schedule(BaseModel):
     enhance the workflow object by adding the alias and values fields.
     """
 
+    extras: DictData = Field(
+        default_factory=dict,
+        description="An extra parameters that want to override config values.",
+    )
+
     desc: Optional[str] = Field(
         default=None,
         description=(
@@ -281,6 +292,9 @@ class Schedule(BaseModel):
         # NOTE: Add name to loader data
         loader_data["name"] = name.replace(" ", "_")
 
+        if extras:
+            loader_data["extras"] = extras
+
         return cls.model_validate(obj=loader_data)
 
     @classmethod
@@ -288,7 +302,7 @@ class Schedule(BaseModel):
         cls,
         name: str,
         path: Path,
-        externals: DictData | None = None,
+        extras: DictData | None = None,
     ) -> Self:
         """Create Schedule instance from the SimLoad object that receive an
         input schedule name and conf path. The loader object will use this
@@ -297,7 +311,7 @@ class Schedule(BaseModel):
 
         :param name: (str) A schedule name that want to pass to Loader object.
         :param path: (Path) A config path that want to search.
-        :param externals: An external parameters that want to pass to Loader
+        :param extras: An external parameters that want to pass to Loader
             object.
 
         :raise ValueError: If the type does not match with current object.
@@ -305,7 +319,7 @@ class Schedule(BaseModel):
         :rtype: Self
         """
         loader: SimLoad = SimLoad(
-            name, conf_path=path, externals=(externals or {})
+            name, conf_path=path, externals=(extras or {})
         )
 
         # NOTE: Validate the config type match with current connection model
@@ -317,14 +331,15 @@ class Schedule(BaseModel):
         # NOTE: Add name to loader data
         loader_data["name"] = name.replace(" ", "_")
 
+        if extras:
+            loader_data["extras"] = extras
+
         return cls.model_validate(obj=loader_data)
 
     def tasks(
         self,
         start_date: datetime,
         queue: dict[str, ReleaseQueue],
-        *,
-        extras: DictData | None = None,
     ) -> list[WorkflowTask]:
         """Return the list of WorkflowTask object from the specific input
         datetime that mapping with the on field from workflow schedule model.
@@ -332,8 +347,6 @@ class Schedule(BaseModel):
         :param start_date: A start date that get from the workflow schedule.
         :param queue: (dict[str, ReleaseQueue]) A mapping of name and list of
             datetime for queue.
-        :param extras: (DictData) An extra parameters that pass to the Loader
-            object.
 
         :rtype: list[WorkflowTask]
         :return: Return the list of WorkflowTask object from the specific
@@ -342,13 +355,13 @@ class Schedule(BaseModel):
         workflow_tasks: list[WorkflowTask] = []
 
         for workflow in self.workflows:
+            if self.extras:
+                workflow.extras = self.extras
 
             if workflow.alias not in queue:
                 queue[workflow.alias] = ReleaseQueue()
 
-            workflow_tasks.extend(
-                workflow.tasks(start_date, queue=queue, extras=extras)
-            )
+            workflow_tasks.extend(workflow.tasks(start_date, queue=queue))
 
         return workflow_tasks
 
@@ -356,24 +369,26 @@ class Schedule(BaseModel):
         self,
         *,
         stop: datetime | None = None,
-        extras: DictData | None = None,
         audit: type[Audit] | None = None,
         parent_run_id: str | None = None,
     ) -> Result:  # pragma: no cov
         """Pending this schedule tasks with the schedule package.
 
         :param stop: A datetime value that use to stop running schedule.
-        :param extras: An extra parameters that pass to Loader.
         :param audit: An audit class that use on the workflow task release for
             writing its release audit context.
         :param parent_run_id: A parent workflow running ID for this release.
         """
-        audit: type[Audit] = audit or get_audit()
+        audit: type[Audit] = audit or get_audit(extras=self.extras)
         result: Result = Result().set_parent_run_id(parent_run_id)
 
         # NOTE: Create the start and stop datetime.
-        start_date: datetime = datetime.now(tz=config.tz)
-        stop_date: datetime = stop or (start_date + config.stop_boundary_delta)
+        start_date: datetime = datetime.now(
+            tz=dynamic("tz", extras=self.extras)
+        )
+        stop_date: datetime = stop or (
+            start_date + dynamic("stop_boundary_delta", extras=self.extras)
+        )
 
         # IMPORTANT: Create main mapping of queue and thread object.
         queue: dict[str, ReleaseQueue] = {}
@@ -384,7 +399,7 @@ class Schedule(BaseModel):
         ) + timedelta(minutes=1)
 
         scheduler_pending(
-            tasks=self.tasks(start_date_waiting, queue=queue, extras=extras),
+            tasks=self.tasks(start_date_waiting, queue=queue),
             stop=stop_date,
             queue=queue,
             threads=threads,
@@ -459,6 +474,7 @@ def schedule_task(
     audit: type[Audit],
     *,
     parent_run_id: str | None = None,
+    extras: Optional[DictData] = None,
 ) -> ResultOrCancel:
     """Schedule task function that generate thread of workflow task release
     method in background. This function do the same logic as the workflow poke
@@ -473,12 +489,13 @@ def schedule_task(
     :param threads: A mapping of alias name and Thread object.
     :param audit: An audit class that want to make audit object.
     :param parent_run_id: A parent workflow running ID for this release.
+    :param extras: An extra parameter that want to override the core config.
 
     :rtype: ResultOrCancel
     """
     result: Result = Result().set_parent_run_id(parent_run_id)
-    current_date: datetime = datetime.now(tz=config.tz)
-    if current_date > stop.replace(tzinfo=config.tz):
+    current_date: datetime = datetime.now(tz=dynamic("tz", extras=extras))
+    if current_date > stop.replace(tzinfo=dynamic("tz", extras=extras)):
         return CancelJob
 
     # IMPORTANT:
@@ -559,7 +576,7 @@ def schedule_task(
 
         threads[thread_name] = {
             "thread": thread,
-            "start_date": datetime.now(tz=config.tz),
+            "start_date": datetime.now(tz=dynamic("tz", extras=extras)),
             "release_date": release.date,
         }
 
@@ -704,8 +721,8 @@ def scheduler_pending(
 def schedule_control(
     schedules: list[str],
     stop: datetime | None = None,
-    extras: DictData | None = None,
     *,
+    extras: DictData | None = None,
     audit: type[Audit] | None = None,
     parent_run_id: str | None = None,
 ) -> Result:  # pragma: no cov
@@ -715,19 +732,21 @@ def schedule_control(
 
     :param schedules: A list of workflow names that want to schedule running.
     :param stop: A datetime value that use to stop running schedule.
-    :param extras: An extra parameters that pass to Loader.
+    :param extras: An extra parameters that want to override core config.
     :param audit: An audit class that use on the workflow task release for
         writing its release audit context.
     :param parent_run_id: A parent workflow running ID for this release.
 
     :rtype: Result
     """
-    audit: type[Audit] = audit or get_audit()
+    audit: type[Audit] = audit or get_audit(extras=extras)
     result: Result = Result().set_parent_run_id(parent_run_id)
 
     # NOTE: Create the start and stop datetime.
-    start_date: datetime = datetime.now(tz=config.tz)
-    stop_date: datetime = stop or (start_date + config.stop_boundary_delta)
+    start_date: datetime = datetime.now(tz=dynamic("tz", extras=extras))
+    stop_date: datetime = stop or (
+        start_date + dynamic("stop_boundary_delta", extras=extras)
+    )
 
     # IMPORTANT: Create main mapping of queue and thread object.
     queue: dict[str, ReleaseQueue] = {}
@@ -740,10 +759,10 @@ def schedule_control(
     tasks: list[WorkflowTask] = []
     for name in schedules:
         tasks.extend(
-            Schedule.from_conf(name, extras=extras).tasks(
-                start_date_waiting,
-                queue=queue,
-                extras=extras,
+            (
+                Schedule.from_conf(name, extras=extras).tasks(
+                    start_date_waiting, queue=queue
+                )
             ),
         )
 
@@ -761,7 +780,8 @@ def schedule_control(
 
 def schedule_runner(
     stop: datetime | None = None,
-    externals: DictData | None = None,
+    *,
+    extras: DictData | None = None,
     excluded: list[str] | None = None,
 ) -> Result:  # pragma: no cov
     """Schedule runner function it the multiprocess controller function for
@@ -770,7 +790,7 @@ def schedule_runner(
     path by `WORKFLOW_APP_MAX_SCHEDULE_PER_PROCESS` value.
 
     :param stop: A stop datetime object that force stop running scheduler.
-    :param externals:
+    :param extras: An extra parameter that want to override core config.
     :param excluded: A list of schedule name that want to exclude from finding.
 
         This function will get all workflows that include on value that was
@@ -793,7 +813,7 @@ def schedule_runner(
     context: DictData = {"schedules": [], "threads": []}
 
     with ProcessPoolExecutor(
-        max_workers=config.max_schedule_process,
+        max_workers=dynamic("max_schedule_process", extras=extras),
     ) as executor:
 
         futures: list[Future] = [
@@ -801,12 +821,12 @@ def schedule_runner(
                 schedule_control,
                 schedules=[load[0] for load in loader],
                 stop=stop,
-                externals=(externals or {}),
+                extras=extras,
                 parent_run_id=result.parent_run_id,
             )
             for loader in batch(
                 Loader.finds(Schedule, excluded=excluded),
-                n=config.max_schedule_per_process,
+                n=dynamic("max_schedule_per_process", extras=extras),
             )
         ]
 
@@ -821,4 +841,4 @@ def schedule_runner(
             context["schedule"].extend(rs.context.get("schedules", []))
             context["threads"].extend(rs.context.get("threads", []))
 
-    return result.catch(status=0, context=context)
+    return result.catch(status=SUCCESS, context=context)