ddeutil-workflow 0.0.31__tar.gz → 0.0.32__tar.gz

{ddeutil_workflow-0.0.31 → ddeutil_workflow-0.0.32}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: ddeutil-workflow
-Version: 0.0.31
+Version: 0.0.32
 Summary: Lightweight workflow orchestration
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT
@@ -110,7 +110,7 @@ flowchart LR
     E -.->|read| G
 ```
 
-> [!NOTE]
+> [!WARNING]
 > _Disclaimer_: I inspire the dynamic statement from the [**GitHub Action**](https://github.com/features/actions)
 > with `.yml` files and all configs file from several data orchestration framework
 > tools from my experience on Data Engineer. :grimacing:
@@ -201,7 +201,8 @@ result: Result = workflow.execute(
 )
 ```
 
-So, this package provide the `Schedule` template for this action.
+So, this package provide the `Schedule` template for this action, and you can dynamic
+pass the parameters for changing align with that running time by the `release` prefix.
 
 ```yaml
 schedule-run-local-wf:
@@ -219,6 +220,20 @@ schedule-run-local-wf:
           asat-dt: "${{ release.logical_date }}"
 ```
 
+The main method of the `Schedule` model that use to running is `pending`. If you
+do not pass the `stop` date on this method, it will use config with `WORKFLOW_APP_STOP_BOUNDARY_DELTA`
+key for generate this stop date.
+
+```python
+from ddeutil.workflow import Schedule
+
+(
+   Schedule
+   .from_loader("schedule-run-local-wf")
+   .pending(stop=None)
+)
+```
+
 ## :cookie: Configuration
 
 The main configuration that use to dynamic changing this workflow engine for your

{ddeutil_workflow-0.0.31 → ddeutil_workflow-0.0.32}/README.md

@@ -78,7 +78,7 @@ flowchart LR
     E -.->|read| G
 ```
 
-> [!NOTE]
+> [!WARNING]
 > _Disclaimer_: I inspire the dynamic statement from the [**GitHub Action**](https://github.com/features/actions)
 > with `.yml` files and all configs file from several data orchestration framework
 > tools from my experience on Data Engineer. :grimacing:
@@ -169,7 +169,8 @@ result: Result = workflow.execute(
 )
 ```
 
-So, this package provide the `Schedule` template for this action.
+So, this package provide the `Schedule` template for this action, and you can dynamic
+pass the parameters for changing align with that running time by the `release` prefix.
 
 ```yaml
 schedule-run-local-wf:
@@ -187,6 +188,20 @@ schedule-run-local-wf:
           asat-dt: "${{ release.logical_date }}"
 ```
 
+The main method of the `Schedule` model that use to running is `pending`. If you
+do not pass the `stop` date on this method, it will use config with `WORKFLOW_APP_STOP_BOUNDARY_DELTA`
+key for generate this stop date.
+
+```python
+from ddeutil.workflow import Schedule
+
+(
+   Schedule
+   .from_loader("schedule-run-local-wf")
+   .pending(stop=None)
+)
+```
+
 ## :cookie: Configuration
 
 The main configuration that use to dynamic changing this workflow engine for your

ddeutil_workflow-0.0.32/src/ddeutil/workflow/__about__.py

@@ -0,0 +1 @@
+__version__: str = "0.0.32"

{ddeutil_workflow-0.0.31 → ddeutil_workflow-0.0.32}/src/ddeutil/workflow/api/api.py

@@ -43,9 +43,7 @@ async def lifespan(a: FastAPI) -> AsyncIterator[State]:
     a.state.workflow_queue = {}
 
     yield {
-        "upper_queue": a.state.upper_queue,
-        "upper_result": a.state.upper_result,
-        # NOTE: Scheduler value should be contain a key of workflow workflow and
+        # NOTE: Scheduler value should be contained a key of workflow and
         #   list of datetime of queue and running.
         #
         #   ... {
@@ -88,7 +86,7 @@ if config.enable_route_workflow:
 
 # NOTE: Enable the schedule route.
 if config.enable_route_schedule:
-    from ..conf import FileLog
+    from ..conf import get_log
     from ..scheduler import schedule_task
     from .route import schedule_route
 
@@ -108,11 +106,11 @@ if config.enable_route_schedule:
                 stop=datetime.now(config.tz) + timedelta(minutes=1),
                 queue=app.state.workflow_queue,
                 threads=app.state.workflow_threads,
-                log=FileLog,
+                log=get_log(),
             )
 
     @schedule_route.on_event("startup")
-    @repeat_at(cron="*/5 * * * *")
+    @repeat_at(cron="*/5 * * * *", delay=10)
     def monitoring():
         logger.debug("[MONITOR]: Start monitoring threading.")
         snapshot_threads: list[str] = list(app.state.workflow_threads.keys())

{ddeutil_workflow-0.0.31 → ddeutil_workflow-0.0.32}/src/ddeutil/workflow/api/route.py

@@ -169,7 +169,7 @@ async def get_schedules(name: str):
     )
 
 
-@schedule_route.get(path="/deploy")
+@schedule_route.get(path="/deploy/")
 async def get_deploy_schedulers(request: Request):
     snapshot = copy.deepcopy(request.state.scheduler)
     return {"schedule": snapshot}
@@ -178,9 +178,9 @@ async def get_deploy_schedulers(request: Request):
 @schedule_route.get(path="/deploy/{name}")
 async def get_deploy_scheduler(request: Request, name: str):
     if name in request.state.scheduler:
-        sch = Schedule.from_loader(name)
+        schedule = Schedule.from_loader(name)
         getter: list[dict[str, dict[str, list[datetime]]]] = []
-        for workflow in sch.workflows:
+        for workflow in schedule.workflows:
             getter.append(
                 {
                     workflow.name: {
@@ -219,7 +219,7 @@ async def add_deploy_scheduler(request: Request, name: str):
         second=0, microsecond=0
     )
 
-    # NOTE: Create pair of workflow and on from schedule model.
+    # NOTE: Create a pair of workflow and on from schedule model.
     try:
         schedule: Schedule = Schedule.from_loader(name)
     except ValueError as err:

{ddeutil_workflow-0.0.31 → ddeutil_workflow-0.0.32}/src/ddeutil/workflow/job.py

@@ -7,6 +7,8 @@
 The job handle the lineage of stages and location of execution of stages that
 mean the job model able to define ``runs-on`` key that allow you to run this
 job.
+
+    This module include Strategy Model that use on the job strategy field.
 """
 from __future__ import annotations
 

{ddeutil_workflow-0.0.31 → ddeutil_workflow-0.0.32}/src/ddeutil/workflow/params.py

@@ -3,6 +3,9 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
+"""Param Model that use for parsing incoming parameters that pass to the
+Workflow and Schedule objects.
+"""
 from __future__ import annotations
 
 import decimal
@@ -70,7 +73,7 @@ class DefaultParam(BaseParam):
 
 
 # TODO: Not implement this parameter yet
-class DateParam(DefaultParam):
+class DateParam(DefaultParam):  # pragma: no cov
     """Date parameter."""
 
     type: Literal["date"] = "date"
@@ -156,7 +159,7 @@ class IntParam(DefaultParam):
 
 
 # TODO: Not implement this parameter yet
-class DecimalParam(DefaultParam):
+class DecimalParam(DefaultParam):  # pragma: no cov
     type: Literal["decimal"] = "decimal"
 
     def receive(self, value: float | None = None) -> decimal.Decimal: ...

{ddeutil_workflow-0.0.31 → ddeutil_workflow-0.0.32}/src/ddeutil/workflow/scheduler.py

@@ -275,37 +275,6 @@ class Schedule(BaseModel):
 
         return cls.model_validate(obj=loader_data)
 
-    @classmethod
-    def extract_tasks(
-        cls,
-        schedules: list[str],
-        start_date: datetime,
-        queue: dict[str, ReleaseQueue],
-        externals: DictData | None = None,
-    ) -> list[WorkflowTask]:
-        """Return the list of WorkflowTask object from all schedule object that
-        include in an input schedules argument.
-
-        :param schedules: A list of schedule name that will use `from_loader`
-            method.
-        :param start_date: A start date that get from the workflow schedule.
-        :param queue: A mapping of name and list of datetime for queue.
-        :param externals: An external parameters that pass to the Loader object.
-
-        :rtype: list[WorkflowTask]
-        """
-        tasks: list[WorkflowTask] = []
-        for name in schedules:
-            schedule: Schedule = Schedule.from_loader(name, externals=externals)
-            tasks.extend(
-                schedule.tasks(
-                    start_date,
-                    queue=queue,
-                    externals=externals,
-                ),
-            )
-        return tasks
-
     def tasks(
         self,
         start_date: datetime,
@@ -339,6 +308,99 @@ class Schedule(BaseModel):
 
         return workflow_tasks
 
+    def pending(
+        self,
+        *,
+        stop: datetime | None = None,
+        externals: DictData | None = None,
+        log: type[Log] | None = None,
+    ) -> None:  # pragma: no cov
+        """Pending this schedule tasks with the schedule package.
+
+        :param stop: A datetime value that use to stop running schedule.
+        :param externals: An external parameters that pass to Loader.
+        :param log: A log class that use on the workflow task release for
+            writing its release log context.
+        """
+        try:
+            from schedule import Scheduler
+        except ImportError:
+            raise ImportError(
+                "Should install schedule package before use this method."
+            ) from None
+
+        # NOTE: Get default logging.
+        log: type[Log] = log or get_log()
+        scheduler: Scheduler = Scheduler()
+
+        # NOTE: Create the start and stop datetime.
+        start_date: datetime = datetime.now(tz=config.tz)
+        stop_date: datetime = stop or (start_date + config.stop_boundary_delta)
+
+        # IMPORTANT: Create main mapping of queue and thread object.
+        queue: dict[str, ReleaseQueue] = {}
+        threads: ReleaseThreads = {}
+
+        start_date_waiting: datetime = start_date.replace(
+            second=0, microsecond=0
+        ) + timedelta(minutes=1)
+
+        # NOTE: This schedule job will start every minute at :02 seconds.
+        (
+            scheduler.every(1)
+            .minutes.at(":02")
+            .do(
+                schedule_task,
+                tasks=self.tasks(
+                    start_date_waiting, queue=queue, externals=externals
+                ),
+                stop=stop_date,
+                queue=queue,
+                threads=threads,
+                log=log,
+            )
+            .tag("control")
+        )
+
+        # NOTE: Checking zombie task with schedule job will start every 5 minute at
+        #   :10 seconds.
+        (
+            scheduler.every(5)
+            .minutes.at(":10")
+            .do(
+                monitor,
+                threads=threads,
+            )
+            .tag("monitor")
+        )
+
+        # NOTE: Start running schedule
+        logger.info(
+            f"[SCHEDULE]: Schedule with stopper: {stop_date:%Y-%m-%d %H:%M:%S}"
+        )
+
+        while True:
+            scheduler.run_pending()
+            time.sleep(1)
+
+            # NOTE: Break the scheduler when the control job does not exist.
+            if not scheduler.get_jobs("control"):
+                scheduler.clear("monitor")
+
+                while len(threads) > 0:
+                    logger.warning(
+                        "[SCHEDULE]: Waiting schedule release thread that still "
+                        "running in background."
+                    )
+                    delay(10)
+                    monitor(threads)
+
+                break
+
+        logger.warning(
+            f"[SCHEDULE]: Queue: {[list(queue[wf].queue) for wf in queue]}"
+        )
+
 
 ResultOrCancelJob = Union[type[CancelJob], Result]
 ReturnCancelJob = Callable[P, ResultOrCancelJob]
@@ -394,7 +456,7 @@ def schedule_task(
     method in background. This function do the same logic as the workflow poke
     method, but it runs with map of schedules and the on values.
 
-        This schedule task start runs every minute at ':02' second and it does
+        This schedule task start runs every minute at ':02' second, and it does
     not allow you to run with offset time.
 
     :param tasks: A list of WorkflowTask object.
@@ -414,15 +476,16 @@ def schedule_task(
     #   function. It will deplicate running with different schedule value
     #   because I use current time in this condition.
     #
-    #       For example, if a workflow A queue has '00:02:00' time that
-    #   should to run and its schedule has '*/2 * * * *' and '*/35 * * * *'.
-    #   This condition will release with 2 threading job.
+    #       For example, if a queue has a time release be '00:02:00' that should
+    #   to run and its schedule has '*/2 * * * *' and '*/35 * * * *'.
+    #   This condition make this function create 2 threading tasks.
     #
-    #   '00:02:00'  --> '*/2 * * * *'   --> running
-    #               --> '*/35 * * * *'  --> skip
+    #       '00:02:00'  --> '*/2 * * * *'   --> run
+    #                   --> '*/35 * * * *'  --> skip
     #
     for task in tasks:
 
+        # NOTE: Get the ReleaseQueue with an alias of the WorkflowTask.
         q: ReleaseQueue = queue[task.alias]
 
         # NOTE: Start adding queue and move the runner date in the WorkflowTask.
@@ -548,15 +611,24 @@ def schedule_control(
         second=0, microsecond=0
     ) + timedelta(minutes=1)
 
+    tasks: list[WorkflowTask] = []
+    for name in schedules:
+        schedule: Schedule = Schedule.from_loader(name, externals=externals)
+        tasks.extend(
+            schedule.tasks(
+                start_date_waiting,
+                queue=queue,
+                externals=externals,
+            ),
+        )
+
     # NOTE: This schedule job will start every minute at :02 seconds.
     (
         scheduler.every(1)
         .minutes.at(":02")
         .do(
             schedule_task,
-            tasks=Schedule.extract_tasks(
-                schedules, start_date_waiting, queue, externals=externals
-            ),
+            tasks=tasks,
             stop=stop_date,
             queue=queue,
             threads=threads,
@@ -596,7 +668,7 @@ def schedule_control(
                     "[SCHEDULE]: Waiting schedule release thread that still "
                     "running in background."
                 )
-                delay(15)
+                delay(10)
                 monitor(threads)
 
             break

{ddeutil_workflow-0.0.31 → ddeutil_workflow-0.0.32}/src/ddeutil/workflow/stage.py

@@ -668,12 +668,12 @@ Stage = Union[
 
 
 # TODO: Not implement this stages yet
-class ParallelStage(BaseModel):
+class ParallelStage(BaseModel):  # pragma: no cov
     parallel: list[Stage]
     max_parallel_core: int = Field(default=2)
 
 
 # TODO: Not implement this stages yet
-class ForEachStage(BaseModel):
+class ForEachStage(BaseModel):  # pragma: no cov
     foreach: list[str]
     stages: list[Stage]

{ddeutil_workflow-0.0.31 → ddeutil_workflow-0.0.32}/src/ddeutil/workflow/templates.py

@@ -79,7 +79,7 @@ def custom_filter(name: str) -> Callable[P, FilterFunc]:
 def make_filter_registry() -> dict[str, FilterRegistry]:
     """Return registries of all functions that able to called with task.
 
-    :rtype: dict[str, Registry]
+    :rtype: dict[str, FilterRegistry]
     """
     rs: dict[str, FilterRegistry] = {}
     for module in config.regis_filter:
@@ -108,6 +108,8 @@ def get_args_const(
 ) -> tuple[str, list[Constant], dict[str, Constant]]:
     """Get arguments and keyword-arguments from function calling string.
 
+    :param expr: An expr string value.
+
     :rtype: tuple[str, list[Constant], dict[str, Constant]]
     """
     try:
@@ -150,6 +152,11 @@ def get_args_from_filter(
 ) -> tuple[str, FilterRegistry, list[Any], dict[Any, Any]]:  # pragma: no cov
     """Get arguments and keyword-arguments from filter function calling string.
     and validate it with the filter functions mapping dict.
+
+    :param ft:
+    :param filters:
+
+    :rtype: tuple[str, FilterRegistry, list[Any], dict[Any, Any]]
     """
     func_name, _args, _kwargs = get_args_const(ft)
     args: list[Any] = [arg.value for arg in _args]
@@ -243,7 +250,7 @@ def str2template(
     params: DictData,
     *,
     filters: dict[str, FilterRegistry] | None = None,
-) -> Any:
+) -> str:
     """(Sub-function) Pass param to template string that can search by
     ``RE_CALLER`` regular expression.
 
@@ -255,6 +262,8 @@ def str2template(
     :param params: A parameter value that getting with matched regular
         expression.
     :param filters:
+
+    :rtype: str
     """
     filters: dict[str, FilterRegistry] = filters or make_filter_registry()
 
@@ -295,7 +304,7 @@ def str2template(
     return search_env_replace(value)
 
 
-def param2template(value: Any, params: DictData) -> Any:
+def param2template(value: T, params: DictData) -> T:
     """Pass param to template string that can search by ``RE_CALLER`` regular
     expression.
 
@@ -303,7 +312,7 @@ def param2template(value: Any, params: DictData) -> Any:
     :param params: A parameter value that getting with matched regular
         expression.
 
-    :rtype: Any
+    :rtype: T
     :returns: An any getter value from the params input.
     """
     filters: dict[str, FilterRegistry] = make_filter_registry()

{ddeutil_workflow-0.0.31 → ddeutil_workflow-0.0.32}/src/ddeutil_workflow.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: ddeutil-workflow
-Version: 0.0.31
+Version: 0.0.32
 Summary: Lightweight workflow orchestration
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT
@@ -110,7 +110,7 @@ flowchart LR
     E -.->|read| G
 ```
 
-> [!NOTE]
+> [!WARNING]
 > _Disclaimer_: I inspire the dynamic statement from the [**GitHub Action**](https://github.com/features/actions)
 > with `.yml` files and all configs file from several data orchestration framework
 > tools from my experience on Data Engineer. :grimacing:
@@ -201,7 +201,8 @@ result: Result = workflow.execute(
 )
 ```
 
-So, this package provide the `Schedule` template for this action.
+So, this package provide the `Schedule` template for this action, and you can dynamic
+pass the parameters for changing align with that running time by the `release` prefix.
 
 ```yaml
 schedule-run-local-wf:
@@ -219,6 +220,20 @@ schedule-run-local-wf:
           asat-dt: "${{ release.logical_date }}"
 ```
 
+The main method of the `Schedule` model that use to running is `pending`. If you
+do not pass the `stop` date on this method, it will use config with `WORKFLOW_APP_STOP_BOUNDARY_DELTA`
+key for generate this stop date.
+
+```python
+from ddeutil.workflow import Schedule
+
+(
+   Schedule
+   .from_loader("schedule-run-local-wf")
+   .pending(stop=None)
+)
+```
+
 ## :cookie: Configuration
 
 The main configuration that use to dynamic changing this workflow engine for your

{ddeutil_workflow-0.0.31 → ddeutil_workflow-0.0.32}/src/ddeutil_workflow.egg-info/SOURCES.txt

@@ -42,6 +42,7 @@ tests/test_release_queue.py
 tests/test_result.py
 tests/test_schedule.py
 tests/test_schedule_control.py
+tests/test_schedule_pending.py
 tests/test_schedule_tasks.py
 tests/test_schedule_workflow.py
 tests/test_stage.py

ddeutil_workflow-0.0.32/tests/test_schedule_pending.py

@@ -0,0 +1,13 @@
+from datetime import timedelta
+from unittest import mock
+
+import pytest
+from ddeutil.workflow.conf import Config
+from ddeutil.workflow.scheduler import Schedule
+
+
+@pytest.mark.schedule
+@mock.patch.object(Config, "stop_boundary_delta", timedelta(minutes=1))
+@mock.patch.object(Config, "enable_write_log", False)
+def test_schedule_pending():
+    Schedule.from_loader("schedule-every-minute-wf").pending()

ddeutil_workflow-0.0.31/src/ddeutil/workflow/__about__.py

@@ -1 +0,0 @@
-__version__: str = "0.0.31"