ddeutil-workflow 0.0.30__py3-none-any.whl → 0.0.32__py3-none-any.whl

ddeutil/workflow/workflow.py

@@ -29,7 +29,8 @@ from concurrent.futures import (
 )
 from dataclasses import field
 from datetime import datetime, timedelta
-from functools import total_ordering
+from enum import Enum
+from functools import partial, total_ordering
 from heapq import heappop, heappush
 from queue import Queue
 from textwrap import dedent
@@ -53,21 +54,33 @@ from .utils import (
     cut_id,
     gen_id,
     get_dt_now,
-    wait_a_minute,
+    reach_next_minute,
+    wait_to_next_minute,
 )
 
 logger = get_logger("ddeutil.workflow")
 
 __all__: TupleStr = (
-    "Workflow",
     "Release",
     "ReleaseQueue",
+    "ReleaseType",
+    "Workflow",
     "WorkflowTask",
 )
 
 
+class ReleaseType(str, Enum):
+    """Release Type Enum support the type field on the Release dataclass."""
+
+    DEFAULT: str = "manual"
+    TASK: str = "task"
+    POKE: str = "poking"
+
+
 @total_ordering
-@dataclass(config=ConfigDict(arbitrary_types_allowed=True))
+@dataclass(
+    config=ConfigDict(arbitrary_types_allowed=True, use_enum_values=True)
+)
 class Release:
     """Release Pydantic dataclass object that use for represent
     the release data that use with the `workflow.release` method."""
@@ -76,7 +89,7 @@ class Release:
     offset: float
     end_date: datetime
     runner: CronRunner
-    type: str
+    type: ReleaseType = field(default=ReleaseType.DEFAULT)
 
     def __repr__(self) -> str:
         return repr(f"{self.date:%Y-%m-%d %H:%M:%S}")
@@ -94,13 +107,16 @@ class Release:
         """
         if isinstance(dt, str):
             dt: datetime = datetime.fromisoformat(dt)
+        elif not isinstance(dt, datetime):
+            raise TypeError(
+                "The `from_dt` need argument type be str or datetime only."
+            )
 
         return cls(
             date=dt,
             offset=0,
             end_date=dt + timedelta(days=1),
             runner=CronJob("* * * * *").schedule(dt.replace(tzinfo=config.tz)),
-            type="manual",
         )
 
     def __eq__(self, other: Release | datetime) -> bool:
@@ -174,13 +190,6 @@ class ReleaseQueue:
 
         :rtype: bool
         """
-        # NOTE: Old logic to peeking the first release from waiting queue.
-        #
-        # first_value: Release = heappop(self.queue)
-        # heappush(self.queue, first_value)
-        #
-        # return first_value
-        #
         return self.queue[0]
 
     def check_queue(self, value: Release | datetime) -> bool:
@@ -477,7 +486,7 @@ class Workflow(BaseModel):
         *,
         run_id: str | None = None,
         log: type[Log] = None,
-        queue: ReleaseQueue | list[datetime] | list[Release] | None = None,
+        queue: ReleaseQueue | None = None,
         override_log_name: str | None = None,
     ) -> Result:
         """Release the workflow execution with overriding parameter with the
@@ -487,7 +496,7 @@ class Workflow(BaseModel):
             This method allow workflow use log object to save the execution
         result to log destination like file log to the local `/logs` directory.
 
-        :Steps:
+        Steps:
             - Initialize ReleaseQueue and Release if they do not pass.
             - Create release data for pass to parameter templating function.
             - Execute this workflow with mapping release data to its parameters.
@@ -497,7 +506,7 @@ class Workflow(BaseModel):
 
         :param release: A release datetime or Release object.
         :param params: A workflow parameter that pass to execute method.
-        :param queue: A list of release time that already queue.
+        :param queue: A ReleaseQueue that use for mark complete.
         :param run_id: A workflow running ID for this release.
         :param log: A log class that want to save the execution result.
         :param queue: A ReleaseQueue object.
@@ -510,13 +519,14 @@ class Workflow(BaseModel):
         name: str = override_log_name or self.name
         run_id: str = run_id or gen_id(name, unique=True)
         rs_release: Result = Result(run_id=run_id)
-        rs_release_type: str = "release"
 
-        # VALIDATE: Change queue value to ReleaseQueue object.
-        if queue is None or isinstance(queue, list):
-            queue: ReleaseQueue = ReleaseQueue.from_list(queue)
+        if queue is not None and not isinstance(queue, ReleaseQueue):
+            raise TypeError(
+                "The queue argument should be ReleaseQueue object only."
+            )
 
         # VALIDATE: Change release value to Release object.
+        rs_release_type: str = "release"
         if isinstance(release, datetime):
             rs_release_type: str = "datetime"
             release: Release = Release.from_dt(release)
@@ -548,24 +558,26 @@ class Workflow(BaseModel):
         )
 
         rs.set_parent_run_id(run_id)
-        rs_log: Log = log.model_validate(
-            {
-                "name": name,
-                "release": release.date,
-                "type": release.type,
-                "context": rs.context,
-                "parent_run_id": rs.parent_run_id,
-                "run_id": rs.run_id,
-            }
-        )
 
         # NOTE: Saving execution result to destination of the input log object.
         logger.debug(f"({cut_id(run_id)}) [LOG]: Writing log: {name!r}.")
-        rs_log.save(excluded=None)
+        (
+            log.model_validate(
+                {
+                    "name": name,
+                    "release": release.date,
+                    "type": release.type,
+                    "context": rs.context,
+                    "parent_run_id": rs.parent_run_id,
+                    "run_id": rs.run_id,
+                }
+            ).save(excluded=None)
+        )
 
         # NOTE: Remove this release from running.
-        queue.remove_running(release)
-        queue.mark_complete(release)
+        if queue is not None:
+            queue.remove_running(release)
+            queue.mark_complete(release)
 
         # NOTE: Remove the params key from the result context for deduplicate.
         context: dict[str, Any] = rs.context
@@ -594,8 +606,16 @@ class Workflow(BaseModel):
         *,
         force_run: bool = False,
     ) -> ReleaseQueue:
-        """Generate queue of datetime from the cron runner that initialize from
-        the on field. with offset value.
+        """Generate Release from all on values from the on field and store them
+        to the ReleaseQueue object.
+
+        Steps:
+            - For-loop all the on value in the on field.
+            - Create Release object from the current date that not reach the end
+              date.
+            - Check this release do not store on the release queue object.
+              Generate the next date if it exists.
+            - Push this release to the release queue
 
         :param offset: An offset in second unit for time travel.
         :param end_date: An end datetime object.
@@ -621,7 +641,7 @@ class Workflow(BaseModel):
                 offset=offset,
                 end_date=end_date,
                 runner=runner,
-                type="poking",
+                type=ReleaseType.POKE,
             )
 
             while queue.check_queue(workflow_release) or (
@@ -633,7 +653,7 @@ class Workflow(BaseModel):
                     offset=offset,
                     end_date=end_date,
                     runner=runner,
-                    type="poking",
+                    type=ReleaseType.POKE,
                 )
 
             if runner.date > end_date:
@@ -662,6 +682,9 @@ class Workflow(BaseModel):
             This method will observe its schedule that nearing to run with the
         ``self.release()`` method.
 
+            The limitation of this method is not allow run a date that less
+        than the current date.
+
         :param start_date: A start datetime object.
         :param params: A parameters that want to pass to the release method.
         :param run_id: A workflow running ID for this poke.
@@ -678,6 +701,12 @@ class Workflow(BaseModel):
         log: type[Log] = log or get_log()
         run_id: str = run_id or gen_id(self.name, unique=True)
 
+        # VALIDATE: Check the periods value should gather than 0.
+        if periods <= 0:
+            raise WorkflowException(
+                "The period of poking should be int and grater or equal than 1."
+            )
+
         # NOTE: If this workflow does not set the on schedule, it will return
         #   empty result.
         if len(self.on) == 0:
@@ -687,23 +716,25 @@ class Workflow(BaseModel):
             )
             return []
 
-        if periods <= 0:
-            raise WorkflowException(
-                "The period of poking should be int and grater or equal than 1."
-            )
+        # NOTE: Create the current date that change microsecond to 0
+        current_date: datetime = datetime.now(tz=config.tz).replace(
+            microsecond=0
+        )
 
         # NOTE: Create start_date and offset variables.
-        current_date: datetime = datetime.now(tz=config.tz)
-
         if start_date and start_date <= current_date:
-            start_date = start_date.replace(tzinfo=config.tz)
+            start_date = start_date.replace(tzinfo=config.tz).replace(
+                microsecond=0
+            )
             offset: float = (current_date - start_date).total_seconds()
         else:
+            # NOTE: Force change start date if it gathers than the current date,
+            #   or it does not pass to this method.
             start_date: datetime = current_date
             offset: float = 0
 
-        # NOTE: End date is using to stop generate queue with an input periods
-        #   value.
+        # NOTE: The end date is using to stop generate queue with an input
+        #   periods value.
         end_date: datetime = start_date + timedelta(minutes=periods)
 
         logger.info(
@@ -715,17 +746,17 @@ class Workflow(BaseModel):
         results: list[Result] = []
 
         # NOTE: Create empty ReleaseQueue object.
-        wf_queue: ReleaseQueue = ReleaseQueue()
+        q: ReleaseQueue = ReleaseQueue()
 
-        # NOTE: Make queue to the workflow queue object.
-        self.queue(
-            offset,
-            end_date=end_date,
-            queue=wf_queue,
-            log=log,
-            force_run=force_run,
+        # NOTE: Create reusable partial function and add Release to the release
+        #   queue object.
+        partial_queue = partial(
+            self.queue, offset, end_date, log=log, force_run=force_run
         )
-        if not wf_queue.is_queued:
+        partial_queue(q)
+
+        # NOTE: Return the empty result if it does not have any Release.
+        if not q.is_queued:
             logger.info(
                 f"({cut_id(run_id)}) [POKING]: {self.name!r} does not have "
                 f"any queue."
@@ -741,34 +772,27 @@ class Workflow(BaseModel):
 
             futures: list[Future] = []
 
-            while wf_queue.is_queued:
+            while q.is_queued:
 
-                # NOTE: Pop the latest Release object from queue.
-                release: Release = heappop(wf_queue.queue)
+                # NOTE: Pop the latest Release object from the release queue.
+                release: Release = heappop(q.queue)
 
-                if (
-                    release.date - get_dt_now(tz=config.tz, offset=offset)
-                ).total_seconds() > 60:
+                if reach_next_minute(release.date, tz=config.tz, offset=offset):
                     logger.debug(
-                        f"({cut_id(run_id)}) [POKING]: Wait because the latest "
-                        f"release has diff time more than 60 seconds ..."
+                        f"({cut_id(run_id)}) [POKING]: The latest release, "
+                        f"{release.date:%Y-%m-%d %H:%M:%S}, is not able to run "
+                        f"on this minute"
                     )
-                    heappush(wf_queue.queue, release)
-                    wait_a_minute(get_dt_now(tz=config.tz, offset=offset))
+                    heappush(q.queue, release)
+                    wait_to_next_minute(get_dt_now(tz=config.tz, offset=offset))
 
                     # WARNING: I already call queue poking again because issue
                     #   about the every minute crontab.
-                    self.queue(
-                        offset,
-                        end_date,
-                        queue=wf_queue,
-                        log=log,
-                        force_run=force_run,
-                    )
+                    partial_queue(q)
                     continue
 
                 # NOTE: Push the latest Release to the running queue.
-                heappush(wf_queue.running, release)
+                heappush(q.running, release)
 
                 futures.append(
                     executor.submit(
@@ -776,17 +800,11 @@ class Workflow(BaseModel):
                         release=release,
                         params=params,
                         log=log,
-                        queue=wf_queue,
+                        queue=q,
                     )
                 )
 
-                self.queue(
-                    offset,
-                    end_date,
-                    queue=wf_queue,
-                    log=log,
-                    force_run=force_run,
-                )
+                partial_queue(q)
 
             # WARNING: This poking method does not allow to use fail-fast
             #   logic to catching parallel execution result.
@@ -1148,20 +1166,31 @@ class WorkflowTask:
         release: datetime | Release | None = None,
         run_id: str | None = None,
         log: type[Log] = None,
-        queue: ReleaseQueue | list[datetime] | list[Release] | None = None,
+        queue: ReleaseQueue | None = None,
     ) -> Result:
         """Release the workflow task data.
 
         :param release: A release datetime or Release object.
         :param run_id: A workflow running ID for this release.
         :param log: A log class that want to save the execution result.
-        :param queue: A ReleaseQueue object.
+        :param queue: A ReleaseQueue object that use to mark complete.
 
         :rtype: Result
         """
         log: type[Log] = log or get_log()
 
         if release is None:
+
+            if queue is None:
+                raise ValueError(
+                    "If pass None release value, you should to pass the queue"
+                    "for generate this release."
+                )
+            elif not isinstance(queue, ReleaseQueue):
+                raise TypeError(
+                    "The queue argument should be ReleaseQueue object only."
+                )
+
             if queue.check_queue(self.runner.date):
                 release = self.runner.next
 
@@ -1170,6 +1199,7 @@ class WorkflowTask:
             else:
                 release = self.runner.date
 
+        # NOTE: Call the workflow release method.
         return self.workflow.release(
             release=release,
             params=self.values,
@@ -1186,8 +1216,9 @@ class WorkflowTask:
         log: type[Log],
         *,
         force_run: bool = False,
-    ):
-        """Generate Release to ReleaseQueue object.
+    ) -> ReleaseQueue:
+        """Generate Release from the runner field and store it to the
+        ReleaseQueue object.
 
         :param end_date: An end datetime object.
         :param queue: A workflow queue object.
@@ -1205,7 +1236,7 @@ class WorkflowTask:
             offset=0,
             end_date=end_date,
             runner=self.runner,
-            type="task",
+            type=ReleaseType.TASK,
         )
 
         while queue.check_queue(workflow_release) or (
@@ -1217,7 +1248,7 @@ class WorkflowTask:
                 offset=0,
                 end_date=end_date,
                 runner=self.runner,
-                type="task",
+                type=ReleaseType.TASK,
             )
 
         if self.runner.date > end_date:

{ddeutil_workflow-0.0.30.dist-info → ddeutil_workflow-0.0.32.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: ddeutil-workflow
-Version: 0.0.30
+Version: 0.0.32
 Summary: Lightweight workflow orchestration
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT
@@ -110,7 +110,7 @@ flowchart LR
     E -.->|read| G
 ```
 
-> [!NOTE]
+> [!WARNING]
 > _Disclaimer_: I inspire the dynamic statement from the [**GitHub Action**](https://github.com/features/actions)
 > with `.yml` files and all configs file from several data orchestration framework
 > tools from my experience on Data Engineer. :grimacing:
@@ -192,7 +192,17 @@ The above workflow template is main executor pipeline that you want to do. If yo
 want to schedule this workflow, you want to dynamic its parameters change base on
 execution time such as `run-date` should change base on that workflow running date.
 
-So, this package provide the `Schedule` template for this action.
+```python
+from ddeutil.workflow import Workflow, Result
+
+workflow: Workflow = Workflow.from_loader('run-py-local')
+result: Result = workflow.execute(
+   params={"source-extract": "USD-THB", "asat-dt": "2024-01-01"}
+)
+```
+
+So, this package provide the `Schedule` template for this action, and you can dynamic
+pass the parameters for changing align with that running time by the `release` prefix.
 
 ```yaml
 schedule-run-local-wf:
@@ -210,6 +220,20 @@ schedule-run-local-wf:
           asat-dt: "${{ release.logical_date }}"
 ```
 
+The main method of the `Schedule` model that use to running is `pending`. If you
+do not pass the `stop` date on this method, it will use config with `WORKFLOW_APP_STOP_BOUNDARY_DELTA`
+key for generate this stop date.
+
+```python
+from ddeutil.workflow import Schedule
+
+(
+   Schedule
+   .from_loader("schedule-run-local-wf")
+   .pending(stop=None)
+)
+```
+
 ## :cookie: Configuration
 
 The main configuration that use to dynamic changing this workflow engine for your

ddeutil_workflow-0.0.32.dist-info/RECORD

@@ -0,0 +1,25 @@
+ddeutil/workflow/__about__.py,sha256=xnsfT_lwFz-qKzFIHbOUGMgvgpOAAANig-X3brDri44,28
+ddeutil/workflow/__cron.py,sha256=3i-wmjTlh0ADCzN9pLKaWHzJkXzC72aIBmVEQSbyCCE,26895
+ddeutil/workflow/__init__.py,sha256=pRIZIGwC7Xs8Ur7-jHPIAMLriD5If9zOPc-ZmKZS2XQ,1678
+ddeutil/workflow/__types.py,sha256=CK1jfzyHP9P-MB0ElhpJZ59ZFGJC9MkQuAop5739_9k,4304
+ddeutil/workflow/conf.py,sha256=6yGbSi69lsccYgnrwTzdjdPhU54hUop2e1GjBNres08,17663
+ddeutil/workflow/cron.py,sha256=j8EeoHst70toRfnD_frix41vrI-eLYVJkZ9yeJtpfnI,8871
+ddeutil/workflow/exceptions.py,sha256=5ghT443VLq0IeU87loHNEqqrrrctklP7YfxwJ51ImWU,949
+ddeutil/workflow/hook.py,sha256=MgZFlTGvaRSBrTouZGlxwYpKQoKDOT26PNhESeL3LY0,5469
+ddeutil/workflow/job.py,sha256=siph5FBU5Ah2CHWZyXotHJKMemFWQXSBqSdwk8b_6p8,24366
+ddeutil/workflow/params.py,sha256=LKR7jXyxTb5NVrFav_fl2y9xo3p7qL1S9h-i6CtvNwE,5851
+ddeutil/workflow/result.py,sha256=8LItqF-Xe6pAAWkAsY_QFkKBOA0fEBh97I2og3CZsPc,3409
+ddeutil/workflow/scheduler.py,sha256=r26ZKpx0kuaL0hMJ39tIyZaeFMwhcwitUzBh-ge2ZJg,24394
+ddeutil/workflow/stage.py,sha256=vUne9aUZX7G8K5BkBwirAjXSVwVvAoBt8HXbqUMEgns,24329
+ddeutil/workflow/templates.py,sha256=A0JgZFGkBv-AX-EskZj656nG5zFd3j1PpLpyXihf6Xg,10967
+ddeutil/workflow/utils.py,sha256=rTDQKaaber7cRqzJjWpCP9OTbarti1UMKdLgH6VRjFM,6709
+ddeutil/workflow/workflow.py,sha256=ET1otR5VcfnOMoNiW7EMb1_wIaxNw9yWsBXS5kVWG9s,43428
+ddeutil/workflow/api/__init__.py,sha256=F53NMBWtb9IKaDWkPU5KvybGGfKAcbehgn6TLBwHuuM,21
+ddeutil/workflow/api/api.py,sha256=AwcUkE6S2Txz4hZOD10TsUkuFiKQ47rpMYMYpl7ssis,3954
+ddeutil/workflow/api/repeat.py,sha256=zyvsrXKk-3-_N8ZRZSki0Mueshugum2jtqctEOp9QSc,4927
+ddeutil/workflow/api/route.py,sha256=A9YPfchPNRyN6ev5sg1Z12zSi8JEQ6yZEea0EFPQp8k,8591
+ddeutil_workflow-0.0.32.dist-info/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
+ddeutil_workflow-0.0.32.dist-info/METADATA,sha256=FYn_uM88ehiyBBZto8ECxmWHwBPtFGHAOcBPkaRBNFU,15552
+ddeutil_workflow-0.0.32.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+ddeutil_workflow-0.0.32.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
+ddeutil_workflow-0.0.32.dist-info/RECORD,,

ddeutil_workflow-0.0.30.dist-info/RECORD

@@ -1,25 +0,0 @@
-ddeutil/workflow/__about__.py,sha256=zSy9Xk11PPZkgJ5Db1_kZp2yzt5inkEHVJWwFHPWlmk,28
-ddeutil/workflow/__cron.py,sha256=uA8XcbY_GwA9rJSHaHUaXaJyGDObJN0ZeYlJSinL8y8,26880
-ddeutil/workflow/__init__.py,sha256=dghn2lFl3Own4Pyq7SFHu-FMymOgLontJ6aCfxea9h4,1606
-ddeutil/workflow/__types.py,sha256=CK1jfzyHP9P-MB0ElhpJZ59ZFGJC9MkQuAop5739_9k,4304
-ddeutil/workflow/conf.py,sha256=7lj_Im9jsa95fWUo19Q4-ZAcHa8Pu1HW-vaLgvrjNUM,17559
-ddeutil/workflow/cron.py,sha256=OLgniUxmrn65gzckk-uTmE2Pk1enJJyjYUKVeBbDQz0,7522
-ddeutil/workflow/exceptions.py,sha256=XUnpJSuxOyataClP0w_gpYjzn-NIwZK2BHro-J7Yw24,895
-ddeutil/workflow/hook.py,sha256=MgZFlTGvaRSBrTouZGlxwYpKQoKDOT26PNhESeL3LY0,5469
-ddeutil/workflow/job.py,sha256=XcewyALsLYYq94ycF6mkj3Ydr6if683z7t1oBqEVInE,24290
-ddeutil/workflow/params.py,sha256=svCjmFgEhim8yFJVjZhFmKP8JqTDHQ5EPhwJHVuDGno,5289
-ddeutil/workflow/result.py,sha256=k4pcj5KjbEcEPymsEUXeGY4gyLMfPkMTO6YDrAtfk7Q,3408
-ddeutil/workflow/scheduler.py,sha256=f3d7c5SVgY5Q1JsHQ6cH513CJmJkh4l8YcKAGYudJRc,20426
-ddeutil/workflow/stage.py,sha256=wn8CARTvFJY4ZK1SwjzH8sKoMRz_eIeSGUMgnDWNi6g,24031
-ddeutil/workflow/templates.py,sha256=bVU_8gnMQmdhhw3W28ZqwmpEaOx10Nx_aauqiLS0lqg,10807
-ddeutil/workflow/utils.py,sha256=8LTqpvRPfrEYxsxhwszk6GKkyjrswxnwF3r_9vE8szw,6059
-ddeutil/workflow/workflow.py,sha256=ZLbG-K2gSNAsDGiHPjbtJd7rsEFf6jfVGAVB_9jpy84,42103
-ddeutil/workflow/api/__init__.py,sha256=F53NMBWtb9IKaDWkPU5KvybGGfKAcbehgn6TLBwHuuM,21
-ddeutil/workflow/api/api.py,sha256=Ma9R8yuQAhowG_hox-k53swFsf5IAvheEnSxNQ-8DaQ,4039
-ddeutil/workflow/api/repeat.py,sha256=zyvsrXKk-3-_N8ZRZSki0Mueshugum2jtqctEOp9QSc,4927
-ddeutil/workflow/api/route.py,sha256=v96jNbgjM1cJ2MpVSRWs2kgRqF8DQElEBdRZrVFEpEw,8578
-ddeutil_workflow-0.0.30.dist-info/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
-ddeutil_workflow-0.0.30.dist-info/METADATA,sha256=zbVHOL41qpFRG83SacZVYK9tS2JRTCM61cpnSXty9LU,14868
-ddeutil_workflow-0.0.30.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-ddeutil_workflow-0.0.30.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
-ddeutil_workflow-0.0.30.dist-info/RECORD,,