ddeutil-workflow 0.0.29__py3-none-any.whl → 0.0.31__py3-none-any.whl

ddeutil/workflow/workflow.py

@@ -29,7 +29,8 @@ from concurrent.futures import (
 )
 from dataclasses import field
 from datetime import datetime, timedelta
-from functools import total_ordering
+from enum import Enum
+from functools import partial, total_ordering
 from heapq import heappop, heappush
 from queue import Queue
 from textwrap import dedent
@@ -53,29 +54,42 @@ from .utils import (
     cut_id,
     gen_id,
     get_dt_now,
-    wait_a_minute,
+    reach_next_minute,
+    wait_to_next_minute,
 )
 
 logger = get_logger("ddeutil.workflow")
 
 __all__: TupleStr = (
+    "Release",
+    "ReleaseQueue",
+    "ReleaseType",
     "Workflow",
-    "WorkflowRelease",
-    "WorkflowQueue",
     "WorkflowTask",
 )
 
 
+class ReleaseType(str, Enum):
+    """Release Type Enum support the type field on the Release dataclass."""
+
+    DEFAULT: str = "manual"
+    TASK: str = "task"
+    POKE: str = "poking"
+
+
 @total_ordering
-@dataclass(config=ConfigDict(arbitrary_types_allowed=True))
-class WorkflowRelease:
-    """Workflow release Pydantic dataclass object."""
+@dataclass(
+    config=ConfigDict(arbitrary_types_allowed=True, use_enum_values=True)
+)
+class Release:
+    """Release Pydantic dataclass object that use for represent
+    the release data that use with the `workflow.release` method."""
 
     date: datetime
     offset: float
     end_date: datetime
     runner: CronRunner
-    type: str
+    type: ReleaseType = field(default=ReleaseType.DEFAULT)
 
     def __repr__(self) -> str:
         return repr(f"{self.date:%Y-%m-%d %H:%M:%S}")
@@ -85,7 +99,7 @@ class WorkflowRelease:
 
     @classmethod
     def from_dt(cls, dt: datetime | str) -> Self:
-        """Construct WorkflowRelease via datetime object only.
+        """Construct Release via datetime object only.
 
         :param dt: A datetime object.
 
@@ -93,16 +107,19 @@ class WorkflowRelease:
         """
         if isinstance(dt, str):
             dt: datetime = datetime.fromisoformat(dt)
+        elif not isinstance(dt, datetime):
+            raise TypeError(
+                "The `from_dt` need argument type be str or datetime only."
+            )
 
         return cls(
             date=dt,
             offset=0,
             end_date=dt + timedelta(days=1),
             runner=CronJob("* * * * *").schedule(dt.replace(tzinfo=config.tz)),
-            type="manual",
         )
 
-    def __eq__(self, other: WorkflowRelease | datetime) -> bool:
+    def __eq__(self, other: Release | datetime) -> bool:
         """Override equal property that will compare only the same type or
         datetime.
         """
@@ -112,7 +129,7 @@ class WorkflowRelease:
             return self.date == other
         return NotImplemented
 
-    def __lt__(self, other: WorkflowRelease | datetime) -> bool:
+    def __lt__(self, other: Release | datetime) -> bool:
         """Override equal property that will compare only the same type or
         datetime.
         """
@@ -124,19 +141,19 @@ class WorkflowRelease:
 
 
 @dataclass
-class WorkflowQueue:
-    """Workflow Queue object that is management of WorkflowRelease objects."""
+class ReleaseQueue:
+    """Workflow Queue object that is management of Release objects."""
 
-    queue: list[WorkflowRelease] = field(default_factory=list)
-    running: list[WorkflowRelease] = field(default_factory=list)
-    complete: list[WorkflowRelease] = field(default_factory=list)
+    queue: list[Release] = field(default_factory=list)
+    running: list[Release] = field(default_factory=list)
+    complete: list[Release] = field(default_factory=list)
 
     @classmethod
     def from_list(
-        cls, queue: list[datetime] | list[WorkflowRelease] | None = None
+        cls, queue: list[datetime] | list[Release] | None = None
     ) -> Self:
-        """Construct WorkflowQueue object from an input queue value that passing
-        with list of datetime or list of WorkflowRelease.
+        """Construct ReleaseQueue object from an input queue value that passing
+        with list of datetime or list of Release.
 
         :raise TypeError: If the type of input queue does not valid.
 
@@ -148,14 +165,14 @@ class WorkflowQueue:
         if isinstance(queue, list):
 
             if all(isinstance(q, datetime) for q in queue):
-                return cls(queue=[WorkflowRelease.from_dt(q) for q in queue])
+                return cls(queue=[Release.from_dt(q) for q in queue])
 
-            if all(isinstance(q, WorkflowRelease) for q in queue):
+            if all(isinstance(q, Release) for q in queue):
                 return cls(queue=queue)
 
         raise TypeError(
-            "Type of the queue does not valid with WorkflowQueue "
-            "or list of datetime or list of WorkflowRelease."
+            "Type of the queue does not valid with ReleaseQueue "
+            "or list of datetime or list of Release."
         )
 
     @property
@@ -167,32 +184,25 @@ class WorkflowQueue:
         return len(self.queue) > 0
 
     @property
-    def first_queue(self) -> WorkflowRelease:
-        """Check an input WorkflowRelease object is the first value of the
+    def first_queue(self) -> Release:
+        """Check an input Release object is the first value of the
         waiting queue.
 
         :rtype: bool
         """
-        # NOTE: Old logic to peeking the first release from waiting queue.
-        #
-        # first_value: WorkflowRelease = heappop(self.queue)
-        # heappush(self.queue, first_value)
-        #
-        # return first_value
-        #
         return self.queue[0]
 
-    def check_queue(self, value: WorkflowRelease | datetime) -> bool:
-        """Check a WorkflowRelease value already exists in list of tracking
+    def check_queue(self, value: Release | datetime) -> bool:
+        """Check a Release value already exists in list of tracking
         queues.
 
-        :param value: A WorkflowRelease object that want to check it already in
+        :param value: A Release object that want to check it already in
             queues.
 
         :rtype: bool
         """
         if isinstance(value, datetime):
-            value = WorkflowRelease.from_dt(value)
+            value = Release.from_dt(value)
 
         return (
             (value in self.queue)
@@ -200,13 +210,21 @@ class WorkflowQueue:
             or (value in self.complete)
         )
 
-    def remove_running(self, value: WorkflowRelease) -> Self:
-        """Remove WorkflowRelease in the running queue if it exists."""
+    def remove_running(self, value: Release) -> Self:
+        """Remove Release in the running queue if it exists.
+
+        :rtype: Self
+        """
         if value in self.running:
             self.running.remove(value)
 
-    def mark_complete(self, value: WorkflowRelease) -> Self:
-        """Push WorkflowRelease to the complete queue."""
+        return self
+
+    def mark_complete(self, value: Release) -> Self:
+        """Push Release to the complete queue.
+
+        :rtype: Self
+        """
         heappush(self.complete, value)
 
         # NOTE: Remove complete queue on workflow that keep more than the
@@ -216,7 +234,6 @@ class WorkflowQueue:
         )
 
         if num_complete_delete > 0:
-            print(num_complete_delete)
             for _ in range(num_complete_delete):
                 heappop(self.complete)
 
@@ -464,14 +481,12 @@ class Workflow(BaseModel):
 
     def release(
         self,
-        release: datetime | WorkflowRelease,
+        release: datetime | Release,
         params: DictData,
         *,
         run_id: str | None = None,
         log: type[Log] = None,
-        queue: (
-            WorkflowQueue | list[datetime] | list[WorkflowRelease] | None
-        ) = None,
+        queue: ReleaseQueue | None = None,
         override_log_name: str | None = None,
     ) -> Result:
         """Release the workflow execution with overriding parameter with the
@@ -481,20 +496,20 @@ class Workflow(BaseModel):
             This method allow workflow use log object to save the execution
         result to log destination like file log to the local `/logs` directory.
 
-        :Steps:
-            - Initialize WorkflowQueue and WorkflowRelease if they do not pass.
+        Steps:
+            - Initialize ReleaseQueue and Release if they do not pass.
             - Create release data for pass to parameter templating function.
             - Execute this workflow with mapping release data to its parameters.
             - Writing result log
             - Remove this release on the running queue
             - Push this release to complete queue
 
-        :param release: A release datetime or WorkflowRelease object.
+        :param release: A release datetime or Release object.
         :param params: A workflow parameter that pass to execute method.
-        :param queue: A list of release time that already queue.
+        :param queue: A ReleaseQueue that use for mark complete.
         :param run_id: A workflow running ID for this release.
         :param log: A log class that want to save the execution result.
-        :param queue: A WorkflowQueue object.
+        :param queue: A ReleaseQueue object.
         :param override_log_name: An override logging name that use instead
             the workflow name.
 
@@ -504,16 +519,17 @@ class Workflow(BaseModel):
         name: str = override_log_name or self.name
         run_id: str = run_id or gen_id(name, unique=True)
         rs_release: Result = Result(run_id=run_id)
-        rs_release_type: str = "release"
 
-        # VALIDATE: Change queue value to WorkflowQueue object.
-        if queue is None or isinstance(queue, list):
-            queue: WorkflowQueue = WorkflowQueue.from_list(queue)
+        if queue is not None and not isinstance(queue, ReleaseQueue):
+            raise TypeError(
+                "The queue argument should be ReleaseQueue object only."
+            )
 
-        # VALIDATE: Change release value to WorkflowRelease object.
+        # VALIDATE: Change release value to Release object.
+        rs_release_type: str = "release"
         if isinstance(release, datetime):
             rs_release_type: str = "datetime"
-            release: WorkflowRelease = WorkflowRelease.from_dt(release)
+            release: Release = Release.from_dt(release)
 
         logger.debug(
             f"({cut_id(run_id)}) [RELEASE]: Start release - {name!r} : "
@@ -542,24 +558,26 @@ class Workflow(BaseModel):
         )
 
         rs.set_parent_run_id(run_id)
-        rs_log: Log = log.model_validate(
-            {
-                "name": name,
-                "release": release.date,
-                "type": release.type,
-                "context": rs.context,
-                "parent_run_id": rs.parent_run_id,
-                "run_id": rs.run_id,
-            }
-        )
 
         # NOTE: Saving execution result to destination of the input log object.
         logger.debug(f"({cut_id(run_id)}) [LOG]: Writing log: {name!r}.")
-        rs_log.save(excluded=None)
+        (
+            log.model_validate(
+                {
+                    "name": name,
+                    "release": release.date,
+                    "type": release.type,
+                    "context": rs.context,
+                    "parent_run_id": rs.parent_run_id,
+                    "run_id": rs.run_id,
+                }
+            ).save(excluded=None)
+        )
 
         # NOTE: Remove this release from running.
-        queue.remove_running(release)
-        queue.mark_complete(release)
+        if queue is not None:
+            queue.remove_running(release)
+            queue.mark_complete(release)
 
         # NOTE: Remove the params key from the result context for deduplicate.
         context: dict[str, Any] = rs.context
@@ -583,13 +601,21 @@ class Workflow(BaseModel):
         self,
         offset: float,
         end_date: datetime,
-        queue: WorkflowQueue,
+        queue: ReleaseQueue,
         log: type[Log],
         *,
         force_run: bool = False,
-    ) -> WorkflowQueue:
-        """Generate queue of datetime from the cron runner that initialize from
-        the on field. with offset value.
+    ) -> ReleaseQueue:
+        """Generate Release from all on values from the on field and store them
+        to the ReleaseQueue object.
+
+        Steps:
+            - For-loop all the on value in the on field.
+            - Create Release object from the current date that not reach the end
+              date.
+            - Check this release do not store on the release queue object.
+              Generate the next date if it exists.
+            - Push this release to the release queue
 
         :param offset: An offset in second unit for time travel.
         :param end_date: An end datetime object.
@@ -598,7 +624,7 @@ class Workflow(BaseModel):
         :param force_run: A flag that allow to release workflow if the log with
             that release was pointed.
 
-        :rtype: WorkflowQueue
+        :rtype: ReleaseQueue
         """
         for on in self.on:
 
@@ -610,30 +636,30 @@ class Workflow(BaseModel):
             if runner.date > end_date:
                 continue
 
-            workflow_release = WorkflowRelease(
+            workflow_release = Release(
                 date=runner.date,
                 offset=offset,
                 end_date=end_date,
                 runner=runner,
-                type="poking",
+                type=ReleaseType.POKE,
             )
 
             while queue.check_queue(workflow_release) or (
                 log.is_pointed(name=self.name, release=workflow_release.date)
                 and not force_run
             ):
-                workflow_release = WorkflowRelease(
+                workflow_release = Release(
                     date=runner.next,
                     offset=offset,
                     end_date=end_date,
                     runner=runner,
-                    type="poking",
+                    type=ReleaseType.POKE,
                 )
 
             if runner.date > end_date:
                 continue
 
-            # NOTE: Push the WorkflowRelease object to queue.
+            # NOTE: Push the Release object to queue.
             heappush(queue.queue, workflow_release)
 
         return queue
@@ -656,6 +682,9 @@ class Workflow(BaseModel):
             This method will observe its schedule that nearing to run with the
         ``self.release()`` method.
 
+            The limitation of this method is not allow run a date that less
+        than the current date.
+
         :param start_date: A start datetime object.
         :param params: A parameters that want to pass to the release method.
         :param run_id: A workflow running ID for this poke.
@@ -672,6 +701,12 @@ class Workflow(BaseModel):
         log: type[Log] = log or get_log()
         run_id: str = run_id or gen_id(self.name, unique=True)
 
+        # VALIDATE: Check the periods value should gather than 0.
+        if periods <= 0:
+            raise WorkflowException(
+                "The period of poking should be int and grater or equal than 1."
+            )
+
         # NOTE: If this workflow does not set the on schedule, it will return
         #   empty result.
         if len(self.on) == 0:
@@ -681,23 +716,25 @@ class Workflow(BaseModel):
             )
             return []
 
-        if periods <= 0:
-            raise WorkflowException(
-                "The period of poking should be int and grater or equal than 1."
-            )
+        # NOTE: Create the current date that change microsecond to 0
+        current_date: datetime = datetime.now(tz=config.tz).replace(
+            microsecond=0
+        )
 
         # NOTE: Create start_date and offset variables.
-        current_date: datetime = datetime.now(tz=config.tz)
-
         if start_date and start_date <= current_date:
-            start_date = start_date.replace(tzinfo=config.tz)
+            start_date = start_date.replace(tzinfo=config.tz).replace(
+                microsecond=0
+            )
             offset: float = (current_date - start_date).total_seconds()
         else:
+            # NOTE: Force change start date if it gathers than the current date,
+            #   or it does not pass to this method.
             start_date: datetime = current_date
             offset: float = 0
 
-        # NOTE: End date is using to stop generate queue with an input periods
-        #   value.
+        # NOTE: The end date is using to stop generate queue with an input
+        #   periods value.
         end_date: datetime = start_date + timedelta(minutes=periods)
 
         logger.info(
@@ -708,18 +745,18 @@ class Workflow(BaseModel):
         params: DictData = {} if params is None else params
         results: list[Result] = []
 
-        # NOTE: Create empty WorkflowQueue object.
-        wf_queue: WorkflowQueue = WorkflowQueue()
+        # NOTE: Create empty ReleaseQueue object.
+        q: ReleaseQueue = ReleaseQueue()
 
-        # NOTE: Make queue to the workflow queue object.
-        self.queue(
-            offset,
-            end_date=end_date,
-            queue=wf_queue,
-            log=log,
-            force_run=force_run,
+        # NOTE: Create reusable partial function and add Release to the release
+        #   queue object.
+        partial_queue = partial(
+            self.queue, offset, end_date, log=log, force_run=force_run
         )
-        if not wf_queue.is_queued:
+        partial_queue(q)
+
+        # NOTE: Return the empty result if it does not have any Release.
+        if not q.is_queued:
             logger.info(
                 f"({cut_id(run_id)}) [POKING]: {self.name!r} does not have "
                 f"any queue."
@@ -735,34 +772,27 @@ class Workflow(BaseModel):
 
             futures: list[Future] = []
 
-            while wf_queue.is_queued:
+            while q.is_queued:
 
-                # NOTE: Pop the latest WorkflowRelease object from queue.
-                release: WorkflowRelease = heappop(wf_queue.queue)
+                # NOTE: Pop the latest Release object from the release queue.
+                release: Release = heappop(q.queue)
 
-                if (
-                    release.date - get_dt_now(tz=config.tz, offset=offset)
-                ).total_seconds() > 60:
+                if reach_next_minute(release.date, tz=config.tz, offset=offset):
                     logger.debug(
-                        f"({cut_id(run_id)}) [POKING]: Wait because the latest "
-                        f"release has diff time more than 60 seconds ..."
+                        f"({cut_id(run_id)}) [POKING]: The latest release, "
+                        f"{release.date:%Y-%m-%d %H:%M:%S}, is not able to run "
+                        f"on this minute"
                     )
-                    heappush(wf_queue.queue, release)
-                    wait_a_minute(get_dt_now(tz=config.tz, offset=offset))
+                    heappush(q.queue, release)
+                    wait_to_next_minute(get_dt_now(tz=config.tz, offset=offset))
 
                     # WARNING: I already call queue poking again because issue
                     #   about the every minute crontab.
-                    self.queue(
-                        offset,
-                        end_date,
-                        queue=wf_queue,
-                        log=log,
-                        force_run=force_run,
-                    )
+                    partial_queue(q)
                     continue
 
-                # NOTE: Push the latest WorkflowRelease to the running queue.
-                heappush(wf_queue.running, release)
+                # NOTE: Push the latest Release to the running queue.
+                heappush(q.running, release)
 
                 futures.append(
                     executor.submit(
@@ -770,17 +800,11 @@ class Workflow(BaseModel):
                         release=release,
                         params=params,
                         log=log,
-                        queue=wf_queue,
+                        queue=q,
                     )
                 )
 
-                self.queue(
-                    offset,
-                    end_date,
-                    queue=wf_queue,
-                    log=log,
-                    force_run=force_run,
-                )
+                partial_queue(q)
 
             # WARNING: This poking method does not allow to use fail-fast
             #   logic to catching parallel execution result.
@@ -1127,6 +1151,9 @@ class WorkflowTask:
 
         This dataclass object is mapping 1-to-1 with workflow and cron runner
     objects.
+
+        This dataclass has the release method for itself.
+
     """
 
     alias: str
@@ -1136,25 +1163,34 @@ class WorkflowTask:
 
     def release(
         self,
-        release: datetime | WorkflowRelease | None = None,
+        release: datetime | Release | None = None,
         run_id: str | None = None,
         log: type[Log] = None,
-        queue: (
-            WorkflowQueue | list[datetime] | list[WorkflowRelease] | None
-        ) = None,
+        queue: ReleaseQueue | None = None,
     ) -> Result:
         """Release the workflow task data.
 
-        :param release: A release datetime or WorkflowRelease object.
+        :param release: A release datetime or Release object.
         :param run_id: A workflow running ID for this release.
         :param log: A log class that want to save the execution result.
-        :param queue: A WorkflowQueue object.
+        :param queue: A ReleaseQueue object that use to mark complete.
 
         :rtype: Result
         """
         log: type[Log] = log or get_log()
 
         if release is None:
+
+            if queue is None:
+                raise ValueError(
+                    "If pass None release value, you should to pass the queue"
+                    "for generate this release."
+                )
+            elif not isinstance(queue, ReleaseQueue):
+                raise TypeError(
+                    "The queue argument should be ReleaseQueue object only."
+                )
+
             if queue.check_queue(self.runner.date):
                 release = self.runner.next
 
@@ -1163,6 +1199,7 @@ class WorkflowTask:
             else:
                 release = self.runner.date
 
+        # NOTE: Call the workflow release method.
         return self.workflow.release(
             release=release,
             params=self.values,
@@ -1175,12 +1212,13 @@ class WorkflowTask:
     def queue(
         self,
         end_date: datetime,
-        queue: WorkflowQueue,
+        queue: ReleaseQueue,
         log: type[Log],
         *,
         force_run: bool = False,
-    ):
-        """Generate WorkflowRelease to WorkflowQueue object.
+    ) -> ReleaseQueue:
+        """Generate Release from the runner field and store it to the
+        ReleaseQueue object.
 
         :param end_date: An end datetime object.
         :param queue: A workflow queue object.
@@ -1188,35 +1226,35 @@ class WorkflowTask:
         :param force_run: A flag that allow to release workflow if the log with
             that release was pointed.
 
-        :rtype: WorkflowQueue
+        :rtype: ReleaseQueue
         """
         if self.runner.date > end_date:
             return queue
 
-        workflow_release = WorkflowRelease(
+        workflow_release = Release(
             date=self.runner.date,
             offset=0,
             end_date=end_date,
             runner=self.runner,
-            type="task",
+            type=ReleaseType.TASK,
         )
 
         while queue.check_queue(workflow_release) or (
             log.is_pointed(name=self.alias, release=workflow_release.date)
             and not force_run
         ):
-            workflow_release = WorkflowRelease(
+            workflow_release = Release(
                 date=self.runner.next,
                 offset=0,
                 end_date=end_date,
                 runner=self.runner,
-                type="task",
+                type=ReleaseType.TASK,
             )
 
         if self.runner.date > end_date:
             return queue
 
-        # NOTE: Push the WorkflowRelease object to queue.
+        # NOTE: Push the Release object to queue.
         heappush(queue.queue, workflow_release)
 
         return queue