ddeutil-workflow 0.0.9__py3-none-any.whl → 0.0.10__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1 +1 @@
-__version__: str = "0.0.9"
+__version__: str = "0.0.10"

ddeutil/workflow/api.py

@@ -6,7 +6,6 @@
 from __future__ import annotations
 
 import asyncio
-import logging
 import os
 import uuid
 from queue import Empty, Queue
@@ -18,33 +17,33 @@ from fastapi.middleware.gzip import GZipMiddleware
 from fastapi.responses import UJSONResponse
 from pydantic import BaseModel
 
+from .__about__ import __version__
+from .log import get_logger
 from .repeat import repeat_every
 
 load_dotenv()
-logger = logging.getLogger(__name__)
-logging.basicConfig(
-    level=logging.DEBUG,
-    format=(
-        "%(asctime)s.%(msecs)03d (%(name)-10s, %(process)-5d, %(thread)-5d) "
-        "[%(levelname)-7s] %(message)-120s (%(filename)s:%(lineno)s)"
-    ),
-    handlers=[logging.StreamHandler()],
-    datefmt="%Y-%m-%d %H:%M:%S",
-)
+logger = get_logger("ddeutil.workflow")
 
 
-app = FastAPI()
+app = FastAPI(
+    titile="Workflow API",
+    description=(
+        "This is workflow FastAPI web application that use to manage manual "
+        "execute or schedule workflow via RestAPI."
+    ),
+    version=__version__,
+)
 app.add_middleware(GZipMiddleware, minimum_size=1000)
 app.queue = Queue()
 app.output_dict = {}
-app.queue_limit = 2
+app.queue_limit = 5
 
 
 @app.on_event("startup")
-@repeat_every(seconds=10, logger=logger)
+@repeat_every(seconds=10)
 def broker_upper_messages():
     """Broker for receive message from the `/upper` path and change it to upper
-    case.
+    case. This broker use interval running in background every 10 seconds.
     """
     for _ in range(app.queue_limit):
         try:
@@ -66,11 +65,12 @@ async def get_result(request_id):
             result = app.output_dict[request_id]
             del app.output_dict[request_id]
             return {"message": result}
-        await asyncio.sleep(0.001)
+        await asyncio.sleep(0.0025)
 
 
 @app.post("/upper", response_class=UJSONResponse)
 async def message_upper(payload: Payload):
+    """Convert message from any case to the upper case."""
     request_id: str = str(uuid.uuid4())
     app.queue.put(
         {"text": payload.text, "request_id": request_id},

ddeutil/workflow/cli.py

@@ -5,46 +5,129 @@
 # ------------------------------------------------------------------------------
 from __future__ import annotations
 
-from typing import Optional
+import json
+import os
+from datetime import datetime
+from enum import Enum
+from typing import Annotated, Optional
+from zoneinfo import ZoneInfo
 
-from typer import Typer
+from ddeutil.core import str2list
+from typer import Argument, Option, Typer
 
+from .log import get_logger
+
+logger = get_logger("ddeutil.workflow")
 cli: Typer = Typer()
-state = {"verbose": False}
+cli_log: Typer = Typer()
+cli.add_typer(
+    cli_log,
+    name="log",
+    help="Logging of workflow CLI",
+)
 
 
 @cli.command()
-def run(pipeline: str):
-    """Run workflow manually"""
-    if state["verbose"]:
-        print("About to create a user")
+def run(
+    pipeline: Annotated[
+        str,
+        Argument(help="A pipeline name that want to run manually"),
+    ],
+    params: Annotated[
+        str,
+        Argument(
+            help="A json string for parameters of this pipeline execution."
+        ),
+    ],
+):
+    """Run pipeline workflow manually with an input custom parameters that able
+    to receive with pipeline params config.
+    """
+    logger.info(f"Running pipeline name: {pipeline}")
+    logger.info(f"... with Parameters: {json.dumps(json.loads(params))}")
 
-    print(f"Creating user: {pipeline}")
 
-    if state["verbose"]:
-        print("Just created a user")
+@cli.command()
+def schedule(
+    stop: Annotated[
+        Optional[datetime],
+        Argument(
+            formats=["%Y-%m-%d", "%Y-%m-%d %H:%M:%S"],
+            help="A stopping datetime that want to stop on schedule app.",
+        ),
+    ] = None,
+    excluded: Annotated[
+        Optional[str],
+        Argument(help="A list of exclude workflow name in str."),
+    ] = None,
+    externals: Annotated[
+        Optional[str],
+        Argument(
+            help="A json string for parameters of this pipeline execution."
+        ),
+    ] = None,
+):
+    """Start workflow scheduler that will call workflow function from scheduler
+    module.
+    """
+    excluded: list[str] = str2list(excluded) if excluded else []
+    externals: str = externals or "{}"
+    if stop:
+        stop: datetime = stop.astimezone(
+            tz=ZoneInfo(os.getenv("WORKFLOW_CORE_TIMEZONE", "UTC"))
+        )
 
+    from .scheduler import workflow
 
-@cli.command()
-def schedule(exclude: Optional[str]):
-    """Start workflow scheduler"""
-    if state["verbose"]:
-        print("About to delete a user")
+    # NOTE: Start running workflow scheduler application.
+    workflow_rs: list[str] = workflow(
+        stop=stop, excluded=excluded, externals=json.loads(externals)
+    )
+    logger.info(f"Application run success: {workflow_rs}")
+
+
+@cli_log.command("pipeline-get")
+def pipeline_log_get(
+    name: Annotated[
+        str,
+        Argument(help="A pipeline name that want to getting log"),
+    ],
+    limit: Annotated[
+        int,
+        Argument(help="A number of the limitation of logging"),
+    ] = 100,
+    desc: Annotated[
+        bool,
+        Option(
+            "--desc",
+            help="A descending flag that order by logging release datetime.",
+        ),
+    ] = True,
+):
+    logger.info(f"{name} : limit {limit} : desc: {desc}")
+    return [""]
+
+
+class LogMode(str, Enum):
+    get = "get"
+    delete = "delete"
 
-    print(f"Deleting user: {exclude}")
 
-    if state["verbose"]:
-        print("Just deleted a user")
+@cli_log.command("pipeline-delete")
+def pipeline_log_delete(
+    mode: Annotated[
+        LogMode,
+        Argument(case_sensitive=True),
+    ]
+):
+    logger.info(mode)
 
 
 @cli.callback()
-def main(verbose: bool = False):
+def main():
     """
     Manage workflow with CLI.
     """
-    if verbose:
-        print("Will write verbose output")
-        state["verbose"] = True
 
 
 if __name__ == "__main__":

ddeutil/workflow/cron.py

@@ -10,7 +10,7 @@ from collections.abc import Iterator
 from dataclasses import dataclass, field
 from datetime import datetime, timedelta
 from functools import partial, total_ordering
-from typing import Callable, Optional, Union
+from typing import ClassVar, Optional, Union
 from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
 
 from ddeutil.core import (
@@ -34,6 +34,34 @@ WEEKDAYS: dict[str, int] = {
 }
 
 
+class CronYearLimit(Exception): ...
+
+
+def str2cron(value: str) -> str:
+    """Convert Special String to Crontab.
+
+    @reboot 	Run once, at system startup
+    @yearly 	Run once every year, "0 0 1 1 *"
+    @annually   (same as @yearly)
+    @monthly    Run once every month, "0 0 1 * *"
+    @weekly 	Run once every week, "0 0 * * 0"
+    @daily  	Run once each day, "0 0 * * *"
+    @midnight   (same as @daily)
+    @hourly 	Run once an hour, "0 * * * *"
+    """
+    mapping_spacial_str = {
+        "@reboot": "",
+        "@yearly": "0 0 1 1 *",
+        "@annually": "0 0 1 1 *",
+        "@monthly": "0 0 1 * *",
+        "@weekly": "0 0 * * 0",
+        "@daily": "0 0 * * *",
+        "@midnight": "0 0 * * *",
+        "@hourly": "0 * * * *",
+    }
+    return mapping_spacial_str[value]
+
+
 @dataclass(frozen=True)
 class Unit:
     name: str
@@ -232,17 +260,21 @@ class CronPart:
         :type value: str
 
         TODO: support for `L`, `W`, and `#`
-        TODO: if you didn't care what day of the week the 7th was, you
-            could enter ? in the Day-of-week field.
-        TODO: L : the Day-of-month or Day-of-week fields specifies the last day
-            of the month or week.
+        ---
+        TODO: The ? (question mark) wildcard specifies one or another.
+            In the Day-of-month field you could enter 7, and if you didn't care
+            what day of the week the seventh was, you could enter ? in the
+            Day-of-week field.
+        TODO: L : The L wildcard in the Day-of-month or Day-of-week fields
+            specifies the last day of the month or week.
             DEV: use -1 for represent with L
-        TODO: W : In the Day-of-month field, 3W specifies the weekday closest
-            to the third day of the month.
-        TODO: # : 3#2 would be the second Tuesday of the month,
+        TODO: W : The W wildcard in the Day-of-month field specifies a weekday.
+            In the Day-of-month field, 3W specifies the weekday closest to the
+            third day of the month.
+        TODO: # : 3#2 would be the second Tuesday of every month,
             the 3 refers to Tuesday because it is the third day of each week.
 
-        Noted:
+        Examples:
             -   0 10 * * ? *
                 Run at 10:00 am (UTC) every day
 
@@ -275,6 +307,7 @@ class CronPart:
         :rtype: tuple[int, ...]
         """
         interval_list: list[list[int]] = []
+        # NOTE: Start replace alternative like JAN to FEB or MON to SUN.
         for _value in self.replace_alternative(value.upper()).split(","):
             if _value == "?":
                 continue
@@ -294,11 +327,18 @@ class CronPart:
                     f"{self.unit.name!r}"
                 )
 
+            # NOTE: Generate interval that has step
             interval_list.append(self._interval(value_range_list, value_step))
+
         return tuple(item for sublist in interval_list for item in sublist)
 
     def replace_alternative(self, value: str) -> str:
-        """Replaces the alternative representations of numbers in a string."""
+        """Replaces the alternative representations of numbers in a string.
+
+            For example if value == 'JAN,AUG' it will replace to '1,8'.
+
+        :param value: A string value that want to replace alternative to int.
+        """
         for i, alt in enumerate(self.unit.alt):
             if alt in value:
                 value: str = value.replace(alt, str(self.unit.min + i))
@@ -455,7 +495,7 @@ class CronJob:
     """The Cron Job Converter object that generate datetime dimension of cron
     job schedule format,
 
-            ... * * * * * <command to execute>
+        *  *  *  *  *  <command to execute>
 
         (i)     minute (0 - 59)
         (ii)    hour (0 - 23)
@@ -469,9 +509,20 @@ class CronJob:
         Support special value with `/`, `*`, `-`, `,`, and `?` (in day of month
     and day of week value).
 
+        Fields          | Values            | Wildcards
+        ---             | ---               | ---
+        Minutes         | 0–59              | , - * /
+        Hours           | 0–23              | , - * /
+        Day-of-month    | 1–31              | , - * ? / L W
+        Month           | 1–12 or JAN-DEC   | , - * /
+        Day-of-week     | 1–7 or SUN-SAT    | , - * ? / L
+        Year            | 1970–2199         | , - * /
+
     References:
         - https://github.com/Sonic0/cron-converter
         - https://pypi.org/project/python-crontab/
+        - https://docs.aws.amazon.com/glue/latest/dg/ -
+            monitor-data-warehouse-schedule.html
     """
 
     cron_length: int = 5
@@ -567,6 +618,10 @@ class CronJob:
         """Returns the cron schedule as a 2-dimensional list of integers."""
         return [part.values for part in self.parts]
 
+    def check(self, date: datetime, mode: str) -> bool:
+        assert mode in ("year", "month", "day", "hour", "minute")
+        return getattr(date, mode) in getattr(self, mode).values
+
     def schedule(
         self,
         date: datetime | None = None,
@@ -598,9 +653,12 @@ class CronRunner:
     cron schedule object value.
     """
 
+    shift_limit: ClassVar[int] = 25
+
     __slots__: tuple[str, ...] = (
         "__start_date",
         "cron",
+        "is_year",
         "date",
         "reset_flag",
         "tz",
@@ -639,6 +697,7 @@ class CronRunner:
 
         self.__start_date: datetime = self.date
         self.cron: CronJob | CronJobYear = cron
+        self.is_year: bool = isinstance(cron, CronJobYear)
         self.reset_flag: bool = True
 
     def reset(self) -> None:
@@ -663,45 +722,76 @@ class CronRunner:
         return self.find_date(reverse=True)
 
     def find_date(self, reverse: bool = False) -> datetime:
-        """Returns the time the schedule would run by `next` or `prev`.
+        """Returns the time the schedule would run by `next` or `prev` methods.
 
         :param reverse: A reverse flag.
         """
         # NOTE: Set reset flag to false if start any action.
         self.reset_flag: bool = False
-        for _ in range(25):
+
+        # NOTE: For loop with 25 times by default.
+        for _ in range(
+            max(self.shift_limit, 100) if self.is_year else self.shift_limit
+        ):
+
+            # NOTE: Shift the date
             if all(
                 not self.__shift_date(mode, reverse)
-                for mode in ("month", "day", "hour", "minute")
+                for mode in ("year", "month", "day", "hour", "minute")
             ):
                 return copy.deepcopy(self.date.replace(second=0, microsecond=0))
+
         raise RecursionError("Unable to find execution time for schedule")
 
     def __shift_date(self, mode: str, reverse: bool = False) -> bool:
-        """Increments the mode value until matches with the schedule."""
+        """Increments the mode of date value ("month", "day", "hour", "minute")
+        until matches with the schedule.
+
+        :param mode: A mode of date that want to shift.
+        :param reverse: A flag that define shifting next or previous
+        """
         switch: dict[str, str] = {
+            "year": "year",
             "month": "year",
             "day": "month",
             "hour": "day",
             "minute": "hour",
         }
         current_value: int = getattr(self.date, switch[mode])
-        _addition_condition: Callable[[], bool] = (
-            (
-                lambda: WEEKDAYS.get(self.date.strftime("%a"))
-                not in self.cron.dow.values
+
+        if not self.is_year and mode == "year":
+            return False
+
+        # NOTE: Additional condition for weekdays
+        def addition_cond(dt: datetime) -> bool:
+            return (
+                WEEKDAYS.get(dt.strftime("%a")) not in self.cron.dow.values
+                if mode == "day"
+                else False
             )
-            if mode == "day"
-            else lambda: False
-        )
-        # NOTE: Start while-loop for checking this date include in this cronjob.
-        while (
-            getattr(self.date, mode) not in getattr(self.cron, mode).values
-        ) or _addition_condition():
+
+        # NOTE:
+        #   Start while-loop for checking this date include in this cronjob.
+        while not self.cron.check(self.date, mode) or addition_cond(self.date):
+            if mode == "year" and (
+                getattr(self.date, mode)
+                > (max_year := max(self.cron.year.values))
+            ):
+                raise CronYearLimit(
+                    f"The year is out of limit with this crontab value: "
+                    f"{max_year}."
+                )
+
+            # NOTE: Shift date with it mode matrix unit.
             self.date: datetime = next_date(self.date, mode, reverse=reverse)
+
+            # NOTE: Replace date that less than it mode to zero.
             self.date: datetime = replace_date(self.date, mode, reverse=reverse)
+
             if current_value != getattr(self.date, switch[mode]):
                 return mode != "month"
+
+        # NOTE: Return False if the date that match with condition.
         return False
 
 

ddeutil/workflow/exceptions.py

@@ -21,4 +21,7 @@ class JobException(WorkflowException): ...
 class PipelineException(WorkflowException): ...
 
 
+class PipelineFailException(WorkflowException): ...
+
+
 class ParamValueException(WorkflowException): ...