ddeutil-workflow 0.0.9__py3-none-any.whl → 0.0.11__py3-none-any.whl

ddeutil/workflow/log.py

@@ -6,32 +6,54 @@
 from __future__ import annotations
 
 import json
+import logging
 import os
-import re
 from abc import ABC, abstractmethod
 from datetime import datetime
-from heapq import heappop, heappush
+from functools import lru_cache
 from pathlib import Path
 from typing import Optional, Union
 
 from ddeutil.core import str2bool
 from pydantic import BaseModel, Field
 from pydantic.functional_validators import model_validator
+from typing_extensions import Self
 
 from .__types import DictData
 from .utils import config
 
 
+@lru_cache
+def get_logger(name: str):
+    """Return logger with an input module name."""
+    logger = logging.getLogger(name)
+    formatter = logging.Formatter(
+        fmt=(
+            "%(asctime)s.%(msecs)03d (%(name)-10s, %(process)-5d, "
+            "%(thread)-5d) [%(levelname)-7s] %(message)-120s "
+            "(%(filename)s:%(lineno)s)"
+        ),
+        datefmt="%Y-%m-%d %H:%M:%S",
+    )
+    stream = logging.StreamHandler()
+    stream.setFormatter(formatter)
+    logger.addHandler(stream)
+
+    debug: bool = str2bool(os.getenv("WORKFLOW_LOG_DEBUG_MODE", "true"))
+    logger.setLevel(logging.DEBUG if debug else logging.INFO)
+    return logger
+
+
 class BaseLog(BaseModel, ABC):
-    """Base Log Pydantic Model"""
+    """Base Log Pydantic Model abstraction that implement only model fields."""
 
-    name: str = Field(description="A pipeline name.")
+    name: str = Field(description="A workflow name.")
     on: str = Field(description="A cronjob string of this piepline schedule.")
     release: datetime = Field(description="A release datetime.")
     context: DictData = Field(
         default_factory=dict,
         description=(
-            "A context data that receive from a pipeline execution result.",
+            "A context data that receive from a workflow execution result.",
         ),
     )
     parent_run_id: Optional[str] = Field(default=None)
@@ -46,17 +68,16 @@ class BaseLog(BaseModel, ABC):
 
     def do_before(self) -> None:
         """To something before end up of initial log model."""
-        return
 
     @abstractmethod
-    def save(self) -> None:
+    def save(self, excluded: list[str] | None) -> None:
         """Save logging"""
         raise NotImplementedError("Log should implement ``save`` method.")
 
 
 class FileLog(BaseLog):
     """File Log Pydantic Model that use to saving log data from result of
-    pipeline execution. It inherit from BaseLog model that implement the
+    workflow execution. It inherit from BaseLog model that implement the
     ``self.save`` method for file.
     """
 
@@ -65,43 +86,28 @@ class FileLog(BaseLog):
         self.pointer().mkdir(parents=True, exist_ok=True)
 
     @classmethod
-    def latest_point(
-        cls,
-        name: str,
-        *,
-        queue: list[datetime] | None = None,
-    ) -> datetime | None:
-        """Return latest point that exist in current logging pointer keeping.
-
-        :param name: A pipeline name
-        :param queue: A release queue.
-        """
-        keeping: Path = config().engine.paths.root / f"./logs/pipeline={name}/"
-        if not keeping.exists():
-            return None
-
-        keeping_files: list[int] = [
-            int(found.stem)
-            for found in keeping.glob("*")
-            if found.is_dir() and re.match(r"\d{14}", found.stem)
-        ]
-
-        latest = max(keeping_files or [None])
+    def find_logs(cls, name: str):
+        pointer: Path = config().engine.paths.root / f"./logs/workflow={name}"
+        for file in pointer.glob("./release=*/*.log"):
+            with file.open(mode="r", encoding="utf-8") as f:
+                yield json.load(f)
 
-        if not queue:
-            if latest is None:
-                return None
-            return datetime.strptime(str(latest), "%Y%m%d%H%M%S")
-
-        latest_queue: datetime = max(queue)
-
-        if latest is None:
-            return latest_queue
-
-        latest_dt: datetime = datetime.strptime(
-            str(latest), "%Y%m%d%H%M%S"
-        ).replace(tzinfo=latest_queue.tzinfo)
-        return max(latest_dt, latest_queue)
+    @classmethod
+    def find_log(cls, name: str, release: datetime | None = None):
+        if release is not None:
+            pointer: Path = (
+                config().engine.paths.root
+                / f"./logs/workflow={name}/release={release:%Y%m%d%H%M%S}"
+            )
+            if not pointer.exists():
+                raise FileNotFoundError(
+                    f"Pointer: ./logs/workflow={name}/"
+                    f"release={release:%Y%m%d%H%M%S} does not found."
+                )
+            return cls.model_validate(
+                obj=json.loads(pointer.read_text(encoding="utf-8"))
+            )
+        raise NotImplementedError("Find latest log does not implement yet.")
 
     @classmethod
     def is_pointed(
@@ -111,63 +117,64 @@ class FileLog(BaseLog):
         *,
         queue: list[datetime] | None = None,
     ) -> bool:
-        """Check this log already point.
+        """Check this log already point in the destination.
 
-        :param name: A pipeline name.
+        :param name: A workflow name.
         :param release: A release datetime.
         :param queue: A list of queue of datetime that already run in the
             future.
         """
+        # NOTE: Check environ variable was set for real writing.
         if not str2bool(os.getenv("WORKFLOW_LOG_ENABLE_WRITE", "false")):
             return False
 
         # NOTE: create pointer path that use the same logic of pointer method.
         pointer: Path = (
             config().engine.paths.root
-            / f"./logs/pipeline={name}/release={release:%Y%m%d%H%M%S}"
+            / f"./logs/workflow={name}/release={release:%Y%m%d%H%M%S}"
         )
 
-        if queue is None:
+        if not queue:
             return pointer.exists()
-
-        if pointer.exists() and not queue:
-            return True
-
-        if queue:
-            latest: datetime = heappop(queue)
-            heappush(queue, latest)
-            if release == latest:
-                return True
-
-        return False
+        return pointer.exists() or (release in queue)
 
     def pointer(self) -> Path:
-        """Return release directory path that was generated from model data."""
+        """Return release directory path that was generated from model data.
+
+        :rtype: Path
+        """
         return (
             config().engine.paths.root
-            / f"./logs/pipeline={self.name}/release={self.release:%Y%m%d%H%M%S}"
+            / f"./logs/workflow={self.name}/release={self.release:%Y%m%d%H%M%S}"
         )
 
-    def save(self) -> None:
-        """Save logging data that receive a context data from a pipeline
+    def save(self, excluded: list[str] | None) -> Self:
+        """Save logging data that receive a context data from a workflow
         execution result.
+
+        :param excluded: An excluded list of key name that want to pass in the
+            model_dump method.
+        :rtype: Self
         """
+        # NOTE: Check environ variable was set for real writing.
         if not str2bool(os.getenv("WORKFLOW_LOG_ENABLE_WRITE", "false")):
-            return
+            return self
 
         log_file: Path = self.pointer() / f"{self.run_id}.log"
         log_file.write_text(
             json.dumps(
-                self.model_dump(),
+                self.model_dump(exclude=excluded),
                 default=str,
+                indent=2,
             ),
             encoding="utf-8",
         )
+        return self
 
 
 class SQLiteLog(BaseLog):
 
-    def save(self) -> None:
+    def save(self, excluded: list[str] | None) -> None:
         raise NotImplementedError("SQLiteLog does not implement yet.")
 
 

ddeutil/workflow/on.py

@@ -10,12 +10,13 @@ from typing import Annotated, Literal
 from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
 
 from pydantic import BaseModel, ConfigDict, Field
+from pydantic.functional_serializers import field_serializer
 from pydantic.functional_validators import field_validator, model_validator
 from typing_extensions import Self
 
 from .__types import DictData, DictStr, TupleStr
 from .cron import WEEKDAYS, CronJob, CronJobYear, CronRunner
-from .loader import Loader
+from .utils import Loader
 
 __all__: TupleStr = (
     "On",
@@ -101,7 +102,7 @@ class On(BaseModel):
         passing["cronjob"] = interval2crontab(
             **{v: value[v] for v in value if v in ("interval", "day", "time")}
         )
-        return cls(extras=externals, **passing)
+        return cls(extras=externals | passing.pop("extras", {}), **passing)
 
     @classmethod
     def from_loader(
@@ -116,6 +117,7 @@ class On(BaseModel):
         :param externals: A extras external parameter that will keep in extras.
         """
         loader: Loader = Loader(name, externals=externals)
+
         # NOTE: Validate the config type match with current connection model
         if loader.type != cls:
             raise ValueError(f"Type {loader.type} does not match with {cls}")
@@ -131,7 +133,7 @@ class On(BaseModel):
                             if v in ("interval", "day", "time")
                         }
                     ),
-                    extras=externals,
+                    extras=externals | loader_data.pop("extras", {}),
                     **loader_data,
                 )
             )
@@ -140,7 +142,7 @@ class On(BaseModel):
         return cls.model_validate(
             obj=dict(
                 cronjob=loader_data.pop("cronjob"),
-                extras=externals,
+                extras=externals | loader_data.pop("extras", {}),
                 **loader_data,
             )
         )
@@ -166,6 +168,10 @@ class On(BaseModel):
         """Prepare crontab value that able to receive with string type."""
         return CronJob(value) if isinstance(value, str) else value
 
+    @field_serializer("cronjob")
+    def __serialize_cronjob(self, value: CronJob) -> str:
+        return str(value)
+
     def generate(self, start: str | datetime) -> CronRunner:
         """Return Cron runner object."""
         if not isinstance(start, datetime):

ddeutil/workflow/repeat.py

@@ -3,33 +3,63 @@
 # Licensed under the MIT License.
 # This code refs from: https://github.com/priyanshu-panwar/fastapi-utilities
 # ------------------------------------------------------------------------------
+from __future__ import annotations
+
 import asyncio
-import logging
 import os
 from asyncio import ensure_future
 from datetime import datetime
 from functools import wraps
 from zoneinfo import ZoneInfo
 
-from croniter import croniter
 from starlette.concurrency import run_in_threadpool
 
+from .cron import CronJob
+from .log import get_logger
+
+logger = get_logger("ddeutil.workflow")
 
-def get_cron_delta(cron: str):
+
+def get_cronjob_delta(cron: str) -> float:
     """This function returns the time delta between now and the next cron
     execution time.
     """
     now: datetime = datetime.now(
         tz=ZoneInfo(os.getenv("WORKFLOW_CORE_TIMEZONE", "UTC"))
     )
-    cron = croniter(cron, now)
-    return (cron.get_next(datetime) - now).total_seconds()
+    cron = CronJob(cron)
+    return (cron.schedule(now).next - now).total_seconds()
+
+
+def cron_valid(cron: str):
+    try:
+        CronJob(cron)
+    except Exception as err:
+        raise ValueError(f"Crontab value does not valid, {cron}") from err
+
+
+async def run_func(
+    is_coroutine,
+    func,
+    *args,
+    raise_exceptions: bool = False,
+    **kwargs,
+):
+    try:
+        if is_coroutine:
+            await func(*args, **kwargs)
+        else:
+            await run_in_threadpool(func, *args, **kwargs)
+    except Exception as e:
+        logger.exception(e)
+        if raise_exceptions:
+            raise e
 
 
 def repeat_at(
     *,
     cron: str,
-    logger: logging.Logger = None,
+    delay: float = 0,
     raise_exceptions: bool = False,
     max_repetitions: int = None,
 ):
@@ -38,40 +68,38 @@ def repeat_at(
 
     :param cron: str
         Cron-style string for periodic execution, eg. '0 0 * * *' every midnight
-    :param logger: logging.Logger (default None)
-        Logger object to log exceptions
+    :param delay:
     :param raise_exceptions: bool (default False)
         Whether to raise exceptions or log them
     :param max_repetitions: int (default None)
         Maximum number of times to repeat the function. If None, repeat
         indefinitely.
-
     """
+    if max_repetitions and max_repetitions <= 0:
+        raise ValueError(
+            "max_repetitions should more than zero if it want to set"
+        )
 
     def decorator(func):
-        is_coroutine = asyncio.iscoroutinefunction(func)
+        is_coroutine: bool = asyncio.iscoroutinefunction(func)
 
         @wraps(func)
         def wrapper(*_args, **_kwargs):
-            repititions = 0
-            if not croniter.is_valid(cron):
-                raise ValueError("Invalid cron expression")
+            repititions: int = 0
+            cron_valid(cron)
 
             async def loop(*args, **kwargs):
                 nonlocal repititions
                 while max_repetitions is None or repititions < max_repetitions:
-                    try:
-                        sleep_time = get_cron_delta(cron)
-                        await asyncio.sleep(sleep_time)
-                        if is_coroutine:
-                            await func(*args, **kwargs)
-                        else:
-                            await run_in_threadpool(func, *args, **kwargs)
-                    except Exception as e:
-                        if logger:
-                            logger.exception(e)
-                        if raise_exceptions:
-                            raise e
+                    sleep_time = get_cronjob_delta(cron) + delay
+                    await asyncio.sleep(sleep_time)
+                    await run_func(
+                        is_coroutine,
+                        func,
+                        *args,
+                        raise_exceptions=raise_exceptions,
+                        **kwargs,
+                    )
                     repititions += 1
 
             ensure_future(loop(*_args, **_kwargs))
@@ -85,7 +113,6 @@ def repeat_every(
     *,
     seconds: float,
     wait_first: bool = False,
-    logger: logging.Logger = None,
     raise_exceptions: bool = False,
     max_repetitions: int = None,
 ):
@@ -97,17 +124,19 @@ def repeat_every(
     :param wait_first: bool (default False)
         Whether to wait `seconds` seconds before executing the function for the
         first time.
-    :param logger: logging.Logger (default None)
-        The logger to use for logging exceptions.
     :param raise_exceptions: bool (default False)
         Whether to raise exceptions instead of logging them.
     :param max_repetitions: int (default None)
         The maximum number of times to repeat the function. If None, the
         function will repeat indefinitely.
     """
+    if max_repetitions and max_repetitions <= 0:
+        raise ValueError(
+            "max_repetitions should more than zero if it want to set"
+        )
 
     def decorator(func):
-        is_coroutine = asyncio.iscoroutinefunction(func)
+        is_coroutine: bool = asyncio.iscoroutinefunction(func)
 
         @wraps(func)
         async def wrapper(*_args, **_kwargs):
@@ -115,19 +144,19 @@ def repeat_every(
 
             async def loop(*args, **kwargs):
                 nonlocal repetitions
+
                 if wait_first:
                     await asyncio.sleep(seconds)
+
                 while max_repetitions is None or repetitions < max_repetitions:
-                    try:
-                        if is_coroutine:
-                            await func(*args, **kwargs)
-                        else:
-                            await run_in_threadpool(func, *args, **kwargs)
-                    except Exception as e:
-                        if logger is not None:
-                            logger.exception(e)
-                        if raise_exceptions:
-                            raise e
+                    await run_func(
+                        is_coroutine,
+                        func,
+                        *args,
+                        raise_exceptions=raise_exceptions,
+                        **kwargs,
+                    )
+
                     repetitions += 1
                     await asyncio.sleep(seconds)