ddeutil-workflow 0.0.9__py3-none-any.whl → 0.0.10__py3-none-any.whl

ddeutil/workflow/log.py

@@ -6,24 +6,46 @@
 from __future__ import annotations
 
 import json
+import logging
 import os
-import re
 from abc import ABC, abstractmethod
 from datetime import datetime
-from heapq import heappop, heappush
+from functools import lru_cache
 from pathlib import Path
 from typing import Optional, Union
 
 from ddeutil.core import str2bool
 from pydantic import BaseModel, Field
 from pydantic.functional_validators import model_validator
+from typing_extensions import Self
 
 from .__types import DictData
 from .utils import config
 
 
+@lru_cache
+def get_logger(name: str):
+    """Return logger with an input module name."""
+    logger = logging.getLogger(name)
+    formatter = logging.Formatter(
+        fmt=(
+            "%(asctime)s.%(msecs)03d (%(name)-10s, %(process)-5d, "
+            "%(thread)-5d) [%(levelname)-7s] %(message)-120s "
+            "(%(filename)s:%(lineno)s)"
+        ),
+        datefmt="%Y-%m-%d %H:%M:%S",
+    )
+    stream = logging.StreamHandler()
+    stream.setFormatter(formatter)
+    logger.addHandler(stream)
+
+    debug: bool = str2bool(os.getenv("WORKFLOW_LOG_DEBUG_MODE", "true"))
+    logger.setLevel(logging.DEBUG if debug else logging.INFO)
+    return logger
+
+
 class BaseLog(BaseModel, ABC):
-    """Base Log Pydantic Model"""
+    """Base Log Pydantic Model abstraction that implement only model fields."""
 
     name: str = Field(description="A pipeline name.")
     on: str = Field(description="A cronjob string of this piepline schedule.")
@@ -46,10 +68,9 @@ class BaseLog(BaseModel, ABC):
 
     def do_before(self) -> None:
         """To something before end up of initial log model."""
-        return
 
     @abstractmethod
-    def save(self) -> None:
+    def save(self, excluded: list[str] | None) -> None:
         """Save logging"""
         raise NotImplementedError("Log should implement ``save`` method.")
 
@@ -65,43 +86,28 @@ class FileLog(BaseLog):
         self.pointer().mkdir(parents=True, exist_ok=True)
 
     @classmethod
-    def latest_point(
-        cls,
-        name: str,
-        *,
-        queue: list[datetime] | None = None,
-    ) -> datetime | None:
-        """Return latest point that exist in current logging pointer keeping.
-
-        :param name: A pipeline name
-        :param queue: A release queue.
-        """
-        keeping: Path = config().engine.paths.root / f"./logs/pipeline={name}/"
-        if not keeping.exists():
-            return None
-
-        keeping_files: list[int] = [
-            int(found.stem)
-            for found in keeping.glob("*")
-            if found.is_dir() and re.match(r"\d{14}", found.stem)
-        ]
-
-        latest = max(keeping_files or [None])
+    def find_logs(cls, name: str):
+        pointer: Path = config().engine.paths.root / f"./logs/pipeline={name}"
+        for file in pointer.glob("./release=*/*.log"):
+            with file.open(mode="r", encoding="utf-8") as f:
+                yield json.load(f)
 
-        if not queue:
-            if latest is None:
-                return None
-            return datetime.strptime(str(latest), "%Y%m%d%H%M%S")
-
-        latest_queue: datetime = max(queue)
-
-        if latest is None:
-            return latest_queue
-
-        latest_dt: datetime = datetime.strptime(
-            str(latest), "%Y%m%d%H%M%S"
-        ).replace(tzinfo=latest_queue.tzinfo)
-        return max(latest_dt, latest_queue)
+    @classmethod
+    def find_log(cls, name: str, release: datetime | None = None):
+        if release is not None:
+            pointer: Path = (
+                config().engine.paths.root
+                / f"./logs/pipeline={name}/release={release:%Y%m%d%H%M%S}"
+            )
+            if not pointer.exists():
+                raise FileNotFoundError(
+                    f"Pointer: ./logs/pipeline={name}/"
+                    f"release={release:%Y%m%d%H%M%S} does not found."
+                )
+            return cls.model_validate(
+                obj=json.loads(pointer.read_text(encoding="utf-8"))
+            )
+        raise NotImplementedError("Find latest log does not implement yet.")
 
     @classmethod
     def is_pointed(
@@ -111,13 +117,14 @@ class FileLog(BaseLog):
         *,
         queue: list[datetime] | None = None,
     ) -> bool:
-        """Check this log already point.
+        """Check this log already point in the destination.
 
         :param name: A pipeline name.
         :param release: A release datetime.
         :param queue: A list of queue of datetime that already run in the
             future.
         """
+        # NOTE: Check environ variable was set for real writing.
         if not str2bool(os.getenv("WORKFLOW_LOG_ENABLE_WRITE", "false")):
             return False
 
@@ -127,47 +134,47 @@ class FileLog(BaseLog):
             / f"./logs/pipeline={name}/release={release:%Y%m%d%H%M%S}"
         )
 
-        if queue is None:
+        if not queue:
             return pointer.exists()
-
-        if pointer.exists() and not queue:
-            return True
-
-        if queue:
-            latest: datetime = heappop(queue)
-            heappush(queue, latest)
-            if release == latest:
-                return True
-
-        return False
+        return pointer.exists() or (release in queue)
 
     def pointer(self) -> Path:
-        """Return release directory path that was generated from model data."""
+        """Return release directory path that was generated from model data.
+
+        :rtype: Path
+        """
         return (
             config().engine.paths.root
             / f"./logs/pipeline={self.name}/release={self.release:%Y%m%d%H%M%S}"
         )
 
-    def save(self) -> None:
+    def save(self, excluded: list[str] | None) -> Self:
         """Save logging data that receive a context data from a pipeline
         execution result.
+
+        :param excluded: An excluded list of key name that want to pass in the
+            model_dump method.
+        :rtype: Self
         """
+        # NOTE: Check environ variable was set for real writing.
         if not str2bool(os.getenv("WORKFLOW_LOG_ENABLE_WRITE", "false")):
-            return
+            return self
 
         log_file: Path = self.pointer() / f"{self.run_id}.log"
         log_file.write_text(
             json.dumps(
-                self.model_dump(),
+                self.model_dump(exclude=excluded),
                 default=str,
+                indent=2,
             ),
             encoding="utf-8",
         )
+        return self
 
 
 class SQLiteLog(BaseLog):
 
-    def save(self) -> None:
+    def save(self, excluded: list[str] | None) -> None:
         raise NotImplementedError("SQLiteLog does not implement yet.")
 
 

ddeutil/workflow/on.py

@@ -10,12 +10,13 @@ from typing import Annotated, Literal
 from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
 
 from pydantic import BaseModel, ConfigDict, Field
+from pydantic.functional_serializers import field_serializer
 from pydantic.functional_validators import field_validator, model_validator
 from typing_extensions import Self
 
 from .__types import DictData, DictStr, TupleStr
 from .cron import WEEKDAYS, CronJob, CronJobYear, CronRunner
-from .loader import Loader
+from .utils import Loader
 
 __all__: TupleStr = (
     "On",
@@ -101,7 +102,7 @@ class On(BaseModel):
         passing["cronjob"] = interval2crontab(
             **{v: value[v] for v in value if v in ("interval", "day", "time")}
         )
-        return cls(extras=externals, **passing)
+        return cls(extras=externals | passing.pop("extras", {}), **passing)
 
     @classmethod
     def from_loader(
@@ -116,6 +117,7 @@ class On(BaseModel):
         :param externals: A extras external parameter that will keep in extras.
         """
         loader: Loader = Loader(name, externals=externals)
+
         # NOTE: Validate the config type match with current connection model
         if loader.type != cls:
             raise ValueError(f"Type {loader.type} does not match with {cls}")
@@ -131,7 +133,7 @@ class On(BaseModel):
                             if v in ("interval", "day", "time")
                         }
                     ),
-                    extras=externals,
+                    extras=externals | loader_data.pop("extras", {}),
                     **loader_data,
                 )
             )
@@ -140,7 +142,7 @@ class On(BaseModel):
         return cls.model_validate(
             obj=dict(
                 cronjob=loader_data.pop("cronjob"),
-                extras=externals,
+                extras=externals | loader_data.pop("extras", {}),
                 **loader_data,
             )
         )
@@ -166,6 +168,10 @@ class On(BaseModel):
         """Prepare crontab value that able to receive with string type."""
         return CronJob(value) if isinstance(value, str) else value
 
+    @field_serializer("cronjob")
+    def __serialize_cronjob(self, value: CronJob) -> str:
+        return str(value)
+
     def generate(self, start: str | datetime) -> CronRunner:
         """Return Cron runner object."""
         if not isinstance(start, datetime):