ddeutil-workflow 0.0.8__py3-none-any.whl → 0.0.10__py3-none-any.whl

ddeutil/workflow/exceptions.py

@@ -21,4 +21,7 @@ class JobException(WorkflowException): ...
 class PipelineException(WorkflowException): ...
 
 
+class PipelineFailException(WorkflowException): ...
+
+
 class ParamValueException(WorkflowException): ...

ddeutil/workflow/log.py

@@ -5,75 +5,180 @@
 # ------------------------------------------------------------------------------
 from __future__ import annotations
 
+import json
 import logging
+import os
+from abc import ABC, abstractmethod
 from datetime import datetime
 from functools import lru_cache
-from typing import Union
+from pathlib import Path
+from typing import Optional, Union
 
+from ddeutil.core import str2bool
 from pydantic import BaseModel, Field
-from rich.console import Console
-from rich.logging import RichHandler
+from pydantic.functional_validators import model_validator
+from typing_extensions import Self
 
 from .__types import DictData
-
-console = Console(color_system="256", width=200, style="blue")
+from .utils import config
 
 
 @lru_cache
-def get_logger(module_name):
-    logger = logging.getLogger(module_name)
-    handler = RichHandler(
-        rich_tracebacks=True, console=console, tracebacks_show_locals=True
-    )
-    handler.setFormatter(
-        logging.Formatter(
-            "[ %(threadName)s:%(funcName)s:%(process)d ] - %(message)s"
-        )
+def get_logger(name: str):
+    """Return logger with an input module name."""
+    logger = logging.getLogger(name)
+    formatter = logging.Formatter(
+        fmt=(
+            "%(asctime)s.%(msecs)03d (%(name)-10s, %(process)-5d, "
+            "%(thread)-5d) [%(levelname)-7s] %(message)-120s "
+            "(%(filename)s:%(lineno)s)"
+        ),
+        datefmt="%Y-%m-%d %H:%M:%S",
     )
-    logger.addHandler(handler)
-    logger.setLevel(logging.DEBUG)
+    stream = logging.StreamHandler()
+    stream.setFormatter(formatter)
+    logger.addHandler(stream)
+
+    debug: bool = str2bool(os.getenv("WORKFLOW_LOG_DEBUG_MODE", "true"))
+    logger.setLevel(logging.DEBUG if debug else logging.INFO)
     return logger
 
 
-class BaseLog(BaseModel):
-    """Base logging model."""
+class BaseLog(BaseModel, ABC):
+    """Base Log Pydantic Model abstraction that implement only model fields."""
 
-    parent_id: str
-    id: str
-    input: DictData
-    output: DictData
-    update_time: datetime = Field(default_factory=datetime.now)
+    name: str = Field(description="A pipeline name.")
+    on: str = Field(description="A cronjob string of this piepline schedule.")
+    release: datetime = Field(description="A release datetime.")
+    context: DictData = Field(
+        default_factory=dict,
+        description=(
+            "A context data that receive from a pipeline execution result.",
+        ),
+    )
+    parent_run_id: Optional[str] = Field(default=None)
+    run_id: str
+    update: datetime = Field(default_factory=datetime.now)
+
+    @model_validator(mode="after")
+    def __model_action(self):
+        if str2bool(os.getenv("WORKFLOW_LOG_ENABLE_WRITE", "false")):
+            self.do_before()
+        return self
+
+    def do_before(self) -> None:
+        """To something before end up of initial log model."""
+
+    @abstractmethod
+    def save(self, excluded: list[str] | None) -> None:
+        """Save logging"""
+        raise NotImplementedError("Log should implement ``save`` method.")
+
+
+class FileLog(BaseLog):
+    """File Log Pydantic Model that use to saving log data from result of
+    pipeline execution. It inherit from BaseLog model that implement the
+    ``self.save`` method for file.
+    """
+
+    def do_before(self) -> None:
+        """Create directory of release before saving log file."""
+        self.pointer().mkdir(parents=True, exist_ok=True)
+
+    @classmethod
+    def find_logs(cls, name: str):
+        pointer: Path = config().engine.paths.root / f"./logs/pipeline={name}"
+        for file in pointer.glob("./release=*/*.log"):
+            with file.open(mode="r", encoding="utf-8") as f:
+                yield json.load(f)
+
+    @classmethod
+    def find_log(cls, name: str, release: datetime | None = None):
+        if release is not None:
+            pointer: Path = (
+                config().engine.paths.root
+                / f"./logs/pipeline={name}/release={release:%Y%m%d%H%M%S}"
+            )
+            if not pointer.exists():
+                raise FileNotFoundError(
+                    f"Pointer: ./logs/pipeline={name}/"
+                    f"release={release:%Y%m%d%H%M%S} does not found."
+                )
+            return cls.model_validate(
+                obj=json.loads(pointer.read_text(encoding="utf-8"))
+            )
+        raise NotImplementedError("Find latest log does not implement yet.")
+
+    @classmethod
+    def is_pointed(
+        cls,
+        name: str,
+        release: datetime,
+        *,
+        queue: list[datetime] | None = None,
+    ) -> bool:
+        """Check this log already point in the destination.
+
+        :param name: A pipeline name.
+        :param release: A release datetime.
+        :param queue: A list of queue of datetime that already run in the
+            future.
+        """
+        # NOTE: Check environ variable was set for real writing.
+        if not str2bool(os.getenv("WORKFLOW_LOG_ENABLE_WRITE", "false")):
+            return False
+
+        # NOTE: create pointer path that use the same logic of pointer method.
+        pointer: Path = (
+            config().engine.paths.root
+            / f"./logs/pipeline={name}/release={release:%Y%m%d%H%M%S}"
+        )
+
+        if not queue:
+            return pointer.exists()
+        return pointer.exists() or (release in queue)
 
+    def pointer(self) -> Path:
+        """Return release directory path that was generated from model data.
 
-class StageLog(BaseLog): ...
+        :rtype: Path
+        """
+        return (
+            config().engine.paths.root
+            / f"./logs/pipeline={self.name}/release={self.release:%Y%m%d%H%M%S}"
+        )
 
+    def save(self, excluded: list[str] | None) -> Self:
+        """Save logging data that receive a context data from a pipeline
+        execution result.
+
+        :param excluded: An excluded list of key name that want to pass in the
+            model_dump method.
+        :rtype: Self
+        """
+        # NOTE: Check environ variable was set for real writing.
+        if not str2bool(os.getenv("WORKFLOW_LOG_ENABLE_WRITE", "false")):
+            return self
+
+        log_file: Path = self.pointer() / f"{self.run_id}.log"
+        log_file.write_text(
+            json.dumps(
+                self.model_dump(exclude=excluded),
+                default=str,
+                indent=2,
+            ),
+            encoding="utf-8",
+        )
+        return self
 
-class JobLog(BaseLog): ...
 
+class SQLiteLog(BaseLog):
 
-class PipelineLog(BaseLog): ...
+    def save(self, excluded: list[str] | None) -> None:
+        raise NotImplementedError("SQLiteLog does not implement yet.")
 
 
 Log = Union[
-    StageLog,
-    JobLog,
-    PipelineLog,
+    FileLog,
+    SQLiteLog,
 ]
-
-
-def push_log_memory(log: DictData):
-    """Push message log to globals log queue."""
-    print(log)
-
-
-LOGS_REGISTRY = {
-    "memory": push_log_memory,
-}
-
-
-def push_log(log: DictData, mode: str = "memory"):
-    return LOGS_REGISTRY[mode](log)
-
-
-def save_log():
-    """Save log that push to queue to target saving"""

ddeutil/workflow/on.py

@@ -10,22 +10,18 @@ from typing import Annotated, Literal
 from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
 
 from pydantic import BaseModel, ConfigDict, Field
+from pydantic.functional_serializers import field_serializer
 from pydantic.functional_validators import field_validator, model_validator
 from typing_extensions import Self
 
-try:
-    from .__types import DictData, DictStr
-    from .loader import Loader
-    from .scheduler import WEEKDAYS, CronJob, CronJobYear, CronRunner
-except ImportError:
-    from ddeutil.workflow.__types import DictData, DictStr
-    from ddeutil.workflow.loader import Loader
-    from ddeutil.workflow.scheduler import (
-        WEEKDAYS,
-        CronJob,
-        CronJobYear,
-        CronRunner,
-    )
+from .__types import DictData, DictStr, TupleStr
+from .cron import WEEKDAYS, CronJob, CronJobYear, CronRunner
+from .utils import Loader
+
+__all__: TupleStr = (
+    "On",
+    "interval2crontab",
+)
 
 
 def interval2crontab(
@@ -106,7 +102,7 @@ class On(BaseModel):
         passing["cronjob"] = interval2crontab(
             **{v: value[v] for v in value if v in ("interval", "day", "time")}
         )
-        return cls(extras=externals, **passing)
+        return cls(extras=externals | passing.pop("extras", {}), **passing)
 
     @classmethod
     def from_loader(
@@ -121,6 +117,7 @@ class On(BaseModel):
         :param externals: A extras external parameter that will keep in extras.
         """
         loader: Loader = Loader(name, externals=externals)
+
         # NOTE: Validate the config type match with current connection model
         if loader.type != cls:
             raise ValueError(f"Type {loader.type} does not match with {cls}")
@@ -136,7 +133,7 @@ class On(BaseModel):
                             if v in ("interval", "day", "time")
                         }
                     ),
-                    extras=externals,
+                    extras=externals | loader_data.pop("extras", {}),
                     **loader_data,
                 )
             )
@@ -145,19 +142,19 @@ class On(BaseModel):
         return cls.model_validate(
             obj=dict(
                 cronjob=loader_data.pop("cronjob"),
-                extras=externals,
+                extras=externals | loader_data.pop("extras", {}),
                 **loader_data,
             )
         )
 
     @model_validator(mode="before")
-    def __prepare_values(cls, values):
+    def __prepare_values(cls, values: DictData) -> DictData:
         if tz := values.pop("tz", None):
             values["timezone"] = tz
         return values
 
     @field_validator("tz")
-    def __validate_tz(cls, value: str):
+    def __validate_tz(cls, value: str) -> str:
         """Validate timezone value that able to initialize with ZoneInfo after
         it passing to this model in before mode."""
         try:
@@ -171,12 +168,24 @@ class On(BaseModel):
         """Prepare crontab value that able to receive with string type."""
         return CronJob(value) if isinstance(value, str) else value
 
+    @field_serializer("cronjob")
+    def __serialize_cronjob(self, value: CronJob) -> str:
+        return str(value)
+
     def generate(self, start: str | datetime) -> CronRunner:
         """Return Cron runner object."""
         if not isinstance(start, datetime):
             start: datetime = datetime.fromisoformat(start)
         return self.cronjob.schedule(date=start, tz=self.tz)
 
+    def next(self, start: str | datetime) -> datetime:
+        """Return a next datetime from Cron runner object that start with any
+        date that given from input.
+        """
+        if not isinstance(start, datetime):
+            start: datetime = datetime.fromisoformat(start)
+        return self.cronjob.schedule(date=start, tz=self.tz).next
+
 
 class AwsOn(On):
     """Implement On AWS Schedule for AWS Service like AWS Glue."""