ddeutil-workflow 0.0.8__py3-none-any.whl → 0.0.9__py3-none-any.whl

ddeutil/workflow/loader.py

@@ -5,6 +5,7 @@
 # ------------------------------------------------------------------------------
 from __future__ import annotations
 
+from collections.abc import Iterator
 from functools import cached_property
 from typing import TypeVar
 
@@ -35,18 +36,65 @@ class SimLoad:
         self,
         name: str,
         params: ConfParams,
-        externals: DictData,
+        externals: DictData | None = None,
     ) -> None:
         self.data: DictData = {}
         for file in PathSearch(params.engine.paths.conf).files:
-            if any(file.suffix.endswith(s) for s in ("yml", "yaml")) and (
+            if any(file.suffix.endswith(s) for s in (".yml", ".yaml")) and (
                 data := YamlFlResolve(file).read().get(name, {})
             ):
                 self.data = data
         if not self.data:
             raise ValueError(f"Config {name!r} does not found on conf path")
+
+        # TODO: Validate the version of template data that mean if version of
+        #   Template were change it should raise to upgrade package version.
+        #   ---
+        #   <pipeline-name>:
+        #       version: 1
+        #       type: pipeline.Pipeline
+        #
         self.conf_params: ConfParams = params
-        self.externals: DictData = externals
+        self.externals: DictData = externals or {}
+        self.data.update(self.externals)
+
+    @classmethod
+    def find(
+        cls,
+        obj: object,
+        params: ConfParams,
+        *,
+        include: list[str] | None = None,
+        exclude: list[str] | None = None,
+    ) -> Iterator[tuple[str, DictData]]:
+        """Find all object"""
+        exclude: list[str] = exclude or []
+        for file in PathSearch(params.engine.paths.conf).files:
+            if any(file.suffix.endswith(s) for s in (".yml", ".yaml")) and (
+                values := YamlFlResolve(file).read()
+            ):
+                for key, data in values.items():
+                    if key in exclude:
+                        continue
+                    if (
+                        (t := data.get("type"))
+                        and issubclass(cls.get_type(t, params), obj)
+                        and all(i in data for i in (include or data.keys()))
+                    ):
+                        yield key, data
+
+    @classmethod
+    def get_type(cls, t: str, params: ConfParams) -> AnyModelType:
+        try:
+            # NOTE: Auto adding module prefix if it does not set
+            return import_string(f"ddeutil.workflow.{t}")
+        except ModuleNotFoundError:
+            for registry in params.engine.registry:
+                try:
+                    return import_string(f"{registry}.{t}")
+                except ModuleNotFoundError:
+                    continue
+            return import_string(f"{t}")
 
     @cached_property
     def type(self) -> AnyModelType:
@@ -57,16 +105,7 @@ class SimLoad:
             raise ValueError(
                 f"the 'type' value: {_typ} does not exists in config data."
             )
-        try:
-            # NOTE: Auto adding module prefix if it does not set
-            return import_string(f"ddeutil.workflow.{_typ}")
-        except ModuleNotFoundError:
-            for registry in self.conf_params.engine.registry:
-                try:
-                    return import_string(f"{registry}.{_typ}")
-                except ModuleNotFoundError:
-                    continue
-            return import_string(f"{_typ}")
+        return self.get_type(_typ, self.conf_params)
 
 
 class Loader(SimLoad):
@@ -76,5 +115,18 @@ class Loader(SimLoad):
     :param externals: An external parameters
     """
 
+    @classmethod
+    def find(
+        cls,
+        obj,
+        *,
+        include: list[str] | None = None,
+        exclude: list[str] | None = None,
+        **kwargs,
+    ) -> DictData:
+        return super().find(
+            obj=obj, params=config(), include=include, exclude=exclude
+        )
+
     def __init__(self, name: str, externals: DictData) -> None:
         super().__init__(name, config(), externals)

ddeutil/workflow/log.py

@@ -5,75 +5,173 @@
 # ------------------------------------------------------------------------------
 from __future__ import annotations
 
-import logging
+import json
+import os
+import re
+from abc import ABC, abstractmethod
 from datetime import datetime
-from functools import lru_cache
-from typing import Union
+from heapq import heappop, heappush
+from pathlib import Path
+from typing import Optional, Union
 
+from ddeutil.core import str2bool
 from pydantic import BaseModel, Field
-from rich.console import Console
-from rich.logging import RichHandler
+from pydantic.functional_validators import model_validator
 
 from .__types import DictData
+from .utils import config
 
-console = Console(color_system="256", width=200, style="blue")
 
+class BaseLog(BaseModel, ABC):
+    """Base Log Pydantic Model"""
 
-@lru_cache
-def get_logger(module_name):
-    logger = logging.getLogger(module_name)
-    handler = RichHandler(
-        rich_tracebacks=True, console=console, tracebacks_show_locals=True
+    name: str = Field(description="A pipeline name.")
+    on: str = Field(description="A cronjob string of this piepline schedule.")
+    release: datetime = Field(description="A release datetime.")
+    context: DictData = Field(
+        default_factory=dict,
+        description=(
+            "A context data that receive from a pipeline execution result.",
+        ),
     )
-    handler.setFormatter(
-        logging.Formatter(
-            "[ %(threadName)s:%(funcName)s:%(process)d ] - %(message)s"
+    parent_run_id: Optional[str] = Field(default=None)
+    run_id: str
+    update: datetime = Field(default_factory=datetime.now)
+
+    @model_validator(mode="after")
+    def __model_action(self):
+        if str2bool(os.getenv("WORKFLOW_LOG_ENABLE_WRITE", "false")):
+            self.do_before()
+        return self
+
+    def do_before(self) -> None:
+        """To something before end up of initial log model."""
+        return
+
+    @abstractmethod
+    def save(self) -> None:
+        """Save logging"""
+        raise NotImplementedError("Log should implement ``save`` method.")
+
+
+class FileLog(BaseLog):
+    """File Log Pydantic Model that use to saving log data from result of
+    pipeline execution. It inherit from BaseLog model that implement the
+    ``self.save`` method for file.
+    """
+
+    def do_before(self) -> None:
+        """Create directory of release before saving log file."""
+        self.pointer().mkdir(parents=True, exist_ok=True)
+
+    @classmethod
+    def latest_point(
+        cls,
+        name: str,
+        *,
+        queue: list[datetime] | None = None,
+    ) -> datetime | None:
+        """Return latest point that exist in current logging pointer keeping.
+
+        :param name: A pipeline name
+        :param queue: A release queue.
+        """
+        keeping: Path = config().engine.paths.root / f"./logs/pipeline={name}/"
+        if not keeping.exists():
+            return None
+
+        keeping_files: list[int] = [
+            int(found.stem)
+            for found in keeping.glob("*")
+            if found.is_dir() and re.match(r"\d{14}", found.stem)
+        ]
+
+        latest = max(keeping_files or [None])
+
+        if not queue:
+            if latest is None:
+                return None
+            return datetime.strptime(str(latest), "%Y%m%d%H%M%S")
+
+        latest_queue: datetime = max(queue)
+
+        if latest is None:
+            return latest_queue
+
+        latest_dt: datetime = datetime.strptime(
+            str(latest), "%Y%m%d%H%M%S"
+        ).replace(tzinfo=latest_queue.tzinfo)
+        return max(latest_dt, latest_queue)
+
+    @classmethod
+    def is_pointed(
+        cls,
+        name: str,
+        release: datetime,
+        *,
+        queue: list[datetime] | None = None,
+    ) -> bool:
+        """Check this log already point.
+
+        :param name: A pipeline name.
+        :param release: A release datetime.
+        :param queue: A list of queue of datetime that already run in the
+            future.
+        """
+        if not str2bool(os.getenv("WORKFLOW_LOG_ENABLE_WRITE", "false")):
+            return False
+
+        # NOTE: create pointer path that use the same logic of pointer method.
+        pointer: Path = (
+            config().engine.paths.root
+            / f"./logs/pipeline={name}/release={release:%Y%m%d%H%M%S}"
         )
-    )
-    logger.addHandler(handler)
-    logger.setLevel(logging.DEBUG)
-    return logger
 
+        if queue is None:
+            return pointer.exists()
 
-class BaseLog(BaseModel):
-    """Base logging model."""
+        if pointer.exists() and not queue:
+            return True
 
-    parent_id: str
-    id: str
-    input: DictData
-    output: DictData
-    update_time: datetime = Field(default_factory=datetime.now)
+        if queue:
+            latest: datetime = heappop(queue)
+            heappush(queue, latest)
+            if release == latest:
+                return True
 
+        return False
 
-class StageLog(BaseLog): ...
+    def pointer(self) -> Path:
+        """Return release directory path that was generated from model data."""
+        return (
+            config().engine.paths.root
+            / f"./logs/pipeline={self.name}/release={self.release:%Y%m%d%H%M%S}"
+        )
 
+    def save(self) -> None:
+        """Save logging data that receive a context data from a pipeline
+        execution result.
+        """
+        if not str2bool(os.getenv("WORKFLOW_LOG_ENABLE_WRITE", "false")):
+            return
+
+        log_file: Path = self.pointer() / f"{self.run_id}.log"
+        log_file.write_text(
+            json.dumps(
+                self.model_dump(),
+                default=str,
+            ),
+            encoding="utf-8",
+        )
 
-class JobLog(BaseLog): ...
 
+class SQLiteLog(BaseLog):
 
-class PipelineLog(BaseLog): ...
+    def save(self) -> None:
+        raise NotImplementedError("SQLiteLog does not implement yet.")
 
 
 Log = Union[
-    StageLog,
-    JobLog,
-    PipelineLog,
+    FileLog,
+    SQLiteLog,
 ]
-
-
-def push_log_memory(log: DictData):
-    """Push message log to globals log queue."""
-    print(log)
-
-
-LOGS_REGISTRY = {
-    "memory": push_log_memory,
-}
-
-
-def push_log(log: DictData, mode: str = "memory"):
-    return LOGS_REGISTRY[mode](log)
-
-
-def save_log():
-    """Save log that push to queue to target saving"""

ddeutil/workflow/on.py

@@ -13,19 +13,14 @@ from pydantic import BaseModel, ConfigDict, Field
 from pydantic.functional_validators import field_validator, model_validator
 from typing_extensions import Self
 
-try:
-    from .__types import DictData, DictStr
-    from .loader import Loader
-    from .scheduler import WEEKDAYS, CronJob, CronJobYear, CronRunner
-except ImportError:
-    from ddeutil.workflow.__types import DictData, DictStr
-    from ddeutil.workflow.loader import Loader
-    from ddeutil.workflow.scheduler import (
-        WEEKDAYS,
-        CronJob,
-        CronJobYear,
-        CronRunner,
-    )
+from .__types import DictData, DictStr, TupleStr
+from .cron import WEEKDAYS, CronJob, CronJobYear, CronRunner
+from .loader import Loader
+
+__all__: TupleStr = (
+    "On",
+    "interval2crontab",
+)
 
 
 def interval2crontab(
@@ -151,13 +146,13 @@ class On(BaseModel):
         )
 
     @model_validator(mode="before")
-    def __prepare_values(cls, values):
+    def __prepare_values(cls, values: DictData) -> DictData:
         if tz := values.pop("tz", None):
             values["timezone"] = tz
         return values
 
     @field_validator("tz")
-    def __validate_tz(cls, value: str):
+    def __validate_tz(cls, value: str) -> str:
         """Validate timezone value that able to initialize with ZoneInfo after
         it passing to this model in before mode."""
         try:
@@ -177,6 +172,14 @@ class On(BaseModel):
             start: datetime = datetime.fromisoformat(start)
         return self.cronjob.schedule(date=start, tz=self.tz)
 
+    def next(self, start: str | datetime) -> datetime:
+        """Return a next datetime from Cron runner object that start with any
+        date that given from input.
+        """
+        if not isinstance(start, datetime):
+            start: datetime = datetime.fromisoformat(start)
+        return self.cronjob.schedule(date=start, tz=self.tz).next
+
 
 class AwsOn(On):
     """Implement On AWS Schedule for AWS Service like AWS Glue."""