ddeutil-workflow 0.0.16__py3-none-any.whl → 0.0.18__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1 +1 @@
-__version__: str = "0.0.16"
+__version__: str = "0.0.18"

ddeutil/workflow/{cron.py → __cron.py}

@@ -18,10 +18,7 @@ from ddeutil.core import (
     isinstance_check,
     must_split,
 )
-from ddeutil.core.dtutils import (
-    next_date,
-    replace_date,
-)
+from ddeutil.core.dtutils import next_date, replace_date
 
 WEEKDAYS: dict[str, int] = {
     "Sun": 0,
@@ -37,7 +34,7 @@ WEEKDAYS: dict[str, int] = {
 class CronYearLimit(Exception): ...
 
 
-def str2cron(value: str) -> str:
+def str2cron(value: str) -> str:  # pragma: no cov
     """Convert Special String with the @ prefix to Crontab value.
 
     :param value: A string value that want to convert to cron value.
@@ -69,6 +66,8 @@ def str2cron(value: str) -> str:
 
 @dataclass(frozen=True)
 class Unit:
+    """Unit dataclass for CronPart object."""
+
     name: str
     range: partial
     min: int
@@ -85,6 +84,8 @@ class Unit:
 
 @dataclass
 class Options:
+    """Options dataclass for config CronPart object."""
+
     output_weekday_names: bool = False
     output_month_names: bool = False
     output_hashes: bool = False
@@ -158,7 +159,12 @@ CRON_UNITS_YEAR: tuple[Unit, ...] = CRON_UNITS + (
 
 @total_ordering
 class CronPart:
-    """Part of Cron object that represent a collection of positive integers."""
+    """Part of Cron object that represent a collection of positive integers.
+
+    :param unit: A Unit dataclass object.
+    :param values: A crontab values that want to validate
+    :param options: A Options dataclass object.
+    """
 
     __slots__: tuple[str, ...] = (
         "unit",

ddeutil/workflow/__init__.py

@@ -3,6 +3,11 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
+from .conf import (
+    Config,
+    FileLog,
+    Loader,
+)
 from .exceptions import (
     JobException,
     ParamValueException,
@@ -11,14 +16,57 @@ from .exceptions import (
     WorkflowException,
 )
 from .job import Job, Strategy
-from .on import On, interval2crontab
+from .on import (
+    On,
+    YearOn,
+    interval2crontab,
+)
 from .scheduler import (
     Schedule,
+    ScheduleWorkflow,
     Workflow,
+    WorkflowTaskData,
+)
+from .stage import (
+    BashStage,
+    EmptyStage,
+    HookStage,
+    PyStage,
+    Stage,
+    TriggerStage,
+    handler_result,
 )
-from .stage import Stage, handler_result
 from .utils import (
+    FILTERS,
+    ChoiceParam,
+    DatetimeParam,
+    DefaultParam,
+    FilterFunc,
+    FilterRegistry,
+    IntParam,
     Param,
+    Result,
+    ReturnTagFunc,
+    StrParam,
+    TagFunc,
+    batch,
+    cross_product,
+    custom_filter,
     dash2underscore,
+    delay,
+    filter_func,
+    gen_id,
+    get_args_const,
+    get_diff_sec,
+    get_dt_now,
+    has_template,
+    make_exec,
+    make_filter_registry,
+    make_registry,
+    map_post_filter,
+    not_in_template,
     param2template,
+    queue2str,
+    str2template,
+    tag,
 )

ddeutil/workflow/api.py

@@ -21,8 +21,7 @@ from fastapi.responses import UJSONResponse
 from pydantic import BaseModel
 
 from .__about__ import __version__
-from .conf import config
-from .log import get_logger
+from .conf import config, get_logger
 from .repeat import repeat_at, repeat_every
 from .scheduler import WorkflowTaskData
 

ddeutil/workflow/cli.py

@@ -13,8 +13,7 @@ from typing import Annotated, Optional
 from ddeutil.core import str2list
 from typer import Argument, Option, Typer
 
-from .conf import config
-from .log import get_logger
+from .conf import config, get_logger
 
 logger = get_logger("ddeutil.workflow")
 cli: Typer = Typer()

ddeutil/workflow/conf.py

@@ -6,26 +6,32 @@
 from __future__ import annotations
 
 import json
+import logging
 import os
+from abc import ABC, abstractmethod
 from collections.abc import Iterator
-from datetime import timedelta
-from functools import cached_property
+from datetime import datetime, timedelta
+from functools import cached_property, lru_cache
 from pathlib import Path
-from typing import Any, TypeVar
+from typing import ClassVar, Optional, TypeVar, Union
 from zoneinfo import ZoneInfo
 
 from ddeutil.core import import_string, str2bool
-from ddeutil.io import Paths, PathSearch, YamlFlResolve
+from ddeutil.io import PathSearch, YamlFlResolve
 from dotenv import load_dotenv
 from pydantic import BaseModel, Field
 from pydantic.functional_validators import model_validator
+from typing_extensions import Self
+
+from .__types import DictData
 
-load_dotenv()
-env = os.getenv
-DictData = dict[str, Any]
 AnyModel = TypeVar("AnyModel", bound=BaseModel)
 AnyModelType = type[AnyModel]
 
+load_dotenv()
+
+env = os.getenv
+
 
 class Config:
     """Config object for keeping application configuration on current session
@@ -35,8 +41,16 @@ class Config:
     # NOTE: Core
     root_path: Path = Path(os.getenv("WORKFLOW_ROOT_PATH", "."))
     tz: ZoneInfo = ZoneInfo(env("WORKFLOW_CORE_TIMEZONE", "UTC"))
-    workflow_id_simple_mode: bool = str2bool(
-        os.getenv("WORKFLOW_CORE_WORKFLOW_ID_SIMPLE_MODE", "true")
+    gen_id_simple_mode: bool = str2bool(
+        os.getenv("WORKFLOW_CORE_GENERATE_ID_SIMPLE_MODE", "true")
+    )
+
+    # NOTE: Register
+    regis_hook_str: str = os.getenv(
+        "WORKFLOW_CORE_REGISTRY", "ddeutil.workflow"
+    )
+    regis_filter_str: str = os.getenv(
+        "WORKFLOW_CORE_REGISTRY_FILTER", "ddeutil.workflow.utils"
     )
 
     # NOTE: Logging
@@ -54,6 +68,9 @@ class Config:
     )
 
     # NOTE: Job
+    job_raise_error: bool = str2bool(
+        env("WORKFLOW_CORE_JOB_RAISE_ERROR", "true")
+    )
     job_default_id: bool = str2bool(
         env("WORKFLOW_CORE_JOB_DEFAULT_ID", "false")
     )
@@ -69,7 +86,7 @@ class Config:
     max_schedule_per_process: int = int(
         env("WORKFLOW_APP_MAX_SCHEDULE_PER_PROCESS", "100")
     )
-    __stop_boundary_delta: str = env(
+    stop_boundary_delta_str: str = env(
         "WORKFLOW_APP_STOP_BOUNDARY_DELTA", '{"minutes": 5, "seconds": 20}'
     )
 
@@ -89,93 +106,37 @@ class Config:
             )
         try:
             self.stop_boundary_delta: timedelta = timedelta(
-                **json.loads(self.__stop_boundary_delta)
+                **json.loads(self.stop_boundary_delta_str)
             )
         except Exception as err:
             raise ValueError(
                 "Config ``WORKFLOW_APP_STOP_BOUNDARY_DELTA`` can not parsing to"
-                f"timedelta with {self.__stop_boundary_delta}."
+                f"timedelta with {self.stop_boundary_delta_str}."
             ) from err
 
-    def refresh_dotenv(self):
+    def refresh_dotenv(self) -> Self:
         """Reload environment variables from the current stage."""
         self.tz: ZoneInfo = ZoneInfo(env("WORKFLOW_CORE_TIMEZONE", "UTC"))
         self.stage_raise_error: bool = str2bool(
             env("WORKFLOW_CORE_STAGE_RAISE_ERROR", "false")
         )
+        return self
 
+    @property
+    def conf_path(self) -> Path:
+        """Config path that use root_path class argument for this construction.
 
-class Engine(BaseModel):
-    """Engine Pydantic Model for keeping application path."""
-
-    paths: Paths = Field(default_factory=Paths)
-    registry: list[str] = Field(
-        default_factory=lambda: ["ddeutil.workflow"],  # pragma: no cover
-    )
-    registry_filter: list[str] = Field(
-        default_factory=lambda: ["ddeutil.workflow.utils"],  # pragma: no cover
-    )
-
-    @model_validator(mode="before")
-    def __prepare_registry(cls, values: DictData) -> DictData:
-        """Prepare registry value that passing with string type. It convert the
-        string type to list of string.
+        :rtype: Path
         """
-        if (_regis := values.get("registry")) and isinstance(_regis, str):
-            values["registry"] = [_regis]
-        if (_regis_filter := values.get("registry_filter")) and isinstance(
-            _regis_filter, str
-        ):
-            values["registry_filter"] = [_regis_filter]
-        return values
-
-
-class ConfParams(BaseModel):
-    """Params Model"""
-
-    engine: Engine = Field(
-        default_factory=Engine,
-        description="A engine mapping values.",
-    )
-
-
-def load_config() -> ConfParams:
-    """Load Config data from ``workflows-conf.yaml`` file.
+        return self.root_path / os.getenv("WORKFLOW_CORE_PATH_CONF", "conf")
 
-    Configuration Docs:
-    ---
-    :var engine.registry:
-    :var engine.registry_filter:
-    :var paths.root:
-    :var paths.conf:
-    """
-    root_path: str = config.root_path
+    @property
+    def regis_hook(self) -> list[str]:
+        return [r.strip() for r in self.regis_hook_str.split(",")]
 
-    regis: list[str] = ["ddeutil.workflow"]
-    if regis_env := os.getenv("WORKFLOW_CORE_REGISTRY"):
-        regis = [r.strip() for r in regis_env.split(",")]
-
-    regis_filter: list[str] = ["ddeutil.workflow.utils"]
-    if regis_filter_env := os.getenv("WORKFLOW_CORE_REGISTRY_FILTER"):
-        regis_filter = [r.strip() for r in regis_filter_env.split(",")]
-
-    conf_path: str = (
-        f"{root_path}/{conf_env}"
-        if (conf_env := os.getenv("WORKFLOW_CORE_PATH_CONF"))
-        else None
-    )
-    return ConfParams.model_validate(
-        obj={
-            "engine": {
-                "registry": regis,
-                "registry_filter": regis_filter,
-                "paths": {
-                    "root": root_path,
-                    "conf": conf_path,
-                },
-            },
-        }
-    )
+    @property
+    def regis_filter(self) -> list[str]:
+        return [r.strip() for r in self.regis_filter_str.split(",")]
 
 
 class SimLoad:
@@ -183,7 +144,7 @@ class SimLoad:
     value like name of workflow or on.
 
     :param name: A name of config data that will read by Yaml Loader object.
-    :param params: A Params model object.
+    :param conf: A Params model object.
     :param externals: An external parameters
 
     Noted:
@@ -201,21 +162,19 @@ class SimLoad:
     def __init__(
         self,
         name: str,
-        params: ConfParams,
+        conf: Config,
         externals: DictData | None = None,
     ) -> None:
         self.data: DictData = {}
-        for file in PathSearch(params.engine.paths.conf).files:
-            if any(file.suffix.endswith(s) for s in (".yml", ".yaml")) and (
-                data := YamlFlResolve(file).read().get(name, {})
-            ):
+        for file in PathSearch(conf.conf_path).files:
+            if data := self.filter_suffix(file, name):
                 self.data = data
 
         # VALIDATE: check the data that reading should not empty.
         if not self.data:
             raise ValueError(f"Config {name!r} does not found on conf path")
 
-        self.conf_params: ConfParams = params
+        self.conf: Config = conf
         self.externals: DictData = externals or {}
         self.data.update(self.externals)
 
@@ -223,7 +182,7 @@ class SimLoad:
     def finds(
         cls,
         obj: object,
-        params: ConfParams,
+        conf: Config,
         *,
         include: list[str] | None = None,
         exclude: list[str] | None = None,
@@ -233,23 +192,31 @@ class SimLoad:
         adds-on.
 
         :param obj: A object that want to validate matching before return.
-        :param params:
+        :param conf: A config object.
         :param include:
         :param exclude:
         :rtype: Iterator[tuple[str, DictData]]
         """
         exclude: list[str] = exclude or []
-        for file in PathSearch(params.engine.paths.conf).files:
-            if any(file.suffix.endswith(s) for s in (".yml", ".yaml")) and (
-                values := YamlFlResolve(file).read()
-            ):
-                for key, data in values.items():
-                    if key in exclude:
-                        continue
-                    if issubclass(get_type(data["type"], params), obj) and (
-                        include is None or all(i in data for i in include)
-                    ):
-                        yield key, data
+        for file in PathSearch(conf.conf_path).files:
+            for key, data in cls.filter_suffix(file).items():
+
+                if key in exclude:
+                    continue
+
+                if issubclass(get_type(data["type"], conf), obj):
+                    yield key, (
+                        {k: data[k] for k in data if k in include}
+                        if include
+                        else data
+                    )
+
+    @classmethod
+    def filter_suffix(cls, file: Path, name: str | None = None) -> DictData:
+        if any(file.suffix.endswith(s) for s in (".yml", ".yaml")):
+            values: DictData = YamlFlResolve(file).read()
+            return values.get(name, {}) if name else values
+        return {}
 
     @cached_property
     def type(self) -> AnyModelType:
@@ -258,11 +225,11 @@ class SimLoad:
 
         :rtype: AnyModelType
         """
-        if not (_typ := self.data.get("type")):
-            raise ValueError(
-                f"the 'type' value: {_typ} does not exists in config data."
-            )
-        return get_type(_typ, self.conf_params)
+        if _typ := self.data.get("type"):
+            return get_type(_typ, self.conf)
+        raise ValueError(
+            f"the 'type' value: {_typ} does not exists in config data."
+        )
 
 
 class Loader(SimLoad):
@@ -288,14 +255,14 @@ class Loader(SimLoad):
         :param exclude:
         """
         return super().finds(
-            obj=obj, params=load_config(), include=include, exclude=exclude
+            obj=obj, conf=Config(), include=include, exclude=exclude
         )
 
     def __init__(self, name: str, externals: DictData) -> None:
-        super().__init__(name, load_config(), externals)
+        super().__init__(name, conf=Config(), externals=externals)
 
 
-def get_type(t: str, params: ConfParams) -> AnyModelType:
+def get_type(t: str, params: Config) -> AnyModelType:
     """Return import type from string importable value in the type key.
 
     :param t: A importable type string.
@@ -307,7 +274,7 @@ def get_type(t: str, params: ConfParams) -> AnyModelType:
         # NOTE: Auto adding module prefix if it does not set
         return import_string(f"ddeutil.workflow.{t}")
     except ModuleNotFoundError:
-        for registry in params.engine.registry:
+        for registry in params.regis_hook:
             try:
                 return import_string(f"{registry}.{t}")
             except ModuleNotFoundError:
@@ -316,3 +283,201 @@ def get_type(t: str, params: ConfParams) -> AnyModelType:
 
 
 config = Config()
+
+
+@lru_cache
+def get_logger(name: str):
+    """Return logger object with an input module name.
+
+    :param name: A module name that want to log.
+    """
+    logger = logging.getLogger(name)
+    formatter = logging.Formatter(
+        fmt=(
+            "%(asctime)s.%(msecs)03d (%(name)-10s, %(process)-5d, "
+            "%(thread)-5d) [%(levelname)-7s] %(message)-120s "
+            "(%(filename)s:%(lineno)s)"
+        ),
+        datefmt="%Y-%m-%d %H:%M:%S",
+    )
+    stream = logging.StreamHandler()
+    stream.setFormatter(formatter)
+    logger.addHandler(stream)
+
+    logger.setLevel(logging.DEBUG if config.debug else logging.INFO)
+    return logger
+
+
+class BaseLog(BaseModel, ABC):
+    """Base Log Pydantic Model with abstraction class property that implement
+    only model fields. This model should to use with inherit to logging
+    sub-class like file, sqlite, etc.
+    """
+
+    name: str = Field(description="A workflow name.")
+    on: str = Field(description="A cronjob string of this piepline schedule.")
+    release: datetime = Field(description="A release datetime.")
+    context: DictData = Field(
+        default_factory=dict,
+        description=(
+            "A context data that receive from a workflow execution result.",
+        ),
+    )
+    parent_run_id: Optional[str] = Field(default=None)
+    run_id: str
+    update: datetime = Field(default_factory=datetime.now)
+
+    @model_validator(mode="after")
+    def __model_action(self) -> Self:
+        """Do before the Log action with WORKFLOW_LOG_ENABLE_WRITE env variable.
+
+        :rtype: Self
+        """
+        if config.enable_write_log:
+            self.do_before()
+        return self
+
+    def do_before(self) -> None:  # pragma: no cov
+        """To something before end up of initial log model."""
+
+    @abstractmethod
+    def save(self, excluded: list[str] | None) -> None:  # pragma: no cov
+        """Save this model logging to target logging store."""
+        raise NotImplementedError("Log should implement ``save`` method.")
+
+
+class FileLog(BaseLog):
+    """File Log Pydantic Model that use to saving log data from result of
+    workflow execution. It inherit from BaseLog model that implement the
+    ``self.save`` method for file.
+    """
+
+    filename_fmt: ClassVar[str] = (
+        "./logs/workflow={name}/release={release:%Y%m%d%H%M%S}"
+    )
+
+    def do_before(self) -> None:
+        """Create directory of release before saving log file."""
+        self.pointer().mkdir(parents=True, exist_ok=True)
+
+    @classmethod
+    def find_logs(cls, name: str) -> Iterator[Self]:
+        """Generate the logging data that found from logs path with specific a
+        workflow name.
+
+        :param name: A workflow name that want to search release logging data.
+        """
+        pointer: Path = config.root_path / f"./logs/workflow={name}"
+        if not pointer.exists():
+            raise FileNotFoundError(
+                f"Pointer: ./logs/workflow={name} does not found."
+            )
+
+        for file in pointer.glob("./release=*/*.log"):
+            with file.open(mode="r", encoding="utf-8") as f:
+                yield cls.model_validate(obj=json.load(f))
+
+    @classmethod
+    def find_log_latest(
+        cls,
+        name: str,
+        release: datetime | None = None,
+    ) -> Self:
+        """Return the logging data that found from logs path with specific
+        workflow name and release values. If a release does not pass to an input
+        argument, it will return the latest release from the current log path.
+
+        :raise FileNotFoundError:
+        :raise NotImplementedError:
+
+        :rtype: Self
+        """
+        if release is None:
+            raise NotImplementedError("Find latest log does not implement yet.")
+
+        pointer: Path = (
+            config.root_path
+            / f"./logs/workflow={name}/release={release:%Y%m%d%H%M%S}"
+        )
+        if not pointer.exists():
+            raise FileNotFoundError(
+                f"Pointer: ./logs/workflow={name}/"
+                f"release={release:%Y%m%d%H%M%S} does not found."
+            )
+
+        with max(pointer.glob("./*.log"), key=os.path.getctime).open(
+            mode="r", encoding="utf-8"
+        ) as f:
+            return cls.model_validate(obj=json.load(f))
+
+    @classmethod
+    def is_pointed(
+        cls,
+        name: str,
+        release: datetime,
+        *,
+        queue: list[datetime] | None = None,
+    ) -> bool:
+        """Check this log already point in the destination.
+
+        :param name: A workflow name.
+        :param release: A release datetime.
+        :param queue: A list of queue of datetime that already run in the
+            future.
+        """
+        # NOTE: Check environ variable was set for real writing.
+        if not config.enable_write_log:
+            return False
+
+        # NOTE: create pointer path that use the same logic of pointer method.
+        pointer: Path = config.root_path / cls.filename_fmt.format(
+            name=name, release=release
+        )
+
+        if not queue:
+            return pointer.exists()
+        return pointer.exists() or (release in queue)
+
+    def pointer(self) -> Path:
+        """Return release directory path that was generated from model data.
+
+        :rtype: Path
+        """
+        return config.root_path / self.filename_fmt.format(
+            name=self.name, release=self.release
+        )
+
+    def save(self, excluded: list[str] | None) -> Self:
+        """Save logging data that receive a context data from a workflow
+        execution result.
+
+        :param excluded: An excluded list of key name that want to pass in the
+            model_dump method.
+        :rtype: Self
+        """
+        # NOTE: Check environ variable was set for real writing.
+        if not config.enable_write_log:
+            return self
+
+        log_file: Path = self.pointer() / f"{self.run_id}.log"
+        log_file.write_text(
+            json.dumps(
+                self.model_dump(exclude=excluded),
+                default=str,
+                indent=2,
+            ),
+            encoding="utf-8",
+        )
+        return self
+
+
+class SQLiteLog(BaseLog):  # pragma: no cov
+
+    def save(self, excluded: list[str] | None) -> None:
+        raise NotImplementedError("SQLiteLog does not implement yet.")
+
+
+Log = Union[
+    FileLog,
+    SQLiteLog,
+]