ddeutil-workflow 0.0.26.post0__tar.gz → 0.0.26.post1__tar.gz

{ddeutil_workflow-0.0.26.post0/src/ddeutil_workflow.egg-info → ddeutil_workflow-0.0.26.post1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ddeutil-workflow
-Version: 0.0.26.post0
+Version: 0.0.26.post1
 Summary: Lightweight workflow orchestration with less dependencies
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT

ddeutil_workflow-0.0.26.post1/src/ddeutil/workflow/__about__.py

@@ -0,0 +1 @@
+__version__: str = "0.0.26.post1"

{ddeutil_workflow-0.0.26.post0 → ddeutil_workflow-0.0.26.post1}/src/ddeutil/workflow/__init__.py

@@ -3,11 +3,14 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
-from .__cron import CronRunner
+from .__cron import CronJob, CronRunner
 from .conf import (
     Config,
-    FileLog,
     Loader,
+    Log,
+    env,
+    get_log,
+    get_logger,
 )
 from .cron import (
     On,

{ddeutil_workflow-0.0.26.post0 → ddeutil_workflow-0.0.26.post1}/src/ddeutil/workflow/conf.py

@@ -13,27 +13,30 @@ from collections.abc import Iterator
 from datetime import datetime, timedelta
 from functools import cached_property, lru_cache
 from pathlib import Path
-from typing import ClassVar, Optional, TypeVar, Union
+from typing import ClassVar, Optional, Union
 from zoneinfo import ZoneInfo
 
 from ddeutil.core import str2bool
 from ddeutil.io import YamlFlResolve
-from dotenv import load_dotenv
 from pydantic import BaseModel, Field
 from pydantic.functional_validators import model_validator
 from typing_extensions import Self
 
 from .__types import DictData, TupleStr
 
-AnyModel = TypeVar("AnyModel", bound=BaseModel)
-AnyModelType = type[AnyModel]
 
-load_dotenv()
+def env(var: str, default: str | None = None) -> str | None:  # pragma: no cov
+    return os.getenv(f"WORKFLOW_{var}", default)
+
+
+def glob_files(path: Path) -> Iterator[Path]:  # pragma: no cov
+    yield from (file for file in path.rglob("*") if file.is_file())
 
-env = os.getenv
 
 __all__: TupleStr = (
+    "env",
     "get_logger",
+    "get_log",
     "Config",
     "SimLoad",
     "Loader",
@@ -52,6 +55,14 @@ def get_logger(name: str):
     :param name: A module name that want to log.
     """
     lg = logging.getLogger(name)
+
+    # NOTE: Developers using this package can then disable all logging just for
+    #   this package by;
+    #
+    #   `logging.getLogger('ddeutil.workflow').propagate = False`
+    #
+    lg.addHandler(logging.NullHandler())
+
     formatter = logging.Formatter(
         fmt=(
             "%(asctime)s.%(msecs)03d (%(name)-10s, %(process)-5d, "
@@ -68,115 +79,141 @@ def get_logger(name: str):
     return lg
 
 
-class Config:
+class Config:  # pragma: no cov
     """Config object for keeping application configuration on current session
     without changing when if the application still running.
     """
 
     # NOTE: Core
-    root_path: Path = Path(os.getenv("WORKFLOW_ROOT_PATH", "."))
-    tz: ZoneInfo = ZoneInfo(env("WORKFLOW_CORE_TIMEZONE", "UTC"))
-    gen_id_simple_mode: bool = str2bool(
-        os.getenv("WORKFLOW_CORE_GENERATE_ID_SIMPLE_MODE", "true")
-    )
+    @property
+    def root_path(self) -> Path:
+        return Path(env("ROOT_PATH", "."))
+
+    @property
+    def conf_path(self) -> Path:
+        """Config path that use root_path class argument for this construction.
+
+        :rtype: Path
+        """
+        return self.root_path / env("CORE_PATH_CONF", "conf")
+
+    @property
+    def tz(self) -> ZoneInfo:
+        return ZoneInfo(env("CORE_TIMEZONE", "UTC"))
+
+    @property
+    def gen_id_simple_mode(self) -> bool:
+        return str2bool(env("CORE_GENERATE_ID_SIMPLE_MODE", "true"))
 
     # NOTE: Register
-    regis_hook_str: str = os.getenv(
-        "WORKFLOW_CORE_REGISTRY", "src,src.ddeutil.workflow,tests,tests.utils"
-    )
-    regis_filter_str: str = os.getenv(
-        "WORKFLOW_CORE_REGISTRY_FILTER", "ddeutil.workflow.utils"
-    )
+    @property
+    def regis_hook(self) -> list[str]:
+        regis_hook_str: str = env(
+            "CORE_REGISTRY", "src,src.ddeutil.workflow,tests,tests.utils"
+        )
+        return [r.strip() for r in regis_hook_str.split(",")]
+
+    @property
+    def regis_filter(self) -> list[str]:
+        regis_filter_str: str = env(
+            "CORE_REGISTRY_FILTER", "ddeutil.workflow.utils"
+        )
+        return [r.strip() for r in regis_filter_str.split(",")]
 
     # NOTE: Logging
-    debug: bool = str2bool(os.getenv("WORKFLOW_LOG_DEBUG_MODE", "true"))
-    enable_write_log: bool = str2bool(
-        os.getenv("WORKFLOW_LOG_ENABLE_WRITE", "false")
-    )
-    log_path: Path = Path(os.getenv("WORKFLOW_LOG_PATH", "./logs"))
+    @property
+    def log_path(self) -> Path:
+        return Path(env("LOG_PATH", "./logs"))
+
+    @property
+    def debug(self) -> bool:
+        return str2bool(env("LOG_DEBUG_MODE", "true"))
+
+    @property
+    def enable_write_log(self) -> bool:
+        return str2bool(env("LOG_ENABLE_WRITE", "false"))
 
     # NOTE: Stage
-    stage_raise_error: bool = str2bool(
-        env("WORKFLOW_CORE_STAGE_RAISE_ERROR", "false")
-    )
-    stage_default_id: bool = str2bool(
-        env("WORKFLOW_CORE_STAGE_DEFAULT_ID", "false")
-    )
+    @property
+    def stage_raise_error(self) -> bool:
+        return str2bool(env("CORE_STAGE_RAISE_ERROR", "false"))
 
-    # NOTE: Job
-    job_raise_error: bool = str2bool(
-        env("WORKFLOW_CORE_JOB_RAISE_ERROR", "true")
-    )
-    job_default_id: bool = str2bool(
-        env("WORKFLOW_CORE_JOB_DEFAULT_ID", "false")
-    )
+    @property
+    def stage_default_id(self) -> bool:
+        return str2bool(env("CORE_STAGE_DEFAULT_ID", "false"))
 
-    # NOTE: Workflow
-    max_job_parallel: int = int(env("WORKFLOW_CORE_MAX_JOB_PARALLEL", "2"))
-    max_job_exec_timeout: int = int(
-        env("WORKFLOW_CORE_MAX_JOB_EXEC_TIMEOUT", "600")
-    )
-    max_poking_pool_worker: int = int(
-        os.getenv("WORKFLOW_CORE_MAX_NUM_POKING", "4")
-    )
-    max_on_per_workflow: int = int(
-        env("WORKFLOW_CORE_MAX_CRON_PER_WORKFLOW", "5")
-    )
-    max_queue_complete_hist: int = int(
-        os.getenv("WORKFLOW_CORE_MAX_QUEUE_COMPLETE_HIST", "16")
-    )
+    # NOTE: Job
+    @property
+    def job_raise_error(self) -> bool:
+        return str2bool(env("CORE_JOB_RAISE_ERROR", "true"))
 
-    # NOTE: Schedule App
-    max_schedule_process: int = int(env("WORKFLOW_APP_MAX_PROCESS", "2"))
-    max_schedule_per_process: int = int(
-        env("WORKFLOW_APP_MAX_SCHEDULE_PER_PROCESS", "100")
-    )
-    stop_boundary_delta_str: str = env(
-        "WORKFLOW_APP_STOP_BOUNDARY_DELTA", '{"minutes": 5, "seconds": 20}'
-    )
+    @property
+    def job_default_id(self) -> bool:
+        return str2bool(env("CORE_JOB_DEFAULT_ID", "false"))
 
-    # NOTE: API
-    prefix_path: str = env("WORKFLOW_API_PREFIX_PATH", "/api/v1")
-    enable_route_workflow: bool = str2bool(
-        env("WORKFLOW_API_ENABLE_ROUTE_WORKFLOW", "true")
-    )
-    enable_route_schedule: bool = str2bool(
-        env("WORKFLOW_API_ENABLE_ROUTE_SCHEDULE", "true")
-    )
+    # NOTE: Workflow
+    @property
+    def max_job_parallel(self) -> int:
+        max_job_parallel = int(env("CORE_MAX_JOB_PARALLEL", "2"))
 
-    def __init__(self) -> None:
         # VALIDATE: the MAX_JOB_PARALLEL value should not less than 0.
-        if self.max_job_parallel < 0:
+        if max_job_parallel < 0:
             raise ValueError(
-                f"``MAX_JOB_PARALLEL`` should more than 0 but got "
-                f"{self.max_job_parallel}."
+                f"``WORKFLOW_MAX_JOB_PARALLEL`` should more than 0 but got "
+                f"{max_job_parallel}."
             )
+        return max_job_parallel
+
+    @property
+    def max_job_exec_timeout(self) -> int:
+        return int(env("CORE_MAX_JOB_EXEC_TIMEOUT", "600"))
+
+    @property
+    def max_poking_pool_worker(self) -> int:
+        return int(env("CORE_MAX_NUM_POKING", "4"))
+
+    @property
+    def max_on_per_workflow(self) -> int:
+        return int(env("CORE_MAX_CRON_PER_WORKFLOW", "5"))
+
+    @property
+    def max_queue_complete_hist(self) -> int:
+        return int(env("CORE_MAX_QUEUE_COMPLETE_HIST", "16"))
+
+    # NOTE: Schedule App
+    @property
+    def max_schedule_process(self) -> int:
+        return int(env("APP_MAX_PROCESS", "2"))
+
+    @property
+    def max_schedule_per_process(self) -> int:
+        return int(env("APP_MAX_SCHEDULE_PER_PROCESS", "100"))
 
+    @property
+    def stop_boundary_delta(self) -> timedelta:
+        stop_boundary_delta_str: str = env(
+            "APP_STOP_BOUNDARY_DELTA", '{"minutes": 5, "seconds": 20}'
+        )
         try:
-            self.stop_boundary_delta: timedelta = timedelta(
-                **json.loads(self.stop_boundary_delta_str)
-            )
+            return timedelta(**json.loads(stop_boundary_delta_str))
         except Exception as err:
             raise ValueError(
                 "Config ``WORKFLOW_APP_STOP_BOUNDARY_DELTA`` can not parsing to"
-                f"timedelta with {self.stop_boundary_delta_str}."
+                f"timedelta with {stop_boundary_delta_str}."
             ) from err
 
+    # NOTE: API
     @property
-    def conf_path(self) -> Path:
-        """Config path that use root_path class argument for this construction.
-
-        :rtype: Path
-        """
-        return self.root_path / os.getenv("WORKFLOW_CORE_PATH_CONF", "conf")
+    def prefix_path(self) -> str:
+        return env("API_PREFIX_PATH", "/api/v1")
 
     @property
-    def regis_hook(self) -> list[str]:
-        return [r.strip() for r in self.regis_hook_str.split(",")]
+    def enable_route_workflow(self) -> bool:
+        return str2bool(env("API_ENABLE_ROUTE_WORKFLOW", "true"))
 
     @property
-    def regis_filter(self) -> list[str]:
-        return [r.strip() for r in self.regis_filter_str.split(",")]
+    def enable_route_schedule(self) -> bool:
+        return str2bool(env("API_ENABLE_ROUTE_SCHEDULE", "true"))
 
 
 class SimLoad:
@@ -206,14 +243,9 @@ class SimLoad:
         externals: DictData | None = None,
     ) -> None:
         self.data: DictData = {}
-        for file in conf.conf_path.rglob("*"):
-            if not file.is_file():
-                continue
-
-            if data := self.filter_suffix(
-                file,
-                name,
-            ):
+        for file in glob_files(conf.conf_path):
+
+            if data := self.filter_suffix(file, name):
                 self.data = data
 
         # VALIDATE: check the data that reading should not empty.
@@ -245,10 +277,7 @@ class SimLoad:
         :rtype: Iterator[tuple[str, DictData]]
         """
         exclude: list[str] = excluded or []
-        for file in conf.conf_path.rglob("*"):
-
-            if not file.is_file():
-                continue
+        for file in glob_files(conf.conf_path):
 
             for key, data in cls.filter_suffix(file).items():
 
@@ -274,7 +303,7 @@ class SimLoad:
         """Return object of string type which implement on any registry. The
         object type.
 
-        :rtype: AnyModelType
+        :rtype: str
         """
         if _typ := self.data.get("type"):
             return _typ
@@ -481,6 +510,18 @@ class FileLog(BaseLog):
 
 class SQLiteLog(BaseLog):  # pragma: no cov
 
+    table: str = "workflow_log"
+    ddl: str = """
+        workflow        str,
+        release         int,
+        type            str,
+        context         json,
+        parent_run_id   int,
+        run_id          int,
+        update          datetime
+        primary key ( run_id )
+        """
+
     def save(self, excluded: list[str] | None) -> None:
         raise NotImplementedError("SQLiteLog does not implement yet.")
 
@@ -489,3 +530,9 @@ Log = Union[
     FileLog,
     SQLiteLog,
 ]
+
+
+def get_log() -> Log:  # pragma: no cov
+    if config.log_path.is_file():
+        return SQLiteLog
+    return FileLog

{ddeutil_workflow-0.0.26.post0 → ddeutil_workflow-0.0.26.post1}/src/ddeutil/workflow/scheduler.py

@@ -51,7 +51,7 @@ except ImportError:  # pragma: no cov
 
 from .__cron import CronRunner
 from .__types import DictData, TupleStr
-from .conf import FileLog, Loader, Log, config, get_logger
+from .conf import Loader, Log, config, get_log, get_logger
 from .cron import On
 from .exceptions import WorkflowException
 from .utils import (
@@ -493,7 +493,7 @@ def schedule_control(
             "Should install schedule package before use this module."
         ) from None
 
-    log: type[Log] = log or FileLog
+    log: type[Log] = log or get_log()
     scheduler: Scheduler = Scheduler()
     start_date: datetime = datetime.now(tz=config.tz)
     stop_date: datetime = stop or (start_date + config.stop_boundary_delta)

{ddeutil_workflow-0.0.26.post0 → ddeutil_workflow-0.0.26.post1}/src/ddeutil/workflow/workflow.py

@@ -42,7 +42,7 @@ from typing_extensions import Self
 
 from .__cron import CronJob, CronRunner
 from .__types import DictData, TupleStr
-from .conf import FileLog, Loader, Log, config, get_logger
+from .conf import Loader, Log, config, get_log, get_logger
 from .cron import On
 from .exceptions import JobException, WorkflowException
 from .job import Job
@@ -501,7 +501,7 @@ class Workflow(BaseModel):
 
         :rtype: Result
         """
-        log: type[Log] = log or FileLog
+        log: type[Log] = log or get_log()
         name: str = override_log_name or self.name
         run_id: str = run_id or gen_id(name, unique=True)
         rs_release: Result = Result(run_id=run_id)
@@ -670,7 +670,7 @@ class Workflow(BaseModel):
         :rtype: list[Result]
         :return: A list of all results that return from ``self.release`` method.
         """
-        log: type[Log] = log or FileLog
+        log: type[Log] = log or get_log()
         run_id: str = run_id or gen_id(self.name, unique=True)
 
         # NOTE: If this workflow does not set the on schedule, it will return
@@ -1151,7 +1151,7 @@ class WorkflowTask:
 
         :rtype: Result
         """
-        log: type[Log] = log or FileLog
+        log: type[Log] = log or get_log()
 
         if release is None:
             if queue.check_queue(self.runner.date):

{ddeutil_workflow-0.0.26.post0 → ddeutil_workflow-0.0.26.post1/src/ddeutil_workflow.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ddeutil-workflow
-Version: 0.0.26.post0
+Version: 0.0.26.post1
 Summary: Lightweight workflow orchestration with less dependencies
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT

{ddeutil_workflow-0.0.26.post0 → ddeutil_workflow-0.0.26.post1}/tests/test_conf.py

@@ -13,18 +13,29 @@ from ddeutil.workflow.scheduler import Schedule
 from ddeutil.workflow.workflow import Workflow
 
 
-def test_config():
-    with mock.patch.object(Config, "max_job_parallel", -1):
-        with pytest.raises(ValueError):
-            Config()
+@pytest.fixture(scope="function")
+def adjust_config():
+    origin_max_job = os.getenv("WORKFLOW_CORE_MAX_JOB_PARALLEL")
+    origin_stop = os.getenv("WORKFLOW_APP_STOP_BOUNDARY_DELTA")
+    os.environ["WORKFLOW_CORE_MAX_JOB_PARALLEL"] = "-1"
+    os.environ["WORKFLOW_APP_STOP_BOUNDARY_DELTA"] = "{"
 
-    with mock.patch.object(Config, "stop_boundary_delta_str", "{"):
-        with pytest.raises(ValueError):
-            Config()
+    yield
+
+    os.environ["WORKFLOW_CORE_MAX_JOB_PARALLEL"] = origin_max_job
+    os.environ["WORKFLOW_APP_STOP_BOUNDARY_DELTA"] = origin_stop
+
+
+def test_config(adjust_config):
+
+    with pytest.raises(ValueError):
+        _ = Config().max_job_parallel
+
+    with pytest.raises(ValueError):
+        _ = Config().stop_boundary_delta
 
     conf = Config()
     os.environ["WORKFLOW_CORE_TIMEZONE"] = "Asia/Bangkok"
-    conf = conf.refresh_dotenv()
     assert conf.tz == ZoneInfo("Asia/Bangkok")
 
 

{ddeutil_workflow-0.0.26.post0 → ddeutil_workflow-0.0.26.post1}/tests/test_schedule_control.py

@@ -1,31 +1,31 @@
 from datetime import timedelta
 from unittest import mock
 
-from ddeutil.workflow.conf import Config, config
+from ddeutil.workflow.conf import Config
 from ddeutil.workflow.scheduler import schedule_control
 
 
-@mock.patch.object(config, "stop_boundary_delta", timedelta(minutes=1))
+@mock.patch.object(Config, "stop_boundary_delta", timedelta(minutes=1))
 @mock.patch.object(Config, "enable_write_log", False)
 def test_schedule_control():
     rs = schedule_control(["schedule-every-minute-wf"])
     assert rs == ["schedule-every-minute-wf"]
 
 
-@mock.patch.object(config, "stop_boundary_delta", timedelta(minutes=3))
+@mock.patch.object(Config, "stop_boundary_delta", timedelta(minutes=3))
 @mock.patch.object(Config, "enable_write_log", False)
 def test_schedule_control_multi_on():
     rs = schedule_control(["schedule-multi-on-wf"])
     assert rs == ["schedule-multi-on-wf"]
 
 
-@mock.patch.object(config, "stop_boundary_delta", timedelta(minutes=0))
+@mock.patch.object(Config, "stop_boundary_delta", timedelta(minutes=0))
 def test_schedule_control_stop():
     rs = schedule_control(["schedule-every-minute-wf"])
     assert rs == ["schedule-every-minute-wf"]
 
 
-@mock.patch.object(config, "stop_boundary_delta", timedelta(minutes=2))
+@mock.patch.object(Config, "stop_boundary_delta", timedelta(minutes=2))
 @mock.patch.object(Config, "enable_write_log", False)
 def test_schedule_control_parallel():
     rs = schedule_control(["schedule-every-minute-wf-parallel"])

{ddeutil_workflow-0.0.26.post0 → ddeutil_workflow-0.0.26.post1}/tests/test_utils.py

@@ -1,16 +1,27 @@
+import os
 from pathlib import Path
-from unittest.mock import patch
 
 import ddeutil.workflow.utils as utils
 import pytest
 
 
+@pytest.fixture(scope="function")
+def adjust_config_gen_id():
+    origin_simple = os.getenv("WORKFLOW_CORE_GENERATE_ID_SIMPLE_MODE")
+    os.environ["WORKFLOW_CORE_GENERATE_ID_SIMPLE_MODE"] = "false"
+
+    yield
+
+    os.environ["WORKFLOW_CORE_GENERATE_ID_SIMPLE_MODE"] = origin_simple
+
+
 def test_gen_id():
     assert "1354680202" == utils.gen_id("{}")
     assert "1354680202" == utils.gen_id("{}", sensitive=False)
 
-    with patch("ddeutil.workflow.utils.config.gen_id_simple_mode", False):
-        assert "99914b932bd37a50b983c5e7c90ae93b" == utils.gen_id("{}")
+
+def test_gen_id_not_simple(adjust_config_gen_id):
+    assert "99914b932bd37a50b983c5e7c90ae93b" == utils.gen_id("{}")
 
 
 def test_filter_func():

{ddeutil_workflow-0.0.26.post0 → ddeutil_workflow-0.0.26.post1}/tests/test_utils_filter.py

@@ -25,8 +25,8 @@ def raise_err(_: str) -> None:  # pragma: no cov
 def test_make_registry_raise():
     with mock.patch.object(
         Config,
-        "regis_filter_str",
-        "ddeutil.workflow.utils,tests.test_utils_filter,foo.bar",
+        "regis_filter",
+        ["ddeutil.workflow.utils", "tests.test_utils_filter", "foo.bar"],
     ):
         assert isfunction(make_filter_registry()["foo"])
         assert "bar" == make_filter_registry()["foo"]("")
@@ -59,8 +59,8 @@ def test_get_args_const():
 def test_map_post_filter():
     with mock.patch.object(
         Config,
-        "regis_filter_str",
-        "ddeutil.workflow.utils,tests.test_utils_filter,foo.bar",
+        "regis_filter",
+        ["ddeutil.workflow.utils", "tests.test_utils_filter", "foo.bar"],
     ):
         assert "bar" == map_post_filter("demo", ["foo"], make_filter_registry())
         assert "'bar'" == map_post_filter(

ddeutil_workflow-0.0.26.post0/src/ddeutil/workflow/__about__.py

@@ -1 +0,0 @@
-__version__: str = "0.0.26.post0"