ddeutil-workflow 0.0.26.post0__py3-none-any.whl → 0.0.26.post1__py3-none-any.whl

ddeutil/workflow/__about__.py

@@ -1 +1 @@
-__version__: str = "0.0.26.post0"
+__version__: str = "0.0.26.post1"

ddeutil/workflow/__init__.py

@@ -3,11 +3,14 @@
 # Licensed under the MIT License. See LICENSE in the project root for
 # license information.
 # ------------------------------------------------------------------------------
-from .__cron import CronRunner
+from .__cron import CronJob, CronRunner
 from .conf import (
     Config,
-    FileLog,
     Loader,
+    Log,
+    env,
+    get_log,
+    get_logger,
 )
 from .cron import (
     On,

ddeutil/workflow/conf.py

@@ -13,27 +13,30 @@ from collections.abc import Iterator
 from datetime import datetime, timedelta
 from functools import cached_property, lru_cache
 from pathlib import Path
-from typing import ClassVar, Optional, TypeVar, Union
+from typing import ClassVar, Optional, Union
 from zoneinfo import ZoneInfo
 
 from ddeutil.core import str2bool
 from ddeutil.io import YamlFlResolve
-from dotenv import load_dotenv
 from pydantic import BaseModel, Field
 from pydantic.functional_validators import model_validator
 from typing_extensions import Self
 
 from .__types import DictData, TupleStr
 
-AnyModel = TypeVar("AnyModel", bound=BaseModel)
-AnyModelType = type[AnyModel]
 
-load_dotenv()
+def env(var: str, default: str | None = None) -> str | None:  # pragma: no cov
+    return os.getenv(f"WORKFLOW_{var}", default)
+
+
+def glob_files(path: Path) -> Iterator[Path]:  # pragma: no cov
+    yield from (file for file in path.rglob("*") if file.is_file())
 
-env = os.getenv
 
 __all__: TupleStr = (
+    "env",
     "get_logger",
+    "get_log",
     "Config",
     "SimLoad",
     "Loader",
@@ -52,6 +55,14 @@ def get_logger(name: str):
     :param name: A module name that want to log.
     """
     lg = logging.getLogger(name)
+
+    # NOTE: Developers using this package can then disable all logging just for
+    #   this package by;
+    #
+    #   `logging.getLogger('ddeutil.workflow').propagate = False`
+    #
+    lg.addHandler(logging.NullHandler())
+
     formatter = logging.Formatter(
         fmt=(
             "%(asctime)s.%(msecs)03d (%(name)-10s, %(process)-5d, "
@@ -68,115 +79,141 @@ def get_logger(name: str):
     return lg
 
 
-class Config:
+class Config:  # pragma: no cov
     """Config object for keeping application configuration on current session
     without changing when if the application still running.
     """
 
     # NOTE: Core
-    root_path: Path = Path(os.getenv("WORKFLOW_ROOT_PATH", "."))
-    tz: ZoneInfo = ZoneInfo(env("WORKFLOW_CORE_TIMEZONE", "UTC"))
-    gen_id_simple_mode: bool = str2bool(
-        os.getenv("WORKFLOW_CORE_GENERATE_ID_SIMPLE_MODE", "true")
-    )
+    @property
+    def root_path(self) -> Path:
+        return Path(env("ROOT_PATH", "."))
+
+    @property
+    def conf_path(self) -> Path:
+        """Config path that use root_path class argument for this construction.
+
+        :rtype: Path
+        """
+        return self.root_path / env("CORE_PATH_CONF", "conf")
+
+    @property
+    def tz(self) -> ZoneInfo:
+        return ZoneInfo(env("CORE_TIMEZONE", "UTC"))
+
+    @property
+    def gen_id_simple_mode(self) -> bool:
+        return str2bool(env("CORE_GENERATE_ID_SIMPLE_MODE", "true"))
 
     # NOTE: Register
-    regis_hook_str: str = os.getenv(
-        "WORKFLOW_CORE_REGISTRY", "src,src.ddeutil.workflow,tests,tests.utils"
-    )
-    regis_filter_str: str = os.getenv(
-        "WORKFLOW_CORE_REGISTRY_FILTER", "ddeutil.workflow.utils"
-    )
+    @property
+    def regis_hook(self) -> list[str]:
+        regis_hook_str: str = env(
+            "CORE_REGISTRY", "src,src.ddeutil.workflow,tests,tests.utils"
+        )
+        return [r.strip() for r in regis_hook_str.split(",")]
+
+    @property
+    def regis_filter(self) -> list[str]:
+        regis_filter_str: str = env(
+            "CORE_REGISTRY_FILTER", "ddeutil.workflow.utils"
+        )
+        return [r.strip() for r in regis_filter_str.split(",")]
 
     # NOTE: Logging
-    debug: bool = str2bool(os.getenv("WORKFLOW_LOG_DEBUG_MODE", "true"))
-    enable_write_log: bool = str2bool(
-        os.getenv("WORKFLOW_LOG_ENABLE_WRITE", "false")
-    )
-    log_path: Path = Path(os.getenv("WORKFLOW_LOG_PATH", "./logs"))
+    @property
+    def log_path(self) -> Path:
+        return Path(env("LOG_PATH", "./logs"))
+
+    @property
+    def debug(self) -> bool:
+        return str2bool(env("LOG_DEBUG_MODE", "true"))
+
+    @property
+    def enable_write_log(self) -> bool:
+        return str2bool(env("LOG_ENABLE_WRITE", "false"))
 
     # NOTE: Stage
-    stage_raise_error: bool = str2bool(
-        env("WORKFLOW_CORE_STAGE_RAISE_ERROR", "false")
-    )
-    stage_default_id: bool = str2bool(
-        env("WORKFLOW_CORE_STAGE_DEFAULT_ID", "false")
-    )
+    @property
+    def stage_raise_error(self) -> bool:
+        return str2bool(env("CORE_STAGE_RAISE_ERROR", "false"))
 
-    # NOTE: Job
-    job_raise_error: bool = str2bool(
-        env("WORKFLOW_CORE_JOB_RAISE_ERROR", "true")
-    )
-    job_default_id: bool = str2bool(
-        env("WORKFLOW_CORE_JOB_DEFAULT_ID", "false")
-    )
+    @property
+    def stage_default_id(self) -> bool:
+        return str2bool(env("CORE_STAGE_DEFAULT_ID", "false"))
 
-    # NOTE: Workflow
-    max_job_parallel: int = int(env("WORKFLOW_CORE_MAX_JOB_PARALLEL", "2"))
-    max_job_exec_timeout: int = int(
-        env("WORKFLOW_CORE_MAX_JOB_EXEC_TIMEOUT", "600")
-    )
-    max_poking_pool_worker: int = int(
-        os.getenv("WORKFLOW_CORE_MAX_NUM_POKING", "4")
-    )
-    max_on_per_workflow: int = int(
-        env("WORKFLOW_CORE_MAX_CRON_PER_WORKFLOW", "5")
-    )
-    max_queue_complete_hist: int = int(
-        os.getenv("WORKFLOW_CORE_MAX_QUEUE_COMPLETE_HIST", "16")
-    )
+    # NOTE: Job
+    @property
+    def job_raise_error(self) -> bool:
+        return str2bool(env("CORE_JOB_RAISE_ERROR", "true"))
 
-    # NOTE: Schedule App
-    max_schedule_process: int = int(env("WORKFLOW_APP_MAX_PROCESS", "2"))
-    max_schedule_per_process: int = int(
-        env("WORKFLOW_APP_MAX_SCHEDULE_PER_PROCESS", "100")
-    )
-    stop_boundary_delta_str: str = env(
-        "WORKFLOW_APP_STOP_BOUNDARY_DELTA", '{"minutes": 5, "seconds": 20}'
-    )
+    @property
+    def job_default_id(self) -> bool:
+        return str2bool(env("CORE_JOB_DEFAULT_ID", "false"))
 
-    # NOTE: API
-    prefix_path: str = env("WORKFLOW_API_PREFIX_PATH", "/api/v1")
-    enable_route_workflow: bool = str2bool(
-        env("WORKFLOW_API_ENABLE_ROUTE_WORKFLOW", "true")
-    )
-    enable_route_schedule: bool = str2bool(
-        env("WORKFLOW_API_ENABLE_ROUTE_SCHEDULE", "true")
-    )
+    # NOTE: Workflow
+    @property
+    def max_job_parallel(self) -> int:
+        max_job_parallel = int(env("CORE_MAX_JOB_PARALLEL", "2"))
 
-    def __init__(self) -> None:
         # VALIDATE: the MAX_JOB_PARALLEL value should not less than 0.
-        if self.max_job_parallel < 0:
+        if max_job_parallel < 0:
             raise ValueError(
-                f"``MAX_JOB_PARALLEL`` should more than 0 but got "
-                f"{self.max_job_parallel}."
+                f"``WORKFLOW_MAX_JOB_PARALLEL`` should more than 0 but got "
+                f"{max_job_parallel}."
             )
+        return max_job_parallel
+
+    @property
+    def max_job_exec_timeout(self) -> int:
+        return int(env("CORE_MAX_JOB_EXEC_TIMEOUT", "600"))
+
+    @property
+    def max_poking_pool_worker(self) -> int:
+        return int(env("CORE_MAX_NUM_POKING", "4"))
+
+    @property
+    def max_on_per_workflow(self) -> int:
+        return int(env("CORE_MAX_CRON_PER_WORKFLOW", "5"))
+
+    @property
+    def max_queue_complete_hist(self) -> int:
+        return int(env("CORE_MAX_QUEUE_COMPLETE_HIST", "16"))
+
+    # NOTE: Schedule App
+    @property
+    def max_schedule_process(self) -> int:
+        return int(env("APP_MAX_PROCESS", "2"))
+
+    @property
+    def max_schedule_per_process(self) -> int:
+        return int(env("APP_MAX_SCHEDULE_PER_PROCESS", "100"))
 
+    @property
+    def stop_boundary_delta(self) -> timedelta:
+        stop_boundary_delta_str: str = env(
+            "APP_STOP_BOUNDARY_DELTA", '{"minutes": 5, "seconds": 20}'
+        )
         try:
-            self.stop_boundary_delta: timedelta = timedelta(
-                **json.loads(self.stop_boundary_delta_str)
-            )
+            return timedelta(**json.loads(stop_boundary_delta_str))
         except Exception as err:
             raise ValueError(
                 "Config ``WORKFLOW_APP_STOP_BOUNDARY_DELTA`` can not parsing to"
-                f"timedelta with {self.stop_boundary_delta_str}."
+                f"timedelta with {stop_boundary_delta_str}."
             ) from err
 
+    # NOTE: API
     @property
-    def conf_path(self) -> Path:
-        """Config path that use root_path class argument for this construction.
-
-        :rtype: Path
-        """
-        return self.root_path / os.getenv("WORKFLOW_CORE_PATH_CONF", "conf")
+    def prefix_path(self) -> str:
+        return env("API_PREFIX_PATH", "/api/v1")
 
     @property
-    def regis_hook(self) -> list[str]:
-        return [r.strip() for r in self.regis_hook_str.split(",")]
+    def enable_route_workflow(self) -> bool:
+        return str2bool(env("API_ENABLE_ROUTE_WORKFLOW", "true"))
 
     @property
-    def regis_filter(self) -> list[str]:
-        return [r.strip() for r in self.regis_filter_str.split(",")]
+    def enable_route_schedule(self) -> bool:
+        return str2bool(env("API_ENABLE_ROUTE_SCHEDULE", "true"))
 
 
 class SimLoad:
@@ -206,14 +243,9 @@ class SimLoad:
         externals: DictData | None = None,
     ) -> None:
         self.data: DictData = {}
-        for file in conf.conf_path.rglob("*"):
-            if not file.is_file():
-                continue
-
-            if data := self.filter_suffix(
-                file,
-                name,
-            ):
+        for file in glob_files(conf.conf_path):
+
+            if data := self.filter_suffix(file, name):
                 self.data = data
 
         # VALIDATE: check the data that reading should not empty.
@@ -245,10 +277,7 @@ class SimLoad:
         :rtype: Iterator[tuple[str, DictData]]
         """
         exclude: list[str] = excluded or []
-        for file in conf.conf_path.rglob("*"):
-
-            if not file.is_file():
-                continue
+        for file in glob_files(conf.conf_path):
 
             for key, data in cls.filter_suffix(file).items():
 
@@ -274,7 +303,7 @@ class SimLoad:
         """Return object of string type which implement on any registry. The
         object type.
 
-        :rtype: AnyModelType
+        :rtype: str
         """
         if _typ := self.data.get("type"):
             return _typ
@@ -481,6 +510,18 @@ class FileLog(BaseLog):
 
 class SQLiteLog(BaseLog):  # pragma: no cov
 
+    table: str = "workflow_log"
+    ddl: str = """
+        workflow        str,
+        release         int,
+        type            str,
+        context         json,
+        parent_run_id   int,
+        run_id          int,
+        update          datetime
+        primary key ( run_id )
+        """
+
     def save(self, excluded: list[str] | None) -> None:
         raise NotImplementedError("SQLiteLog does not implement yet.")
 
@@ -489,3 +530,9 @@ Log = Union[
     FileLog,
     SQLiteLog,
 ]
+
+
+def get_log() -> Log:  # pragma: no cov
+    if config.log_path.is_file():
+        return SQLiteLog
+    return FileLog

ddeutil/workflow/scheduler.py

@@ -51,7 +51,7 @@ except ImportError:  # pragma: no cov
 
 from .__cron import CronRunner
 from .__types import DictData, TupleStr
-from .conf import FileLog, Loader, Log, config, get_logger
+from .conf import Loader, Log, config, get_log, get_logger
 from .cron import On
 from .exceptions import WorkflowException
 from .utils import (
@@ -493,7 +493,7 @@ def schedule_control(
             "Should install schedule package before use this module."
         ) from None
 
-    log: type[Log] = log or FileLog
+    log: type[Log] = log or get_log()
     scheduler: Scheduler = Scheduler()
     start_date: datetime = datetime.now(tz=config.tz)
     stop_date: datetime = stop or (start_date + config.stop_boundary_delta)

ddeutil/workflow/workflow.py

@@ -42,7 +42,7 @@ from typing_extensions import Self
 
 from .__cron import CronJob, CronRunner
 from .__types import DictData, TupleStr
-from .conf import FileLog, Loader, Log, config, get_logger
+from .conf import Loader, Log, config, get_log, get_logger
 from .cron import On
 from .exceptions import JobException, WorkflowException
 from .job import Job
@@ -501,7 +501,7 @@ class Workflow(BaseModel):
 
         :rtype: Result
         """
-        log: type[Log] = log or FileLog
+        log: type[Log] = log or get_log()
         name: str = override_log_name or self.name
         run_id: str = run_id or gen_id(name, unique=True)
         rs_release: Result = Result(run_id=run_id)
@@ -670,7 +670,7 @@ class Workflow(BaseModel):
         :rtype: list[Result]
         :return: A list of all results that return from ``self.release`` method.
         """
-        log: type[Log] = log or FileLog
+        log: type[Log] = log or get_log()
         run_id: str = run_id or gen_id(self.name, unique=True)
 
         # NOTE: If this workflow does not set the on schedule, it will return
@@ -1151,7 +1151,7 @@ class WorkflowTask:
 
         :rtype: Result
         """
-        log: type[Log] = log or FileLog
+        log: type[Log] = log or get_log()
 
         if release is None:
             if queue.check_queue(self.runner.date):

{ddeutil_workflow-0.0.26.post0.dist-info → ddeutil_workflow-0.0.26.post1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ddeutil-workflow
-Version: 0.0.26.post0
+Version: 0.0.26.post1
 Summary: Lightweight workflow orchestration with less dependencies
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT

{ddeutil_workflow-0.0.26.post0.dist-info → ddeutil_workflow-0.0.26.post1.dist-info}/RECORD

@@ -1,23 +1,23 @@
-ddeutil/workflow/__about__.py,sha256=9NeOhqXsrgLKL6AmtINFl4dct9jvs8oismQYXutnmJ8,34
+ddeutil/workflow/__about__.py,sha256=jU_KFZf1uiZIWhuownbhRsjIL3oHGR_URL-jKTEnMKo,34
 ddeutil/workflow/__cron.py,sha256=uA8XcbY_GwA9rJSHaHUaXaJyGDObJN0ZeYlJSinL8y8,26880
-ddeutil/workflow/__init__.py,sha256=49eGrCuchPVZKMybRouAviNhbulK_F6VwCmLm76hIss,1478
+ddeutil/workflow/__init__.py,sha256=ozadVrqfqFRuukjv_zXUcgLANdiSrC6wrKkyVjdGg3w,1521
 ddeutil/workflow/__types.py,sha256=Ia7f38kvL3NibwmRKi0wQ1ud_45Z-SojYGhNJwIqcu8,3713
-ddeutil/workflow/conf.py,sha256=MtjeaapHaEuW8GjZadA56KpkmbPQg6Ws_CHcmmAzaFc,15174
+ddeutil/workflow/conf.py,sha256=AU3GKTaxFFGDN-Sg8BGb08xj7vRBCTTjwk0FaORLJIk,16188
 ddeutil/workflow/cron.py,sha256=75A0hqevvouziKoLALncLJspVAeki9qCH3zniAJaxzY,7513
 ddeutil/workflow/exceptions.py,sha256=P56K7VD3etGm9y-k_GXrzEyqsTCaz9EJazTIshZDf9g,943
 ddeutil/workflow/job.py,sha256=cvSLMdc1sMl1MeU7so7Oe2SdRYxQwt6hm55mLV1iP-Y,24219
 ddeutil/workflow/params.py,sha256=uPGkZx18E-iZ8BteqQ2ONgg0frhF3ZmP5cOyfK2j59U,5280
 ddeutil/workflow/result.py,sha256=WIC8MsnfLiWNpZomT6jS4YCdYhlbIVVBjtGGe2dkoKk,3404
-ddeutil/workflow/scheduler.py,sha256=_V812UlqcwfVF2Sl_45nIatMklioBXcXfGZSFoAAjwo,20452
+ddeutil/workflow/scheduler.py,sha256=BbY_3Y3QOdNwDfdvnRa7grGC2_a0Hn1KJbZKAscchk8,20454
 ddeutil/workflow/stage.py,sha256=a2sngzs9DkP6GU2pgAD3QvGoijyBQTR_pOhyJUIuWAo,26692
 ddeutil/workflow/utils.py,sha256=pucRnCi9aLJDptXhzzReHZd5d-S0o5oZif5tr6H4iy8,18736
-ddeutil/workflow/workflow.py,sha256=AD0rs1tRT2EpvUyNVAEr2bBPgF6-KOzGmLedR3o4y0Q,42177
+ddeutil/workflow/workflow.py,sha256=s6E-mKzSVQPTSV0biIAu5lFjslo6blKA-WTAjeOfLuw,42183
 ddeutil/workflow/api/__init__.py,sha256=F53NMBWtb9IKaDWkPU5KvybGGfKAcbehgn6TLBwHuuM,21
 ddeutil/workflow/api/api.py,sha256=Md1cz3Edc7_uz63s_L_i-R3IE4mkO3aTADrX8GOGU-Y,5644
 ddeutil/workflow/api/repeat.py,sha256=zyvsrXKk-3-_N8ZRZSki0Mueshugum2jtqctEOp9QSc,4927
 ddeutil/workflow/api/route.py,sha256=v96jNbgjM1cJ2MpVSRWs2kgRqF8DQElEBdRZrVFEpEw,8578
-ddeutil_workflow-0.0.26.post0.dist-info/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
-ddeutil_workflow-0.0.26.post0.dist-info/METADATA,sha256=GRkczE6ZJ7NhFefqGI2GbzGd6Lu2FDGO6oUafA8n4nw,14364
-ddeutil_workflow-0.0.26.post0.dist-info/WHEEL,sha256=A3WOREP4zgxI0fKrHUG8DC8013e3dK3n7a6HDbcEIwE,91
-ddeutil_workflow-0.0.26.post0.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
-ddeutil_workflow-0.0.26.post0.dist-info/RECORD,,
+ddeutil_workflow-0.0.26.post1.dist-info/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
+ddeutil_workflow-0.0.26.post1.dist-info/METADATA,sha256=B95z9M1Z9DWiKXQr1VoRvtlYcB6eX11RGktlAwn4MvI,14364
+ddeutil_workflow-0.0.26.post1.dist-info/WHEEL,sha256=A3WOREP4zgxI0fKrHUG8DC8013e3dK3n7a6HDbcEIwE,91
+ddeutil_workflow-0.0.26.post1.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
+ddeutil_workflow-0.0.26.post1.dist-info/RECORD,,