ddeutil-workflow 0.0.47__tar.gz → 0.0.48__tar.gz

{ddeutil_workflow-0.0.47/src/ddeutil_workflow.egg-info → ddeutil_workflow-0.0.48}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddeutil-workflow
-Version: 0.0.47
+Version: 0.0.48
 Summary: Lightweight workflow orchestration
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT
@@ -260,31 +260,31 @@ it will use default value and do not raise any error to you.
 > The config value that you will set on the environment should combine with
 > prefix, component, and name which is `WORKFLOW_{component}_{name}` (Upper case).
 
-| Name                         | Component | Default                                                                                                                         | Override | Description                                                                                                        |
-|:-----------------------------|:---------:|:--------------------------------------------------------------------------------------------------------------------------------|:--------:|:-------------------------------------------------------------------------------------------------------------------|
-| **ROOT_PATH**                |   Core    | `.`                                                                                                                             |    No    | The root path of the workflow application.                                                                         |
-| **REGISTRY_CALLER**          |   Core    | `.`                                                                                                                             |   Yes    | List of importable string for the call stage.                                                                      |
-| **REGISTRY_FILTER**          |   Core    | `ddeutil.workflow.templates`                                                                                                    |   Yes    | List of importable string for the filter template.                                                                 |
-| **CONF_PATH**                |   Core    | `conf`                                                                                                                          |   Yes    | The config path that keep all template `.yaml` files.                                                              |
-| **TIMEZONE**                 |   Core    | `Asia/Bangkok`                                                                                                                  |    No    | A Timezone string value that will pass to `ZoneInfo` object.                                                       |
-| **STAGE_DEFAULT_ID**         |   Core    | `true`                                                                                                                          |   Yes    | A flag that enable default stage ID that use for catch an execution output.                                        |
-| **STAGE_RAISE_ERROR**        |   Core    | `false`                                                                                                                         |   Yes    | A flag that all stage raise StageException from stage execution.                                                   |
-| **JOB_DEFAULT_ID**           |   Core    | `false`                                                                                                                         |   Yes    | A flag that enable default job ID that use for catch an execution output. The ID that use will be sequence number. |
-| **JOB_RAISE_ERROR**          |   Core    | `true`                                                                                                                          |   Yes    | A flag that all job raise JobException from job strategy execution.                                                |
-| **MAX_CRON_PER_WORKFLOW**    |   Core    | `5`                                                                                                                             |    No    |                                                                                                                    |
-| **MAX_QUEUE_COMPLETE_HIST**  |   Core    | `16`                                                                                                                            |    No    |                                                                                                                    |
-| **GENERATE_ID_SIMPLE_MODE**  |   Core    | `true`                                                                                                                          |    No    | A flog that enable generating ID with `md5` algorithm.                                                             |
-| **DEBUG_MODE**               |    Log    | `true`                                                                                                                          |    No    | A flag that enable logging with debug level mode.                                                                  |
-| **FORMAT**                   |    Log    | `%(asctime)s.%(msecs)03d (%(name)-10s, %(process)-5d,%(thread)-5d) [%(levelname)-7s] %(message)-120s (%(filename)s:%(lineno)s)` |    No    |                                                                                                                    |
-| **FORMAT_FILE**              |    Log    | `{datetime} ({process:5d}, {thread:5d}) {message:120s} ({filename}:{lineno})`                                                   |    No    |                                                                                                                    |
-| **DATETIME_FORMAT**          |    Log    | `%Y-%m-%d %H:%M:%S`                                                                                                             |    No    |                                                                                                                    |
-| **TRACE_PATH**               |    Log    | `./logs`                                                                                                                        |    No    | The log path of the workflow saving log.                                                                           |
-| **TRACE_ENABLE_WRITE**       |    Log    | `false`                                                                                                                         |    No    |                                                                                                                    |
-| **AUDIT_PATH**               |    Log    | `./audits`                                                                                                                      |    No    |                                                                                                                    |
-| **AUDIT_ENABLE_WRITE**       |    Log    | `true`                                                                                                                          |    No    | A flag that enable logging object saving log to its destination.                                                   |
-| **MAX_PROCESS**              |    App    | `2`                                                                                                                             |    No    | The maximum process worker number that run in scheduler app module.                                                |
-| **MAX_SCHEDULE_PER_PROCESS** |    App    | `100`                                                                                                                           |    No    | A schedule per process that run parallel.                                                                          |
-| **STOP_BOUNDARY_DELTA**      |    App    | `'{"minutes": 5, "seconds": 20}'`                                                                                               |    No    | A time delta value that use to stop scheduler app in json string format.                                           |
+| Name                         | Component | Default                                                                                                                         | Description                                                                                                        |
+|:-----------------------------|:---------:|:--------------------------------------------------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------|
+| **ROOT_PATH**                |   Core    | `.`                                                                                                                             | Root path or the project path for this workflow engine.                                                            |
+| **REGISTRY_CALLER**          |   Core    | `.`                                                                                                                             | List of importable string for the call stage.                                                                      |
+| **REGISTRY_FILTER**          |   Core    | `ddeutil.workflow.templates`                                                                                                    | List of importable string for the filter template.                                                                 |
+| **CONF_PATH**                |   Core    | `conf`                                                                                                                          | The config path that keep all template `.yaml` files.                                                              |
+| **TIMEZONE**                 |   Core    | `Asia/Bangkok`                                                                                                                  | A Timezone string value that will pass to `ZoneInfo` object.                                                       |
+| **STAGE_DEFAULT_ID**         |   Core    | `true`                                                                                                                          | A flag that enable default stage ID that use for catch an execution output.                                        |
+| **STAGE_RAISE_ERROR**        |   Core    | `false`                                                                                                                         | A flag that all stage raise StageException from stage execution.                                                   |
+| **JOB_DEFAULT_ID**           |   Core    | `false`                                                                                                                         | A flag that enable default job ID that use for catch an execution output. The ID that use will be sequence number. |
+| **JOB_RAISE_ERROR**          |   Core    | `true`                                                                                                                          | A flag that all job raise JobException from job strategy execution.                                                |
+| **MAX_CRON_PER_WORKFLOW**    |   Core    | `5`                                                                                                                             |                                                                                                                    |
+| **MAX_QUEUE_COMPLETE_HIST**  |   Core    | `16`                                                                                                                            |                                                                                                                    |
+| **GENERATE_ID_SIMPLE_MODE**  |   Core    | `true`                                                                                                                          | A flog that enable generating ID with `md5` algorithm.                                                             |
+| **DEBUG_MODE**               |    Log    | `true`                                                                                                                          | A flag that enable logging with debug level mode.                                                                  |
+| **FORMAT**                   |    Log    | `%(asctime)s.%(msecs)03d (%(name)-10s, %(process)-5d,%(thread)-5d) [%(levelname)-7s] %(message)-120s (%(filename)s:%(lineno)s)` |                                                                                                                    |
+| **FORMAT_FILE**              |    Log    | `{datetime} ({process:5d}, {thread:5d}) {message:120s} ({filename}:{lineno})`                                                   |                                                                                                                    |
+| **DATETIME_FORMAT**          |    Log    | `%Y-%m-%d %H:%M:%S`                                                                                                             |                                                                                                                    |
+| **TRACE_PATH**               |    Log    | `./logs`                                                                                                                        | The log path of the workflow saving log.                                                                           |
+| **TRACE_ENABLE_WRITE**       |    Log    | `false`                                                                                                                         |                                                                                                                    |
+| **AUDIT_PATH**               |    Log    | `./audits`                                                                                                                      |                                                                                                                    |
+| **AUDIT_ENABLE_WRITE**       |    Log    | `true`                                                                                                                          | A flag that enable logging object saving log to its destination.                                                   |
+| **MAX_PROCESS**              |    App    | `2`                                                                                                                             | The maximum process worker number that run in scheduler app module.                                                |
+| **MAX_SCHEDULE_PER_PROCESS** |    App    | `100`                                                                                                                           | A schedule per process that run parallel.                                                                          |
+| **STOP_BOUNDARY_DELTA**      |    App    | `'{"minutes": 5, "seconds": 20}'`                                                                                               | A time delta value that use to stop scheduler app in json string format.                                           |
 
 **API Application**:
 

{ddeutil_workflow-0.0.47 → ddeutil_workflow-0.0.48}/README.md

@@ -217,31 +217,31 @@ it will use default value and do not raise any error to you.
 > The config value that you will set on the environment should combine with
 > prefix, component, and name which is `WORKFLOW_{component}_{name}` (Upper case).
 
-| Name                         | Component | Default                                                                                                                         | Override | Description                                                                                                        |
-|:-----------------------------|:---------:|:--------------------------------------------------------------------------------------------------------------------------------|:--------:|:-------------------------------------------------------------------------------------------------------------------|
-| **ROOT_PATH**                |   Core    | `.`                                                                                                                             |    No    | The root path of the workflow application.                                                                         |
-| **REGISTRY_CALLER**          |   Core    | `.`                                                                                                                             |   Yes    | List of importable string for the call stage.                                                                      |
-| **REGISTRY_FILTER**          |   Core    | `ddeutil.workflow.templates`                                                                                                    |   Yes    | List of importable string for the filter template.                                                                 |
-| **CONF_PATH**                |   Core    | `conf`                                                                                                                          |   Yes    | The config path that keep all template `.yaml` files.                                                              |
-| **TIMEZONE**                 |   Core    | `Asia/Bangkok`                                                                                                                  |    No    | A Timezone string value that will pass to `ZoneInfo` object.                                                       |
-| **STAGE_DEFAULT_ID**         |   Core    | `true`                                                                                                                          |   Yes    | A flag that enable default stage ID that use for catch an execution output.                                        |
-| **STAGE_RAISE_ERROR**        |   Core    | `false`                                                                                                                         |   Yes    | A flag that all stage raise StageException from stage execution.                                                   |
-| **JOB_DEFAULT_ID**           |   Core    | `false`                                                                                                                         |   Yes    | A flag that enable default job ID that use for catch an execution output. The ID that use will be sequence number. |
-| **JOB_RAISE_ERROR**          |   Core    | `true`                                                                                                                          |   Yes    | A flag that all job raise JobException from job strategy execution.                                                |
-| **MAX_CRON_PER_WORKFLOW**    |   Core    | `5`                                                                                                                             |    No    |                                                                                                                    |
-| **MAX_QUEUE_COMPLETE_HIST**  |   Core    | `16`                                                                                                                            |    No    |                                                                                                                    |
-| **GENERATE_ID_SIMPLE_MODE**  |   Core    | `true`                                                                                                                          |    No    | A flog that enable generating ID with `md5` algorithm.                                                             |
-| **DEBUG_MODE**               |    Log    | `true`                                                                                                                          |    No    | A flag that enable logging with debug level mode.                                                                  |
-| **FORMAT**                   |    Log    | `%(asctime)s.%(msecs)03d (%(name)-10s, %(process)-5d,%(thread)-5d) [%(levelname)-7s] %(message)-120s (%(filename)s:%(lineno)s)` |    No    |                                                                                                                    |
-| **FORMAT_FILE**              |    Log    | `{datetime} ({process:5d}, {thread:5d}) {message:120s} ({filename}:{lineno})`                                                   |    No    |                                                                                                                    |
-| **DATETIME_FORMAT**          |    Log    | `%Y-%m-%d %H:%M:%S`                                                                                                             |    No    |                                                                                                                    |
-| **TRACE_PATH**               |    Log    | `./logs`                                                                                                                        |    No    | The log path of the workflow saving log.                                                                           |
-| **TRACE_ENABLE_WRITE**       |    Log    | `false`                                                                                                                         |    No    |                                                                                                                    |
-| **AUDIT_PATH**               |    Log    | `./audits`                                                                                                                      |    No    |                                                                                                                    |
-| **AUDIT_ENABLE_WRITE**       |    Log    | `true`                                                                                                                          |    No    | A flag that enable logging object saving log to its destination.                                                   |
-| **MAX_PROCESS**              |    App    | `2`                                                                                                                             |    No    | The maximum process worker number that run in scheduler app module.                                                |
-| **MAX_SCHEDULE_PER_PROCESS** |    App    | `100`                                                                                                                           |    No    | A schedule per process that run parallel.                                                                          |
-| **STOP_BOUNDARY_DELTA**      |    App    | `'{"minutes": 5, "seconds": 20}'`                                                                                               |    No    | A time delta value that use to stop scheduler app in json string format.                                           |
+| Name                         | Component | Default                                                                                                                         | Description                                                                                                        |
+|:-----------------------------|:---------:|:--------------------------------------------------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------|
+| **ROOT_PATH**                |   Core    | `.`                                                                                                                             | Root path or the project path for this workflow engine.                                                            |
+| **REGISTRY_CALLER**          |   Core    | `.`                                                                                                                             | List of importable string for the call stage.                                                                      |
+| **REGISTRY_FILTER**          |   Core    | `ddeutil.workflow.templates`                                                                                                    | List of importable string for the filter template.                                                                 |
+| **CONF_PATH**                |   Core    | `conf`                                                                                                                          | The config path that keep all template `.yaml` files.                                                              |
+| **TIMEZONE**                 |   Core    | `Asia/Bangkok`                                                                                                                  | A Timezone string value that will pass to `ZoneInfo` object.                                                       |
+| **STAGE_DEFAULT_ID**         |   Core    | `true`                                                                                                                          | A flag that enable default stage ID that use for catch an execution output.                                        |
+| **STAGE_RAISE_ERROR**        |   Core    | `false`                                                                                                                         | A flag that all stage raise StageException from stage execution.                                                   |
+| **JOB_DEFAULT_ID**           |   Core    | `false`                                                                                                                         | A flag that enable default job ID that use for catch an execution output. The ID that use will be sequence number. |
+| **JOB_RAISE_ERROR**          |   Core    | `true`                                                                                                                          | A flag that all job raise JobException from job strategy execution.                                                |
+| **MAX_CRON_PER_WORKFLOW**    |   Core    | `5`                                                                                                                             |                                                                                                                    |
+| **MAX_QUEUE_COMPLETE_HIST**  |   Core    | `16`                                                                                                                            |                                                                                                                    |
+| **GENERATE_ID_SIMPLE_MODE**  |   Core    | `true`                                                                                                                          | A flog that enable generating ID with `md5` algorithm.                                                             |
+| **DEBUG_MODE**               |    Log    | `true`                                                                                                                          | A flag that enable logging with debug level mode.                                                                  |
+| **FORMAT**                   |    Log    | `%(asctime)s.%(msecs)03d (%(name)-10s, %(process)-5d,%(thread)-5d) [%(levelname)-7s] %(message)-120s (%(filename)s:%(lineno)s)` |                                                                                                                    |
+| **FORMAT_FILE**              |    Log    | `{datetime} ({process:5d}, {thread:5d}) {message:120s} ({filename}:{lineno})`                                                   |                                                                                                                    |
+| **DATETIME_FORMAT**          |    Log    | `%Y-%m-%d %H:%M:%S`                                                                                                             |                                                                                                                    |
+| **TRACE_PATH**               |    Log    | `./logs`                                                                                                                        | The log path of the workflow saving log.                                                                           |
+| **TRACE_ENABLE_WRITE**       |    Log    | `false`                                                                                                                         |                                                                                                                    |
+| **AUDIT_PATH**               |    Log    | `./audits`                                                                                                                      |                                                                                                                    |
+| **AUDIT_ENABLE_WRITE**       |    Log    | `true`                                                                                                                          | A flag that enable logging object saving log to its destination.                                                   |
+| **MAX_PROCESS**              |    App    | `2`                                                                                                                             | The maximum process worker number that run in scheduler app module.                                                |
+| **MAX_SCHEDULE_PER_PROCESS** |    App    | `100`                                                                                                                           | A schedule per process that run parallel.                                                                          |
+| **STOP_BOUNDARY_DELTA**      |    App    | `'{"minutes": 5, "seconds": 20}'`                                                                                               | A time delta value that use to stop scheduler app in json string format.                                           |
 
 **API Application**:
 

ddeutil_workflow-0.0.48/src/ddeutil/workflow/__about__.py

@@ -0,0 +1 @@
+__version__: str = "0.0.48"

{ddeutil_workflow-0.0.47 → ddeutil_workflow-0.0.48}/src/ddeutil/workflow/__init__.py

@@ -10,7 +10,6 @@ from .conf import (
     Loader,
     config,
     env,
-    get_logger,
 )
 from .cron import (
     On,

{ddeutil_workflow-0.0.47 → ddeutil_workflow-0.0.48}/src/ddeutil/workflow/api/api.py

@@ -20,7 +20,8 @@ from fastapi.middleware.gzip import GZipMiddleware
 from fastapi.responses import UJSONResponse
 
 from ..__about__ import __version__
-from ..conf import api_config, config, get_logger
+from ..conf import api_config, config
+from ..logs import get_logger
 from ..scheduler import ReleaseThread, ReleaseThreads
 from ..workflow import ReleaseQueue, WorkflowTask
 from .repeat import repeat_at

{ddeutil_workflow-0.0.47 → ddeutil_workflow-0.0.48}/src/ddeutil/workflow/api/repeat.py

@@ -13,7 +13,8 @@ from functools import wraps
 from starlette.concurrency import run_in_threadpool
 
 from ..__cron import CronJob
-from ..conf import config, get_logger
+from ..conf import config
+from ..logs import get_logger
 
 logger = get_logger("uvicorn.error")
 

{ddeutil_workflow-0.0.47 → ddeutil_workflow-0.0.48}/src/ddeutil/workflow/api/routes/job.py

@@ -12,9 +12,9 @@ from fastapi.responses import UJSONResponse
 from pydantic import BaseModel
 
 from ...__types import DictData
-from ...conf import get_logger
 from ...exceptions import JobException
 from ...job import Job
+from ...logs import get_logger
 from ...result import Result
 
 logger = get_logger("uvicorn.error")

{ddeutil_workflow-0.0.47 → ddeutil_workflow-0.0.48}/src/ddeutil/workflow/api/routes/schedules.py

@@ -12,7 +12,8 @@ from fastapi import APIRouter, HTTPException, Request
 from fastapi import status as st
 from fastapi.responses import UJSONResponse
 
-from ...conf import config, get_logger
+from ...conf import config
+from ...logs import get_logger
 from ...scheduler import Schedule
 
 logger = get_logger("uvicorn.error")

{ddeutil_workflow-0.0.47 → ddeutil_workflow-0.0.48}/src/ddeutil/workflow/api/routes/workflows.py

@@ -15,8 +15,8 @@ from fastapi.responses import UJSONResponse
 from pydantic import BaseModel
 
 from ...__types import DictData
-from ...conf import Loader, get_logger
-from ...logs import Audit, get_audit
+from ...conf import Loader
+from ...logs import Audit, get_audit, get_logger
 from ...result import Result
 from ...workflow import Workflow
 

{ddeutil_workflow-0.0.47 → ddeutil_workflow-0.0.48}/src/ddeutil/workflow/conf.py

@@ -6,11 +6,10 @@
 from __future__ import annotations
 
 import json
-import logging
 import os
 from collections.abc import Iterator
 from datetime import timedelta
-from functools import cached_property, lru_cache
+from functools import cached_property
 from pathlib import Path
 from typing import Optional, TypeVar
 from zoneinfo import ZoneInfo
@@ -36,7 +35,6 @@ def env(var: str, default: str | None = None) -> str | None:  # pragma: no cov
 __all__: TupleStr = (
     "api_config",
     "env",
-    "get_logger",
     "Config",
     "SimLoad",
     "Loader",
@@ -55,7 +53,8 @@ class Config:  # pragma: no cov
     # NOTE: Core
     @property
     def root_path(self) -> Path:
-        """Root path or the project path.
+        """Root path or the project path for this workflow engine that use for
+        combine with `conf_path` value.
 
         :rtype: Path
         """
@@ -63,7 +62,7 @@ class Config:  # pragma: no cov
 
     @property
     def conf_path(self) -> Path:
-        """Config path that use root_path class argument for this construction.
+        """Config path that keep all workflow template YAML files.
 
         :rtype: Path
         """
@@ -71,6 +70,11 @@ class Config:  # pragma: no cov
 
     @property
     def tz(self) -> ZoneInfo:
+        """Timezone value that return with the `ZoneInfo` object and use for all
+        datetime object in this workflow engine.
+
+        :rtype: ZoneInfo
+        """
         return ZoneInfo(env("CORE_TIMEZONE", "UTC"))
 
     @property
@@ -80,7 +84,8 @@ class Config:  # pragma: no cov
     # NOTE: Register
     @property
     def regis_call(self) -> list[str]:
-        """Register Caller module importer str.
+        """Register Caller that is a list of importable string for the call
+        stage model can get.
 
         :rtype: list[str]
         """
@@ -89,7 +94,8 @@ class Config:  # pragma: no cov
 
     @property
     def regis_filter(self) -> list[str]:
-        """Register Filter module.
+        """Register Filter that is a list of importable string for the filter
+        template.
 
         :rtype: list[str]
         """
@@ -222,15 +228,13 @@ class SimLoad:
     :param externals: An external parameters
 
     Noted:
-
         The config data should have ``type`` key for modeling validation that
     make this loader know what is config should to do pass to.
 
         ... <identity-key>:
         ...     type: <importable-object>
-        ...     <key-data>: <value-data>
-        ...     ...
-
+        ...     <key-data-1>: <value-data-1>
+        ...     <key-data-2>: <value-data-2>
     """
 
     def __init__(
@@ -253,7 +257,10 @@ class SimLoad:
 
         # VALIDATE: check the data that reading should not empty.
         if not self.data:
-            raise ValueError(f"Config {name!r} does not found on conf path")
+            raise ValueError(
+                f"Config {name!r} does not found on conf path: "
+                f"{self.conf_path}."
+            )
 
         self.data.update(self.externals)
 
@@ -299,10 +306,25 @@ class SimLoad:
 
     @classmethod
     def is_ignore(cls, file: Path, conf_path: Path) -> bool:
+        """Check this file was ignored.
+
+        :param file: (Path) A file path that want to check.
+        :param conf_path: (Path) A config path that want to read the config
+            ignore file.
+
+        :rtype: bool
+        """
         return is_ignored(file, read_ignore(conf_path / ".confignore"))
 
     @classmethod
     def filter_yaml(cls, file: Path, name: str | None = None) -> DictData:
+        """Read a YAML file context from an input file path and specific name.
+
+        :param file: (Path)
+        :param name: (str)
+
+        :rtype: DictData
+        """
         if any(file.suffix.endswith(s) for s in (".yml", ".yaml")):
             values: DictData = YamlFlResolve(file).read()
             if values is not None:
@@ -388,30 +410,3 @@ class Loader(SimLoad):
             conf_path=dynamic("conf_path", extras=externals),
             externals=externals,
         )
-
-
-@lru_cache
-def get_logger(name: str):
-    """Return logger object with an input module name.
-
-    :param name: A module name that want to log.
-    """
-    logger = logging.getLogger(name)
-
-    # NOTE: Developers using this package can then disable all logging just for
-    #   this package by;
-    #
-    #   `logging.getLogger('ddeutil.workflow').propagate = False`
-    #
-    logger.addHandler(logging.NullHandler())
-
-    formatter = logging.Formatter(
-        fmt=config.log_format,
-        datefmt=config.log_datetime_format,
-    )
-    stream = logging.StreamHandler()
-    stream.setFormatter(formatter)
-    logger.addHandler(stream)
-
-    logger.setLevel(logging.DEBUG if config.debug else logging.INFO)
-    return logger

{ddeutil_workflow-0.0.47 → ddeutil_workflow-0.0.48}/src/ddeutil/workflow/job.py

@@ -570,6 +570,7 @@ class Job(BaseModel):
             run_id=run_id,
             parent_run_id=parent_run_id,
             id_logic=(self.id or "not-set"),
+            extras=self.extras,
         )
 
         if self.runs_on.type == RunsOnType.LOCAL:
@@ -756,6 +757,7 @@ def local_execute(
         run_id=run_id,
         parent_run_id=parent_run_id,
         id_logic=(job.id or "not-set"),
+        extras=job.extras,
     )
     event: Event = Event() if event is None else event
 
@@ -891,6 +893,7 @@ def self_hosted_execute(
         run_id=run_id,
         parent_run_id=parent_run_id,
         id_logic=(job.id or "not-set"),
+        extras=job.extras,
     )
 
     if event and event.is_set():