ddeutil-workflow 0.0.49__tar.gz → 0.0.51__tar.gz

{ddeutil_workflow-0.0.49/src/ddeutil_workflow.egg-info → ddeutil_workflow-0.0.51}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddeutil-workflow
-Version: 0.0.49
+Version: 0.0.51
 Summary: Lightweight workflow orchestration
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT
@@ -22,8 +22,8 @@ Classifier: Programming Language :: Python :: 3.13
 Requires-Python: >=3.9.13
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: ddeutil[checksum]>=0.4.6
-Requires-Dist: ddeutil-io[toml,yaml]>=0.2.10
+Requires-Dist: ddeutil[checksum]>=0.4.7
+Requires-Dist: ddeutil-io[toml,yaml]>=0.2.11
 Requires-Dist: pydantic==2.11.1
 Requires-Dist: python-dotenv==1.1.0
 Requires-Dist: schedule<2.0.0,==1.2.2
@@ -39,6 +39,8 @@ Requires-Dist: ujson; extra == "api"
 Provides-Extra: async
 Requires-Dist: aiofiles; extra == "async"
 Requires-Dist: aiohttp; extra == "async"
+Provides-Extra: docker
+Requires-Dist: docker==7.1.0; extra == "docker"
 Dynamic: license-file
 
 # Workflow Orchestration
@@ -137,7 +139,8 @@ flowchart LR
 
 ## 📦 Installation
 
-This project need `ddeutil` and `ddeutil-io` extension namespace packages.
+This project need `ddeutil` and `ddeutil-io` extension namespace packages to be
+the base deps.
 If you want to install this package with application add-ons, you should add
 `app` in installation;
 
@@ -146,7 +149,7 @@ If you want to install this package with application add-ons, you should add
 | Python         | `ddeutil-workflow`       | :heavy_check_mark:  |
 | FastAPI Server | `ddeutil-workflow[api]`  | :heavy_check_mark:  |
 
-## :beers: Usage
+## 🎯 Usage
 
 This is examples that use workflow file for running common Data Engineering
 use-case.
@@ -196,12 +199,54 @@ run-py-local:
 
                  # Arguments of target data that want to land.
                  writing_mode: flatten
-                 aws_s3_path: my-data/open-data/${{ params.source-extract }}
+                 aws:
+                    path: my-data/open-data/${{ params.source-extract }}
 
-                 # This Authentication code should implement with your custom call
-                 # function. The template allow you to use environment variable.
-                 aws_access_client_id: ${AWS_ACCESS_CLIENT_ID}
-                 aws_access_client_secret: ${AWS_ACCESS_CLIENT_SECRET}
+                    # This Authentication code should implement with your custom call
+                    # function. The template allow you to use environment variable.
+                    access_client_id: ${AWS_ACCESS_CLIENT_ID}
+                    access_client_secret: ${AWS_ACCESS_CLIENT_SECRET}
+```
+
+Before execute this workflow, you should implement caller function first.
+
+```text
+registry-caller/
+  ╰─ tasks.py
+```
+
+This function will store as module that will import from `WORKFLOW_CORE_REGISTRY_CALLER`
+value (This config can override by extra parameters with `registry_caller` key).
+
+```python
+from ddeutil.workflow import Result, tag
+from ddeutil.workflow.exceptions import StageException
+from pydantic import BaseModel, SecretStr
+
+class AwsCredential(BaseModel):
+    path: str
+    access_client_id: str
+    access_client_secret: SecretStr
+
+class RestAuth(BaseModel):
+    type: str
+    keys: SecretStr
+
+@tag("requests", alias="get-api-with-oauth-to-s3")
+def get_api_with_oauth_to_s3(
+    method: str,
+    url: str,
+    body: dict[str, str],
+    auth: RestAuth,
+    writing_node: str,
+    aws: AwsCredential,
+    result: Result,
+) -> dict[str, int]:
+    result.trace.info("[CALLER]: Start get data via RestAPI to S3.")
+    result.trace.info(f"... {method}: {url}")
+    if method != "post":
+       raise StageException(f"RestAPI does not support for {method} action.")
+    return {"records": 1000}
 ```
 
 The above workflow template is main executor pipeline that you want to do. If you
@@ -268,7 +313,6 @@ it will use default value and do not raise any error to you.
 | **TIMEZONE**                 |   Core    | `Asia/Bangkok`                                                                                                                  | A Timezone string value that will pass to `ZoneInfo` object.                                                       |
 | **STAGE_DEFAULT_ID**         |   Core    | `false`                                                                                                                         | A flag that enable default stage ID that use for catch an execution output.                                        |
 | **STAGE_RAISE_ERROR**        |   Core    | `false`                                                                                                                         | A flag that all stage raise StageException from stage execution.                                                   |
-| **JOB_RAISE_ERROR**          |   Core    | `true`                                                                                                                          | A flag that all job raise JobException from job strategy execution.                                                |
 | **MAX_CRON_PER_WORKFLOW**    |   Core    | `5`                                                                                                                             |                                                                                                                    |
 | **MAX_QUEUE_COMPLETE_HIST**  |   Core    | `16`                                                                                                                            |                                                                                                                    |
 | **GENERATE_ID_SIMPLE_MODE**  |   Core    | `true`                                                                                                                          | A flog that enable generating ID with `md5` algorithm.                                                             |
@@ -297,13 +341,15 @@ only.
 ## :rocket: Deployment
 
 This package able to run as an application service for receive manual trigger
-from the master node via RestAPI or use to be Scheduler background service
-like crontab job but via Python API.
+from any node via RestAPI or use to be Scheduler background application
+like crontab job but via Python API or FastAPI app.
 
 ### API Server
 
+This server use FastAPI package to be the base application.
+
 ```shell
-(venv) $ uvicorn ddeutil.workflow.api:app \
+(.venv) $ uvicorn ddeutil.workflow.api:app \
   --host 127.0.0.1 \
   --port 80 \
   --no-access-log
@@ -313,8 +359,19 @@ like crontab job but via Python API.
 > If this package already deploy, it is able to use multiprocess;
 > `uvicorn ddeutil.workflow.api:app --host 127.0.0.1 --port 80 --workers 4`
 
+### Local Schedule
+
+> [!WARNING]
+> This CLI does not implement yet.
+
+```shell
+(.venv) $ ddeutil-workflow schedule
+```
+
 ### Docker Container
 
+Build a Docker container from this package.
+
 ```shell
 $ docker build -t ddeutil-workflow:latest -f .container/Dockerfile .
 $ docker run -i ddeutil-workflow:latest ddeutil-workflow

{ddeutil_workflow-0.0.49 → ddeutil_workflow-0.0.51}/README.md

@@ -94,7 +94,8 @@ flowchart LR
 
 ## 📦 Installation
 
-This project need `ddeutil` and `ddeutil-io` extension namespace packages.
+This project need `ddeutil` and `ddeutil-io` extension namespace packages to be
+the base deps.
 If you want to install this package with application add-ons, you should add
 `app` in installation;
 
@@ -103,7 +104,7 @@ If you want to install this package with application add-ons, you should add
 | Python         | `ddeutil-workflow`       | :heavy_check_mark:  |
 | FastAPI Server | `ddeutil-workflow[api]`  | :heavy_check_mark:  |
 
-## :beers: Usage
+## 🎯 Usage
 
 This is examples that use workflow file for running common Data Engineering
 use-case.
@@ -153,12 +154,54 @@ run-py-local:
 
                  # Arguments of target data that want to land.
                  writing_mode: flatten
-                 aws_s3_path: my-data/open-data/${{ params.source-extract }}
+                 aws:
+                    path: my-data/open-data/${{ params.source-extract }}
 
-                 # This Authentication code should implement with your custom call
-                 # function. The template allow you to use environment variable.
-                 aws_access_client_id: ${AWS_ACCESS_CLIENT_ID}
-                 aws_access_client_secret: ${AWS_ACCESS_CLIENT_SECRET}
+                    # This Authentication code should implement with your custom call
+                    # function. The template allow you to use environment variable.
+                    access_client_id: ${AWS_ACCESS_CLIENT_ID}
+                    access_client_secret: ${AWS_ACCESS_CLIENT_SECRET}
+```
+
+Before execute this workflow, you should implement caller function first.
+
+```text
+registry-caller/
+  ╰─ tasks.py
+```
+
+This function will store as module that will import from `WORKFLOW_CORE_REGISTRY_CALLER`
+value (This config can override by extra parameters with `registry_caller` key).
+
+```python
+from ddeutil.workflow import Result, tag
+from ddeutil.workflow.exceptions import StageException
+from pydantic import BaseModel, SecretStr
+
+class AwsCredential(BaseModel):
+    path: str
+    access_client_id: str
+    access_client_secret: SecretStr
+
+class RestAuth(BaseModel):
+    type: str
+    keys: SecretStr
+
+@tag("requests", alias="get-api-with-oauth-to-s3")
+def get_api_with_oauth_to_s3(
+    method: str,
+    url: str,
+    body: dict[str, str],
+    auth: RestAuth,
+    writing_node: str,
+    aws: AwsCredential,
+    result: Result,
+) -> dict[str, int]:
+    result.trace.info("[CALLER]: Start get data via RestAPI to S3.")
+    result.trace.info(f"... {method}: {url}")
+    if method != "post":
+       raise StageException(f"RestAPI does not support for {method} action.")
+    return {"records": 1000}
 ```
 
 The above workflow template is main executor pipeline that you want to do. If you
@@ -225,7 +268,6 @@ it will use default value and do not raise any error to you.
 | **TIMEZONE**                 |   Core    | `Asia/Bangkok`                                                                                                                  | A Timezone string value that will pass to `ZoneInfo` object.                                                       |
 | **STAGE_DEFAULT_ID**         |   Core    | `false`                                                                                                                         | A flag that enable default stage ID that use for catch an execution output.                                        |
 | **STAGE_RAISE_ERROR**        |   Core    | `false`                                                                                                                         | A flag that all stage raise StageException from stage execution.                                                   |
-| **JOB_RAISE_ERROR**          |   Core    | `true`                                                                                                                          | A flag that all job raise JobException from job strategy execution.                                                |
 | **MAX_CRON_PER_WORKFLOW**    |   Core    | `5`                                                                                                                             |                                                                                                                    |
 | **MAX_QUEUE_COMPLETE_HIST**  |   Core    | `16`                                                                                                                            |                                                                                                                    |
 | **GENERATE_ID_SIMPLE_MODE**  |   Core    | `true`                                                                                                                          | A flog that enable generating ID with `md5` algorithm.                                                             |
@@ -254,13 +296,15 @@ only.
 ## :rocket: Deployment
 
 This package able to run as an application service for receive manual trigger
-from the master node via RestAPI or use to be Scheduler background service
-like crontab job but via Python API.
+from any node via RestAPI or use to be Scheduler background application
+like crontab job but via Python API or FastAPI app.
 
 ### API Server
 
+This server use FastAPI package to be the base application.
+
 ```shell
-(venv) $ uvicorn ddeutil.workflow.api:app \
+(.venv) $ uvicorn ddeutil.workflow.api:app \
   --host 127.0.0.1 \
   --port 80 \
   --no-access-log
@@ -270,8 +314,19 @@ like crontab job but via Python API.
 > If this package already deploy, it is able to use multiprocess;
 > `uvicorn ddeutil.workflow.api:app --host 127.0.0.1 --port 80 --workers 4`
 
+### Local Schedule
+
+> [!WARNING]
+> This CLI does not implement yet.
+
+```shell
+(.venv) $ ddeutil-workflow schedule
+```
+
 ### Docker Container
 
+Build a Docker container from this package.
+
 ```shell
 $ docker build -t ddeutil-workflow:latest -f .container/Dockerfile .
 $ docker run -i ddeutil-workflow:latest ddeutil-workflow

{ddeutil_workflow-0.0.49 → ddeutil_workflow-0.0.51}/pyproject.toml

@@ -25,8 +25,8 @@ classifiers = [
 ]
 requires-python = ">=3.9.13"
 dependencies = [
-    "ddeutil[checksum]>=0.4.6",
-    "ddeutil-io[yaml,toml]>=0.2.10",
+    "ddeutil[checksum]>=0.4.7",
+    "ddeutil-io[yaml,toml]>=0.2.11",
     "pydantic==2.11.1",
     "python-dotenv==1.1.0",
     "schedule==1.2.2,<2.0.0",
@@ -49,6 +49,9 @@ async = [
     "aiofiles",
     "aiohttp",
 ]
+docker = [
+    "docker==7.1.0"
+]
 
 [project.urls]
 Homepage = "https://github.com/ddeutils/ddeutil-workflow/"

ddeutil_workflow-0.0.51/src/ddeutil/workflow/__about__.py

@@ -0,0 +1 @@
+__version__: str = "0.0.51"

{ddeutil_workflow-0.0.49 → ddeutil_workflow-0.0.51}/src/ddeutil/workflow/__init__.py

@@ -11,25 +11,9 @@ from .conf import (
     config,
     env,
 )
-from .cron import (
-    On,
-    YearOn,
-    interval2crontab,
-)
-from .exceptions import (
-    JobException,
-    ParamValueException,
-    StageException,
-    UtilException,
-    WorkflowException,
-)
-from .job import (
-    Job,
-    RunsOn,
-    Strategy,
-    local_execute,
-    local_execute_strategy,
-)
+from .cron import *
+from .exceptions import *
+from .job import *
 from .logs import (
     Audit,
     AuditModel,
@@ -41,14 +25,9 @@ from .logs import (
     get_dt_tznow,
     get_trace,
 )
-from .params import (
-    ChoiceParam,
-    DatetimeParam,
-    IntParam,
-    Param,
-    StrParam,
-)
+from .params import *
 from .result import (
+    CANCEL,
     FAILED,
     SKIP,
     SUCCESS,
@@ -101,6 +80,9 @@ from .utils import (
     get_diff_sec,
     get_dt_now,
     make_exec,
+    reach_next_minute,
+    replace_sec,
+    wait_to_next_minute,
 )
 from .workflow import (
     Release,

{ddeutil_workflow-0.0.49 → ddeutil_workflow-0.0.51}/src/ddeutil/workflow/conf.py

@@ -53,7 +53,6 @@ class Config:  # pragma: no cov
         The config value can change when you call that config property again.
     """
 
-    # NOTE: Core
     @property
     def conf_path(self) -> Path:
         """Config path that keep all workflow template YAML files.
@@ -73,9 +72,13 @@ class Config:  # pragma: no cov
 
     @property
     def generate_id_simple_mode(self) -> bool:
+        """Flag for generate running ID with simple mode. That does not use
+        `md5` function after generate simple mode.
+
+        :rtype: bool
+        """
         return str2bool(env("CORE_GENERATE_ID_SIMPLE_MODE", "true"))
 
-    # NOTE: Register
     @property
     def registry_caller(self) -> list[str]:
         """Register Caller that is a list of importable string for the call
@@ -98,13 +101,16 @@ class Config:  # pragma: no cov
         )
         return [r.strip() for r in regis_filter_str.split(",")]
 
-    # NOTE: Log
     @property
     def trace_path(self) -> Path:
         return Path(env("LOG_TRACE_PATH", "./logs"))
 
     @property
     def debug(self) -> bool:
+        """Debug flag for echo log that use DEBUG mode.
+
+        :rtype: bool
+        """
         return str2bool(env("LOG_DEBUG_MODE", "true"))
 
     @property
@@ -144,7 +150,6 @@ class Config:  # pragma: no cov
     def log_datetime_format(self) -> str:
         return env("LOG_DATETIME_FORMAT", "%Y-%m-%d %H:%M:%S")
 
-    # NOTE: Stage
     @property
     def stage_raise_error(self) -> bool:
         return str2bool(env("CORE_STAGE_RAISE_ERROR", "false"))
@@ -153,11 +158,6 @@ class Config:  # pragma: no cov
     def stage_default_id(self) -> bool:
         return str2bool(env("CORE_STAGE_DEFAULT_ID", "false"))
 
-    # NOTE: Job
-    @property
-    def job_raise_error(self) -> bool:
-        return str2bool(env("CORE_JOB_RAISE_ERROR", "true"))
-
     @property
     def max_cron_per_workflow(self) -> int:
         """The maximum on value that store in workflow model.
@@ -360,13 +360,13 @@ def dynamic(
     :param extras: An extra values that pass at run-time.
     """
     rsx: Optional[T] = extras[key] if extras and key in extras else None
-    rs: Optional[T] = f or getattr(config, key, None)
+    rs: Optional[T] = getattr(config, key, None) if f is None else f
     if rsx is not None and not isinstance(rsx, type(rs)):
         raise TypeError(
             f"Type of config {key!r} from extras: {rsx!r} does not valid "
             f"as config {type(rs)}."
         )
-    return rsx or rs
+    return rsx if rsx is not None else rs
 
 
 class Loader(SimLoad):

{ddeutil_workflow-0.0.49 → ddeutil_workflow-0.0.51}/src/ddeutil/workflow/cron.py

@@ -16,18 +16,13 @@ from pydantic.functional_validators import field_validator, model_validator
 from typing_extensions import Self
 
 from .__cron import WEEKDAYS, CronJob, CronJobYear, CronRunner, Options
-from .__types import DictData, DictStr, TupleStr
+from .__types import DictData, DictStr
 from .conf import Loader
 
-__all__: TupleStr = (
-    "On",
-    "YearOn",
-    "interval2crontab",
-)
-
 
 def interval2crontab(
     interval: Literal["daily", "weekly", "monthly"],
+    *,
     day: str | None = None,
     time: str = "00:00",
 ) -> str:
@@ -67,7 +62,7 @@ class On(BaseModel):
     """On Pydantic model (Warped crontab object by model).
 
     See Also:
-        * ``generate()`` is the main use-case of this schedule object.
+        * `generate()` is the main use-case of this schedule object.
     """
 
     model_config = ConfigDict(arbitrary_types_allowed=True)
@@ -76,12 +71,18 @@ class On(BaseModel):
         DictData,
         Field(
             default_factory=dict,
-            description="An extras mapping parameters.",
+            description=(
+                "An extras parameters that want to pass to the CronJob field."
+            ),
         ),
     ]
     cronjob: Annotated[
         CronJob,
-        Field(description="A Cronjob object of this schedule."),
+        Field(
+            description=(
+                "A Cronjob object that use for validate and generate datetime.",
+            ),
+        ),
     ]
     tz: Annotated[
         str,
@@ -118,6 +119,8 @@ class On(BaseModel):
 
         :param name: A name of config that will get from loader.
         :param extras: An extra parameter that will keep in extras.
+
+        :rtype: Self
         """
         extras: DictData = extras or {}
         loader: Loader = Loader(name, externals=extras)
@@ -142,9 +145,7 @@ class On(BaseModel):
                 )
             )
         if "cronjob" not in loader_data:
-            raise ValueError(
-                "Config does not set ``cronjob`` or ``interval`` keys"
-            )
+            raise ValueError("Config does not set `cronjob` or `interval` keys")
         return cls.model_validate(
             obj=dict(
                 cronjob=loader_data.pop("cronjob"),
@@ -155,7 +156,13 @@ class On(BaseModel):
 
     @model_validator(mode="before")
     def __prepare_values(cls, values: DictData) -> DictData:
-        """Extract tz key from value and change name to timezone key."""
+        """Extract tz key from value and change name to timezone key.
+
+        :param values: (DictData) A data that want to pass for create an On
+            model.
+
+        :rtype: DictData
+        """
         if tz := values.pop("tz", None):
             values["timezone"] = tz
         return values
@@ -170,8 +177,8 @@ class On(BaseModel):
         try:
             _ = ZoneInfo(value)
             return value
-        except ZoneInfoNotFoundError as err:
-            raise ValueError(f"Invalid timezone: {value}") from err
+        except ZoneInfoNotFoundError as e:
+            raise ValueError(f"Invalid timezone: {value}") from e
 
     @field_validator(
         "cronjob", mode="before", json_schema_input_type=Union[CronJob, str]
@@ -180,9 +187,13 @@ class On(BaseModel):
         cls, value: str | CronJob, info: ValidationInfo
     ) -> CronJob:
         """Prepare crontab value that able to receive with string type.
-        This step will get options kwargs from extras and pass to the
+        This step will get options kwargs from extras field and pass to the
         CronJob object.
 
+        :param value: (str | CronJobYear) A cronjob value that want to create.
+        :param info: (ValidationInfo) A validation info object that use to get
+            the extra parameters for create cronjob.
+
         :rtype: CronJob
         """
         extras: DictData = info.data.get("extras", {})
@@ -203,12 +214,17 @@ class On(BaseModel):
     def __serialize_cronjob(self, value: CronJob) -> str:
         """Serialize the cronjob field that store with CronJob object.
 
+        :param value: (CronJob) The CronJob field.
+
         :rtype: str
         """
         return str(value)
 
     def generate(self, start: str | datetime) -> CronRunner:
-        """Return Cron runner object.
+        """Return CronRunner object from an initial datetime.
+
+        :param start: (str | datetime) A string or datetime for generate the
+            CronRunner object.
 
         :rtype: CronRunner
         """
@@ -242,16 +258,26 @@ class YearOn(On):
     # NOTE: This is fields of the base schedule.
     cronjob: Annotated[
         CronJobYear,
-        Field(description="Cron job of this schedule"),
+        Field(
+            description=(
+                "A Cronjob object that use for validate and generate datetime.",
+            ),
+        ),
     ]
 
     @field_validator(
-        "cronjob", mode="before", json_schema_input_type=Union[CronJob, str]
+        "cronjob", mode="before", json_schema_input_type=Union[CronJobYear, str]
     )
     def __prepare_cronjob(
         cls, value: str | CronJobYear, info: ValidationInfo
     ) -> CronJobYear:
         """Prepare crontab value that able to receive with string type.
+        This step will get options kwargs from extras field and pass to the
+        CronJobYear object.
+
+        :param value: (str | CronJobYear) A cronjob value that want to create.
+        :param info: (ValidationInfo) A validation info object that use to get
+            the extra parameters for create cronjob.
 
         :rtype: CronJobYear
         """

{ddeutil_workflow-0.0.49 → ddeutil_workflow-0.0.51}/src/ddeutil/workflow/exceptions.py

@@ -39,6 +39,9 @@ class BaseWorkflowException(Exception):
 class UtilException(BaseWorkflowException): ...
 
 
+class ResultException(UtilException): ...
+
+
 class StageException(BaseWorkflowException): ...
 
 
@@ -48,9 +51,6 @@ class JobException(BaseWorkflowException): ...
 class WorkflowException(BaseWorkflowException): ...
 
 
-class WorkflowFailException(WorkflowException): ...
-
-
 class ParamValueException(WorkflowException): ...