ddeutil-workflow 0.0.63__py3-none-any.whl → 0.0.65__py3-none-any.whl

{ddeutil_workflow-0.0.63.dist-info → ddeutil_workflow-0.0.65.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddeutil-workflow
-Version: 0.0.63
+Version: 0.0.65
 Summary: Lightweight workflow orchestration
 Author-email: ddeutils <korawich.anu@gmail.com>
 License: MIT
@@ -23,7 +23,7 @@ Requires-Python: >=3.9.13
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: ddeutil[checksum]>=0.4.8
-Requires-Dist: ddeutil-io[toml,yaml]>=0.2.13
+Requires-Dist: ddeutil-io[toml,yaml]>=0.2.14
 Requires-Dist: pydantic==2.11.4
 Requires-Dist: pydantic-extra-types==2.10.4
 Requires-Dist: python-dotenv==1.1.0
@@ -221,33 +221,36 @@ value (This config can override by extra parameters with `registry_caller` key).
 > engine will auto use the `model_validate` method before run your caller function.
 
 ```python
-from ddeutil.workflow import Result, WorkflowSecret, tag
-from ddeutil.workflow.exceptions import StageException
+from ddeutil.workflow import Result, CallerSecret, tag
+from ddeutil.workflow.errors import StageError
 from pydantic import BaseModel
 
+
 class AwsCredential(BaseModel):
     path: str
     access_client_id: str
-    access_client_secret: WorkflowSecret
+    access_client_secret: CallerSecret
+
 
 class RestAuth(BaseModel):
     type: str
-    keys: WorkflowSecret
+    keys: CallerSecret
+
 
 @tag("requests", alias="get-api-with-oauth-to-s3")
 def get_api_with_oauth_to_s3(
-    method: str,
-    url: str,
-    body: dict[str, str],
-    auth: RestAuth,
-    writing_node: str,
-    aws: AwsCredential,
-    result: Result,
+        method: str,
+        url: str,
+        body: dict[str, str],
+        auth: RestAuth,
+        writing_node: str,
+        aws: AwsCredential,
+        result: Result,
 ) -> dict[str, int]:
     result.trace.info("[CALLER]: Start get data via RestAPI to S3.")
     result.trace.info(f"... {method}: {url}")
     if method != "post":
-       raise StageException(f"RestAPI does not support for {method} action.")
+        raise StageError(f"RestAPI does not support for {method} action.")
     # NOTE: If you want to use secret, you can use `auth.keys.get_secret_value()`.
     return {"records": 1000}
 ```
@@ -265,45 +268,6 @@ result: Result = workflow.execute(
 )
 ```
 
-> [!NOTE]
-> So, this package provide the `Schedule` template for this action, and you can
-> pass the parameters dynamically for changing align with that running time by
-> the `release` prefix.
->
-> ```yaml
-> schedule-run-local-wf:
->
->    # Validate model that use to parsing exists for template file
->    type: Schedule
->    workflows:
->
->       # Map existing workflow that want to deploy with scheduler application.
->       # It allows you to pass release parameter that dynamic change depend on the
->       # current context of this scheduler application releasing that time.
->       - name: run-py-local
->         params:
->           source-extract: "USD-THB"
->           run-date: "${{ release.logical_date }}"
-> ```
->
-> The main method of the `Schedule` model that use to running is `pending`. If you
-> do not pass the `stop` date on this method, it will use config with
-> `WORKFLOW_APP_STOP_BOUNDARY_DELTA` key for generate this stop date.
->
-> ```python
-> from ddeutil.workflow import Schedule
->
-> (
->    Schedule
->    .from_conf("schedule-run-local-wf")
->    .pending(stop=None)
-> )
-> ```
-
-> [!WARNING]
-> The scheduler feature is the expensive feature of this project. You should
-> avoid to use it and find a scheduler tool instead.
-
 ## :cookie: Configuration
 
 The main configuration that use to dynamic changing this workflow engine for your
@@ -321,7 +285,6 @@ it will use default value and do not raise any error to you.
 | **CONF_PATH**                |   Core    | `./conf`                                                                                                                        | The config path that keep all template `.yaml` files.                                                              |
 | **TIMEZONE**                 |   Core    | `Asia/Bangkok`                                                                                                                  | A Timezone string value that will pass to `ZoneInfo` object.                                                       |
 | **STAGE_DEFAULT_ID**         |   Core    | `false`                                                                                                                         | A flag that enable default stage ID that use for catch an execution output.                                        |
-| **STAGE_RAISE_ERROR**        |   Core    | `false`                                                                                                                         | A flag that all stage raise StageException from stage execution.                                                   |
 | **MAX_CRON_PER_WORKFLOW**    |   Core    | `5`                                                                                                                             |                                                                                                                    |
 | **MAX_QUEUE_COMPLETE_HIST**  |   Core    | `16`                                                                                                                            |                                                                                                                    |
 | **GENERATE_ID_SIMPLE_MODE**  |   Core    | `true`                                                                                                                          | A flog that enable generating ID with `md5` algorithm.                                                             |
@@ -333,19 +296,6 @@ it will use default value and do not raise any error to you.
 | **TRACE_ENABLE_WRITE**       |    Log    | `false`                                                                                                                         |                                                                                                                    |
 | **AUDIT_PATH**               |    Log    | `./audits`                                                                                                                      |                                                                                                                    |
 | **AUDIT_ENABLE_WRITE**       |    Log    | `true`                                                                                                                          | A flag that enable logging object saving log to its destination.                                                   |
-| **MAX_PROCESS**              |    App    | `2`                                                                                                                             | The maximum process worker number that run in scheduler app module.                                                |
-| **MAX_SCHEDULE_PER_PROCESS** |    App    | `100`                                                                                                                           | A schedule per process that run parallel.                                                                          |
-| **STOP_BOUNDARY_DELTA**      |    App    | `'{"minutes": 5, "seconds": 20}'`                                                                                               | A time delta value that use to stop scheduler app in json string format.                                           |
-
-**API Application**:
-
-This config part use for the workflow application that build from the FastAPI
-only.
-
-| Environment                |  Component  | Default | Description                                                                        |
-|:---------------------------|:-----------:|---------|------------------------------------------------------------------------------------|
-| **ENABLE_ROUTE_WORKFLOW**  |     API     | `true`  | A flag that enable workflow route to manage execute manually and workflow logging. |
-| **ENABLE_ROUTE_SCHEDULE**  |     API     | `true`  | A flag that enable run scheduler.                                                  |
 
 ## :rocket: Deployment
 

ddeutil_workflow-0.0.65.dist-info/RECORD

@@ -0,0 +1,28 @@
+ddeutil/workflow/__about__.py,sha256=HQ_82TqpLyt4QUTSUNDK94y4NkrRcrX9-AKR-fDwuWU,28
+ddeutil/workflow/__cron.py,sha256=BOKQcreiex0SAigrK1gnLxpvOeF3aca_rQwyz9Kfve4,28751
+ddeutil/workflow/__init__.py,sha256=JfFZlPRDgR2J0rb0SRejt1OSrOrD3GGv9Um14z8MMfs,901
+ddeutil/workflow/__main__.py,sha256=x-sYedl4T8p6054aySk-EQX6vhytvPR0HvaBNYxMzp0,364
+ddeutil/workflow/__types.py,sha256=uNfoRbVmNK5O37UUMVnqcmoghD9oMS1q9fXC0APnjSI,4584
+ddeutil/workflow/conf.py,sha256=VKtnD-Duuf_tPOCUfM6oa86-CrqeCj8kiQbzPLPUXx0,14807
+ddeutil/workflow/errors.py,sha256=evZYwNrvAvY1jpCqqZFkBCdaZ7GN-JbYJMYL6tJmm-0,2980
+ddeutil/workflow/event.py,sha256=S2eJAZZx_V5TuQ0l417hFVCtjWXnfNPZBgSCICzxQ48,11041
+ddeutil/workflow/job.py,sha256=9toh8L9MlQqyy3U3WYierdA03ohf0LOhffaPIOY2IYU,39126
+ddeutil/workflow/logs.py,sha256=iVtyl8i69y7t07tAuWkihc54WlkHCcBy_Ur0WtzJ_lM,31367
+ddeutil/workflow/params.py,sha256=Pco3DyjptC5Jkx53dhLL9xlIQdJvNAZs4FLzMUfXpbQ,12402
+ddeutil/workflow/result.py,sha256=aUMIXw2nYbCDfFZqj9ABr_b7ZLo1GftTaaW8ATh618g,7855
+ddeutil/workflow/reusables.py,sha256=jPrOCbxagqRvRFGXJzIyDa1wKV5AZ4crZyJ10cldQP0,21620
+ddeutil/workflow/stages.py,sha256=rorKBjdyUAxALtelNJrvc5plJp1WCV35NMkii3XMw2A,102094
+ddeutil/workflow/utils.py,sha256=slhBbsBNl0yaSk9EOiCK6UL-o7smgHVsLT7svRqAWXU,10436
+ddeutil/workflow/workflow.py,sha256=YP1st2y3YCUscsuFpjf3fQgYMOnQbBhiY0s6PW1Lpng,27637
+ddeutil/workflow/api/__init__.py,sha256=0UIilYwW29RL6HrCRHACSWvnATJVLSJzXiCMny0bHQk,2627
+ddeutil/workflow/api/logs.py,sha256=NMTnOnsBrDB5129329xF2myLdrb-z9k1MQrmrP7qXJw,1818
+ddeutil/workflow/api/routes/__init__.py,sha256=jC1pM7q4_eo45IyO3hQbbe6RnL9B8ibRq_K6aCMP6Ag,434
+ddeutil/workflow/api/routes/job.py,sha256=32TkNm7QY9gt6fxIqEPjDqPgc8XqDiMPjUb7disSrCw,2143
+ddeutil/workflow/api/routes/logs.py,sha256=QJH8IF102897WLfCJ29-1g15wl29M9Yq6omroZfbahs,5305
+ddeutil/workflow/api/routes/workflows.py,sha256=Gmg3e-K5rfi95pbRtWI_aIr5C089sIde_vefZVvh3U0,4420
+ddeutil_workflow-0.0.65.dist-info/licenses/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
+ddeutil_workflow-0.0.65.dist-info/METADATA,sha256=ujjqpMK3NkLSGMjPJdg30Cs3oGMJB8pBQd-eGWllYX4,16685
+ddeutil_workflow-0.0.65.dist-info/WHEEL,sha256=Nw36Djuh_5VDukK0H78QzOX-_FQEo6V37m3nkm96gtU,91
+ddeutil_workflow-0.0.65.dist-info/entry_points.txt,sha256=qDTpPSauL0ciO6T4iSVt8bJeYrVEkkoEEw_RlGx6Kgk,63
+ddeutil_workflow-0.0.65.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
+ddeutil_workflow-0.0.65.dist-info/RECORD,,

{ddeutil_workflow-0.0.63.dist-info → ddeutil_workflow-0.0.65.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.4.0)
+Generator: setuptools (80.7.1)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

ddeutil/workflow/api/routes/schedules.py

@@ -1,141 +0,0 @@
-# ------------------------------------------------------------------------------
-# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
-# Licensed under the MIT License. See LICENSE in the project root for
-# license information.
-# ------------------------------------------------------------------------------
-from __future__ import annotations
-
-import copy
-from datetime import datetime, timedelta
-
-from fastapi import APIRouter, HTTPException, Request
-from fastapi import status as st
-from fastapi.responses import UJSONResponse
-
-from ...conf import config
-from ...logs import get_logger
-from ...scheduler import Schedule
-
-logger = get_logger("uvicorn.error")
-schedule_route = APIRouter(
-    prefix="/schedules",
-    tags=["schedules"],
-    default_response_class=UJSONResponse,
-)
-
-
-@schedule_route.get(path="/{name}", status_code=st.HTTP_200_OK)
-async def get_schedules(name: str):
-    """Get schedule object."""
-    try:
-        schedule: Schedule = Schedule.from_conf(name=name, extras={})
-    except ValueError:
-        raise HTTPException(
-            status_code=st.HTTP_404_NOT_FOUND,
-            detail=f"Schedule name: {name!r} does not found in /conf path",
-        ) from None
-    return schedule.model_dump(
-        by_alias=True,
-        exclude_none=True,
-        exclude_unset=True,
-        exclude_defaults=True,
-    )
-
-
-@schedule_route.get(path="/deploy/", status_code=st.HTTP_200_OK)
-async def get_deploy_schedulers(request: Request):
-    snapshot = copy.deepcopy(request.state.scheduler)
-    return {"schedule": snapshot}
-
-
-@schedule_route.get(path="/deploy/{name}", status_code=st.HTTP_200_OK)
-async def get_deploy_scheduler(request: Request, name: str):
-    if name in request.state.scheduler:
-        schedule = Schedule.from_conf(name)
-        getter: list[dict[str, dict[str, list[datetime]]]] = []
-        for workflow in schedule.workflows:
-            getter.append(
-                {
-                    workflow.name: {
-                        "queue": copy.deepcopy(
-                            request.state.workflow_queue[workflow.name]
-                        ),
-                        "running": copy.deepcopy(
-                            request.state.workflow_running[workflow.name]
-                        ),
-                    }
-                }
-            )
-        return {
-            "message": f"Getting {name!r} to schedule listener.",
-            "scheduler": getter,
-        }
-    raise HTTPException(
-        status_code=st.HTTP_404_NOT_FOUND,
-        detail=f"Does not found {name!r} in schedule listener",
-    )
-
-
-@schedule_route.post(path="/deploy/{name}", status_code=st.HTTP_202_ACCEPTED)
-async def add_deploy_scheduler(request: Request, name: str):
-    """Adding schedule name to application state store."""
-    if name in request.state.scheduler:
-        raise HTTPException(
-            status_code=st.HTTP_302_FOUND,
-            detail=f"This schedule {name!r} already exists in scheduler list.",
-        )
-
-    request.state.scheduler.append(name)
-
-    start_date: datetime = datetime.now(tz=config.tz)
-    start_date_waiting: datetime = (start_date + timedelta(minutes=1)).replace(
-        second=0, microsecond=0
-    )
-
-    # NOTE: Create a pair of workflow and on from schedule model.
-    try:
-        schedule: Schedule = Schedule.from_conf(name)
-    except ValueError as err:
-        request.state.scheduler.remove(name)
-        logger.exception(err)
-        raise HTTPException(
-            status_code=st.HTTP_404_NOT_FOUND,
-            detail=str(err),
-        ) from None
-
-    request.state.workflow_tasks.extend(
-        schedule.tasks(
-            start_date_waiting,
-            queue=request.state.workflow_queue,
-        ),
-    )
-    return {
-        "message": f"Adding {name!r} to schedule listener.",
-        "start_date": start_date_waiting,
-    }
-
-
-@schedule_route.delete(path="/deploy/{name}", status_code=st.HTTP_202_ACCEPTED)
-async def del_deploy_scheduler(request: Request, name: str):
-    """Delete workflow task on the schedule listener."""
-    if name in request.state.scheduler:
-
-        # NOTE: Remove current schedule name from the state.
-        request.state.scheduler.remove(name)
-
-        schedule: Schedule = Schedule.from_conf(name)
-
-        for task in schedule.tasks(datetime.now(tz=config.tz), queue={}):
-            if task in request.state.workflow_tasks:
-                request.state.workflow_tasks.remove(task)
-
-        for workflow in schedule.workflows:
-            if workflow.alias in request.state.workflow_queue:
-                request.state.workflow_queue.pop(workflow.alias)
-
-        return {"message": f"Deleted schedule {name!r} in listener."}
-
-    raise HTTPException(
-        status_code=st.HTTP_404_NOT_FOUND,
-        detail=f"Does not found schedule {name!r} in listener",
-    )

ddeutil/workflow/api/utils.py

@@ -1,174 +0,0 @@
-# ------------------------------------------------------------------------------
-# Copyright (c) 2023 Priyanshu Panwar. All rights reserved.
-# Licensed under the MIT License.
-# This code refs from: https://github.com/priyanshu-panwar/fastapi-utilities
-# ------------------------------------------------------------------------------
-from __future__ import annotations
-
-import asyncio
-from asyncio import ensure_future
-from datetime import datetime
-from functools import wraps
-
-from starlette.concurrency import run_in_threadpool
-
-from ..__cron import CronJob
-from ..conf import config
-from ..logs import get_logger
-
-logger = get_logger("uvicorn.error")
-
-
-def get_cronjob_delta(cron: str) -> float:
-    """This function returns the time delta between now and the next cron
-    execution time.
-
-    :rtype: float
-    """
-    now: datetime = datetime.now(tz=config.tz)
-    cron = CronJob(cron)
-    return (cron.schedule(now).next - now).total_seconds()
-
-
-def cron_valid(cron: str, raise_error: bool = True) -> bool:
-    """Check this crontab string value is valid with its cron syntax.
-
-    :rtype: bool
-    """
-    try:
-        CronJob(cron)
-        return True
-    except Exception as err:
-        if raise_error:
-            raise ValueError(f"Crontab value does not valid, {cron}") from err
-        return False
-
-
-async def run_func(
-    is_coroutine,
-    func,
-    *args,
-    raise_exceptions: bool = False,
-    **kwargs,
-):
-    """Run function inside the repeat decorator functions."""
-    try:
-        if is_coroutine:
-            await func(*args, **kwargs)
-        else:
-            await run_in_threadpool(func, *args, **kwargs)
-    except Exception as e:
-        logger.exception(e)
-        if raise_exceptions:
-            raise e
-
-
-def repeat_at(
-    *,
-    cron: str,
-    delay: float = 0,
-    raise_exceptions: bool = False,
-    max_repetitions: int = None,
-):
-    """This function returns a decorator that makes a function execute
-    periodically as per the cron expression provided.
-
-    :param cron: (str) A Cron-style string for periodic execution, e.g.
-        '0 0 * * *' every midnight
-    :param delay: (float) A delay seconds value.
-    :param raise_exceptions: (bool) A raise exception flag. Whether to raise
-        exceptions or log them if raise was set be false.
-    :param max_repetitions: int (default None)
-        Maximum number of times to repeat the function. If None, repeat
-        indefinitely.
-    """
-    if max_repetitions and max_repetitions <= 0:
-        raise ValueError(
-            "max_repetitions should more than zero if it want to set"
-        )
-
-    def decorator(func):
-        is_coroutine: bool = asyncio.iscoroutinefunction(func)
-
-        @wraps(func)
-        def wrapper(*_args, **_kwargs):
-            repetitions: int = 0
-            cron_valid(cron)
-
-            async def loop(*args, **kwargs):
-                nonlocal repetitions
-                while max_repetitions is None or repetitions < max_repetitions:
-                    sleep_time = get_cronjob_delta(cron) + delay
-                    await asyncio.sleep(sleep_time)
-                    await run_func(
-                        is_coroutine,
-                        func,
-                        *args,
-                        raise_exceptions=raise_exceptions,
-                        **kwargs,
-                    )
-                    repetitions += 1
-
-            ensure_future(loop(*_args, **_kwargs))
-
-        return wrapper
-
-    return decorator
-
-
-def repeat_every(
-    *,
-    seconds: float,
-    wait_first: bool = False,
-    raise_exceptions: bool = False,
-    max_repetitions: int = None,
-):
-    """This function returns a decorator that schedules a function to execute
-    periodically after every `seconds` seconds.
-
-    :param seconds: float
-        The number of seconds to wait before executing the function again.
-    :param wait_first: bool (default False)
-        Whether to wait `seconds` seconds before executing the function for the
-        first time.
-    :param raise_exceptions: bool (default False)
-        Whether to raise exceptions instead of logging them.
-    :param max_repetitions: int (default None)
-        The maximum number of times to repeat the function. If None, the
-        function will repeat indefinitely.
-    """
-    if max_repetitions and max_repetitions <= 0:
-        raise ValueError(
-            "max_repetitions should more than zero if it want to set"
-        )
-
-    def decorator(func):
-        is_coroutine: bool = asyncio.iscoroutinefunction(func)
-
-        @wraps(func)
-        async def wrapper(*_args, **_kwargs):
-            repetitions = 0
-
-            async def loop(*args, **kwargs):
-                nonlocal repetitions
-
-                if wait_first:
-                    await asyncio.sleep(seconds)
-
-                while max_repetitions is None or repetitions < max_repetitions:
-                    await run_func(
-                        is_coroutine,
-                        func,
-                        *args,
-                        raise_exceptions=raise_exceptions,
-                        **kwargs,
-                    )
-
-                    repetitions += 1
-                    await asyncio.sleep(seconds)
-
-            ensure_future(loop(*_args, **_kwargs))
-
-        return wrapper
-
-    return decorator