ddeutil-workflow 0.0.63__py3-none-any.whl → 0.0.64__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ddeutil/workflow/__about__.py +1 -1
- ddeutil/workflow/__init__.py +0 -7
- ddeutil/workflow/api/__init__.py +5 -84
- ddeutil/workflow/api/routes/__init__.py +0 -1
- ddeutil/workflow/api/routes/job.py +0 -1
- ddeutil/workflow/api/routes/logs.py +0 -2
- ddeutil/workflow/api/routes/workflows.py +0 -3
- ddeutil/workflow/conf.py +6 -34
- ddeutil/workflow/exceptions.py +0 -3
- ddeutil/workflow/reusables.py +39 -6
- ddeutil/workflow/stages.py +30 -28
- ddeutil/workflow/utils.py +4 -2
- {ddeutil_workflow-0.0.63.dist-info → ddeutil_workflow-0.0.64.dist-info}/METADATA +4 -56
- ddeutil_workflow-0.0.64.dist-info/RECORD +28 -0
- ddeutil/workflow/api/routes/schedules.py +0 -141
- ddeutil/workflow/api/utils.py +0 -174
- ddeutil/workflow/scheduler.py +0 -813
- ddeutil_workflow-0.0.63.dist-info/RECORD +0 -31
- {ddeutil_workflow-0.0.63.dist-info → ddeutil_workflow-0.0.64.dist-info}/WHEEL +0 -0
- {ddeutil_workflow-0.0.63.dist-info → ddeutil_workflow-0.0.64.dist-info}/entry_points.txt +0 -0
- {ddeutil_workflow-0.0.63.dist-info → ddeutil_workflow-0.0.64.dist-info}/licenses/LICENSE +0 -0
- {ddeutil_workflow-0.0.63.dist-info → ddeutil_workflow-0.0.64.dist-info}/top_level.txt +0 -0
@@ -1,141 +0,0 @@
|
|
1
|
-
# ------------------------------------------------------------------------------
|
2
|
-
# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
|
3
|
-
# Licensed under the MIT License. See LICENSE in the project root for
|
4
|
-
# license information.
|
5
|
-
# ------------------------------------------------------------------------------
|
6
|
-
from __future__ import annotations
|
7
|
-
|
8
|
-
import copy
|
9
|
-
from datetime import datetime, timedelta
|
10
|
-
|
11
|
-
from fastapi import APIRouter, HTTPException, Request
|
12
|
-
from fastapi import status as st
|
13
|
-
from fastapi.responses import UJSONResponse
|
14
|
-
|
15
|
-
from ...conf import config
|
16
|
-
from ...logs import get_logger
|
17
|
-
from ...scheduler import Schedule
|
18
|
-
|
19
|
-
logger = get_logger("uvicorn.error")
|
20
|
-
schedule_route = APIRouter(
|
21
|
-
prefix="/schedules",
|
22
|
-
tags=["schedules"],
|
23
|
-
default_response_class=UJSONResponse,
|
24
|
-
)
|
25
|
-
|
26
|
-
|
27
|
-
@schedule_route.get(path="/{name}", status_code=st.HTTP_200_OK)
|
28
|
-
async def get_schedules(name: str):
|
29
|
-
"""Get schedule object."""
|
30
|
-
try:
|
31
|
-
schedule: Schedule = Schedule.from_conf(name=name, extras={})
|
32
|
-
except ValueError:
|
33
|
-
raise HTTPException(
|
34
|
-
status_code=st.HTTP_404_NOT_FOUND,
|
35
|
-
detail=f"Schedule name: {name!r} does not found in /conf path",
|
36
|
-
) from None
|
37
|
-
return schedule.model_dump(
|
38
|
-
by_alias=True,
|
39
|
-
exclude_none=True,
|
40
|
-
exclude_unset=True,
|
41
|
-
exclude_defaults=True,
|
42
|
-
)
|
43
|
-
|
44
|
-
|
45
|
-
@schedule_route.get(path="/deploy/", status_code=st.HTTP_200_OK)
|
46
|
-
async def get_deploy_schedulers(request: Request):
|
47
|
-
snapshot = copy.deepcopy(request.state.scheduler)
|
48
|
-
return {"schedule": snapshot}
|
49
|
-
|
50
|
-
|
51
|
-
@schedule_route.get(path="/deploy/{name}", status_code=st.HTTP_200_OK)
|
52
|
-
async def get_deploy_scheduler(request: Request, name: str):
|
53
|
-
if name in request.state.scheduler:
|
54
|
-
schedule = Schedule.from_conf(name)
|
55
|
-
getter: list[dict[str, dict[str, list[datetime]]]] = []
|
56
|
-
for workflow in schedule.workflows:
|
57
|
-
getter.append(
|
58
|
-
{
|
59
|
-
workflow.name: {
|
60
|
-
"queue": copy.deepcopy(
|
61
|
-
request.state.workflow_queue[workflow.name]
|
62
|
-
),
|
63
|
-
"running": copy.deepcopy(
|
64
|
-
request.state.workflow_running[workflow.name]
|
65
|
-
),
|
66
|
-
}
|
67
|
-
}
|
68
|
-
)
|
69
|
-
return {
|
70
|
-
"message": f"Getting {name!r} to schedule listener.",
|
71
|
-
"scheduler": getter,
|
72
|
-
}
|
73
|
-
raise HTTPException(
|
74
|
-
status_code=st.HTTP_404_NOT_FOUND,
|
75
|
-
detail=f"Does not found {name!r} in schedule listener",
|
76
|
-
)
|
77
|
-
|
78
|
-
|
79
|
-
@schedule_route.post(path="/deploy/{name}", status_code=st.HTTP_202_ACCEPTED)
|
80
|
-
async def add_deploy_scheduler(request: Request, name: str):
|
81
|
-
"""Adding schedule name to application state store."""
|
82
|
-
if name in request.state.scheduler:
|
83
|
-
raise HTTPException(
|
84
|
-
status_code=st.HTTP_302_FOUND,
|
85
|
-
detail=f"This schedule {name!r} already exists in scheduler list.",
|
86
|
-
)
|
87
|
-
|
88
|
-
request.state.scheduler.append(name)
|
89
|
-
|
90
|
-
start_date: datetime = datetime.now(tz=config.tz)
|
91
|
-
start_date_waiting: datetime = (start_date + timedelta(minutes=1)).replace(
|
92
|
-
second=0, microsecond=0
|
93
|
-
)
|
94
|
-
|
95
|
-
# NOTE: Create a pair of workflow and on from schedule model.
|
96
|
-
try:
|
97
|
-
schedule: Schedule = Schedule.from_conf(name)
|
98
|
-
except ValueError as err:
|
99
|
-
request.state.scheduler.remove(name)
|
100
|
-
logger.exception(err)
|
101
|
-
raise HTTPException(
|
102
|
-
status_code=st.HTTP_404_NOT_FOUND,
|
103
|
-
detail=str(err),
|
104
|
-
) from None
|
105
|
-
|
106
|
-
request.state.workflow_tasks.extend(
|
107
|
-
schedule.tasks(
|
108
|
-
start_date_waiting,
|
109
|
-
queue=request.state.workflow_queue,
|
110
|
-
),
|
111
|
-
)
|
112
|
-
return {
|
113
|
-
"message": f"Adding {name!r} to schedule listener.",
|
114
|
-
"start_date": start_date_waiting,
|
115
|
-
}
|
116
|
-
|
117
|
-
|
118
|
-
@schedule_route.delete(path="/deploy/{name}", status_code=st.HTTP_202_ACCEPTED)
|
119
|
-
async def del_deploy_scheduler(request: Request, name: str):
|
120
|
-
"""Delete workflow task on the schedule listener."""
|
121
|
-
if name in request.state.scheduler:
|
122
|
-
|
123
|
-
# NOTE: Remove current schedule name from the state.
|
124
|
-
request.state.scheduler.remove(name)
|
125
|
-
|
126
|
-
schedule: Schedule = Schedule.from_conf(name)
|
127
|
-
|
128
|
-
for task in schedule.tasks(datetime.now(tz=config.tz), queue={}):
|
129
|
-
if task in request.state.workflow_tasks:
|
130
|
-
request.state.workflow_tasks.remove(task)
|
131
|
-
|
132
|
-
for workflow in schedule.workflows:
|
133
|
-
if workflow.alias in request.state.workflow_queue:
|
134
|
-
request.state.workflow_queue.pop(workflow.alias)
|
135
|
-
|
136
|
-
return {"message": f"Deleted schedule {name!r} in listener."}
|
137
|
-
|
138
|
-
raise HTTPException(
|
139
|
-
status_code=st.HTTP_404_NOT_FOUND,
|
140
|
-
detail=f"Does not found schedule {name!r} in listener",
|
141
|
-
)
|
ddeutil/workflow/api/utils.py
DELETED
@@ -1,174 +0,0 @@
|
|
1
|
-
# ------------------------------------------------------------------------------
|
2
|
-
# Copyright (c) 2023 Priyanshu Panwar. All rights reserved.
|
3
|
-
# Licensed under the MIT License.
|
4
|
-
# This code refs from: https://github.com/priyanshu-panwar/fastapi-utilities
|
5
|
-
# ------------------------------------------------------------------------------
|
6
|
-
from __future__ import annotations
|
7
|
-
|
8
|
-
import asyncio
|
9
|
-
from asyncio import ensure_future
|
10
|
-
from datetime import datetime
|
11
|
-
from functools import wraps
|
12
|
-
|
13
|
-
from starlette.concurrency import run_in_threadpool
|
14
|
-
|
15
|
-
from ..__cron import CronJob
|
16
|
-
from ..conf import config
|
17
|
-
from ..logs import get_logger
|
18
|
-
|
19
|
-
logger = get_logger("uvicorn.error")
|
20
|
-
|
21
|
-
|
22
|
-
def get_cronjob_delta(cron: str) -> float:
|
23
|
-
"""This function returns the time delta between now and the next cron
|
24
|
-
execution time.
|
25
|
-
|
26
|
-
:rtype: float
|
27
|
-
"""
|
28
|
-
now: datetime = datetime.now(tz=config.tz)
|
29
|
-
cron = CronJob(cron)
|
30
|
-
return (cron.schedule(now).next - now).total_seconds()
|
31
|
-
|
32
|
-
|
33
|
-
def cron_valid(cron: str, raise_error: bool = True) -> bool:
|
34
|
-
"""Check this crontab string value is valid with its cron syntax.
|
35
|
-
|
36
|
-
:rtype: bool
|
37
|
-
"""
|
38
|
-
try:
|
39
|
-
CronJob(cron)
|
40
|
-
return True
|
41
|
-
except Exception as err:
|
42
|
-
if raise_error:
|
43
|
-
raise ValueError(f"Crontab value does not valid, {cron}") from err
|
44
|
-
return False
|
45
|
-
|
46
|
-
|
47
|
-
async def run_func(
|
48
|
-
is_coroutine,
|
49
|
-
func,
|
50
|
-
*args,
|
51
|
-
raise_exceptions: bool = False,
|
52
|
-
**kwargs,
|
53
|
-
):
|
54
|
-
"""Run function inside the repeat decorator functions."""
|
55
|
-
try:
|
56
|
-
if is_coroutine:
|
57
|
-
await func(*args, **kwargs)
|
58
|
-
else:
|
59
|
-
await run_in_threadpool(func, *args, **kwargs)
|
60
|
-
except Exception as e:
|
61
|
-
logger.exception(e)
|
62
|
-
if raise_exceptions:
|
63
|
-
raise e
|
64
|
-
|
65
|
-
|
66
|
-
def repeat_at(
|
67
|
-
*,
|
68
|
-
cron: str,
|
69
|
-
delay: float = 0,
|
70
|
-
raise_exceptions: bool = False,
|
71
|
-
max_repetitions: int = None,
|
72
|
-
):
|
73
|
-
"""This function returns a decorator that makes a function execute
|
74
|
-
periodically as per the cron expression provided.
|
75
|
-
|
76
|
-
:param cron: (str) A Cron-style string for periodic execution, e.g.
|
77
|
-
'0 0 * * *' every midnight
|
78
|
-
:param delay: (float) A delay seconds value.
|
79
|
-
:param raise_exceptions: (bool) A raise exception flag. Whether to raise
|
80
|
-
exceptions or log them if raise was set be false.
|
81
|
-
:param max_repetitions: int (default None)
|
82
|
-
Maximum number of times to repeat the function. If None, repeat
|
83
|
-
indefinitely.
|
84
|
-
"""
|
85
|
-
if max_repetitions and max_repetitions <= 0:
|
86
|
-
raise ValueError(
|
87
|
-
"max_repetitions should more than zero if it want to set"
|
88
|
-
)
|
89
|
-
|
90
|
-
def decorator(func):
|
91
|
-
is_coroutine: bool = asyncio.iscoroutinefunction(func)
|
92
|
-
|
93
|
-
@wraps(func)
|
94
|
-
def wrapper(*_args, **_kwargs):
|
95
|
-
repetitions: int = 0
|
96
|
-
cron_valid(cron)
|
97
|
-
|
98
|
-
async def loop(*args, **kwargs):
|
99
|
-
nonlocal repetitions
|
100
|
-
while max_repetitions is None or repetitions < max_repetitions:
|
101
|
-
sleep_time = get_cronjob_delta(cron) + delay
|
102
|
-
await asyncio.sleep(sleep_time)
|
103
|
-
await run_func(
|
104
|
-
is_coroutine,
|
105
|
-
func,
|
106
|
-
*args,
|
107
|
-
raise_exceptions=raise_exceptions,
|
108
|
-
**kwargs,
|
109
|
-
)
|
110
|
-
repetitions += 1
|
111
|
-
|
112
|
-
ensure_future(loop(*_args, **_kwargs))
|
113
|
-
|
114
|
-
return wrapper
|
115
|
-
|
116
|
-
return decorator
|
117
|
-
|
118
|
-
|
119
|
-
def repeat_every(
|
120
|
-
*,
|
121
|
-
seconds: float,
|
122
|
-
wait_first: bool = False,
|
123
|
-
raise_exceptions: bool = False,
|
124
|
-
max_repetitions: int = None,
|
125
|
-
):
|
126
|
-
"""This function returns a decorator that schedules a function to execute
|
127
|
-
periodically after every `seconds` seconds.
|
128
|
-
|
129
|
-
:param seconds: float
|
130
|
-
The number of seconds to wait before executing the function again.
|
131
|
-
:param wait_first: bool (default False)
|
132
|
-
Whether to wait `seconds` seconds before executing the function for the
|
133
|
-
first time.
|
134
|
-
:param raise_exceptions: bool (default False)
|
135
|
-
Whether to raise exceptions instead of logging them.
|
136
|
-
:param max_repetitions: int (default None)
|
137
|
-
The maximum number of times to repeat the function. If None, the
|
138
|
-
function will repeat indefinitely.
|
139
|
-
"""
|
140
|
-
if max_repetitions and max_repetitions <= 0:
|
141
|
-
raise ValueError(
|
142
|
-
"max_repetitions should more than zero if it want to set"
|
143
|
-
)
|
144
|
-
|
145
|
-
def decorator(func):
|
146
|
-
is_coroutine: bool = asyncio.iscoroutinefunction(func)
|
147
|
-
|
148
|
-
@wraps(func)
|
149
|
-
async def wrapper(*_args, **_kwargs):
|
150
|
-
repetitions = 0
|
151
|
-
|
152
|
-
async def loop(*args, **kwargs):
|
153
|
-
nonlocal repetitions
|
154
|
-
|
155
|
-
if wait_first:
|
156
|
-
await asyncio.sleep(seconds)
|
157
|
-
|
158
|
-
while max_repetitions is None or repetitions < max_repetitions:
|
159
|
-
await run_func(
|
160
|
-
is_coroutine,
|
161
|
-
func,
|
162
|
-
*args,
|
163
|
-
raise_exceptions=raise_exceptions,
|
164
|
-
**kwargs,
|
165
|
-
)
|
166
|
-
|
167
|
-
repetitions += 1
|
168
|
-
await asyncio.sleep(seconds)
|
169
|
-
|
170
|
-
ensure_future(loop(*_args, **_kwargs))
|
171
|
-
|
172
|
-
return wrapper
|
173
|
-
|
174
|
-
return decorator
|