zrb 0.1.1__py3-none-any.whl → 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- zrb/__main__.py +5 -1
- zrb/action/runner.py +15 -8
- zrb/builtin/helper/reccuring_action.py +27 -2
- zrb/builtin/schedule.py +4 -3
- zrb/builtin/watch_changes.py +13 -2
- zrb/config/config.py +1 -0
- zrb/helper/cli.py +26 -11
- zrb/helper/default_env.py +23 -13
- zrb/helper/file/copy_tree.py +4 -1
- zrb/helper/file/match.py +20 -0
- zrb/helper/loader/load_module.py +13 -5
- zrb/task/any_task.py +114 -2
- zrb/task/base_task/base_task.py +39 -3
- zrb/task/base_task/component/base_task_model.py +28 -5
- zrb/task/base_task/component/common_task_model.py +30 -15
- zrb/task/base_task/component/renderer.py +6 -0
- zrb/task/cmd_task.py +17 -24
- zrb/task/docker_compose_task.py +0 -1
- zrb/task/path_checker.py +25 -4
- zrb/task/path_watcher.py +48 -7
- zrb/task/recurring_task.py +60 -9
- zrb/task/time_watcher.py +7 -0
- zrb/task_group/group.py +3 -3
- {zrb-0.1.1.dist-info → zrb-0.2.0.dist-info}/METADATA +5 -5
- {zrb-0.1.1.dist-info → zrb-0.2.0.dist-info}/RECORD +28 -27
- {zrb-0.1.1.dist-info → zrb-0.2.0.dist-info}/LICENSE +0 -0
- {zrb-0.1.1.dist-info → zrb-0.2.0.dist-info}/WHEEL +0 -0
- {zrb-0.1.1.dist-info → zrb-0.2.0.dist-info}/entry_points.txt +0 -0
@@ -2,7 +2,7 @@ from zrb.helper.typing import (
|
|
2
2
|
Any, Callable, Iterable, List, Mapping, Optional, Union
|
3
3
|
)
|
4
4
|
from zrb.helper.typecheck import typechecked
|
5
|
-
from zrb.config.config import show_time
|
5
|
+
from zrb.config.config import show_time, logging_level
|
6
6
|
from zrb.task.any_task import AnyTask
|
7
7
|
from zrb.task.any_task_event_handler import (
|
8
8
|
OnTriggered, OnWaiting, OnSkipped, OnStarted, OnReady, OnRetry, OnFailed
|
@@ -19,8 +19,10 @@ from zrb.task.base_task.component.trackers import TimeTracker
|
|
19
19
|
from zrb.config.config import env_prefix
|
20
20
|
from zrb.helper.string.modification import double_quote
|
21
21
|
from zrb.helper.string.conversion import to_variable_name
|
22
|
+
from functools import lru_cache
|
22
23
|
|
23
24
|
import datetime
|
25
|
+
import logging
|
24
26
|
import os
|
25
27
|
import sys
|
26
28
|
|
@@ -90,12 +92,20 @@ class BaseTaskModel(CommonTaskModel, PidModel, TimeTracker):
|
|
90
92
|
self.__kwargs: Mapping[str, Any] = {}
|
91
93
|
|
92
94
|
def _set_args(self, args: Iterable[Any]):
|
95
|
+
'''
|
96
|
+
Set args that will be shown at the end of the execution
|
97
|
+
'''
|
93
98
|
self.__args = list(args)
|
94
99
|
|
95
100
|
def _set_kwargs(self, kwargs: Mapping[str, Any]):
|
101
|
+
'''
|
102
|
+
Set kwargs that will be shown at the end of the execution
|
103
|
+
'''
|
96
104
|
self.__kwargs = kwargs
|
97
105
|
|
98
106
|
def log_debug(self, message: Any):
|
107
|
+
if logging_level > logging.DEBUG:
|
108
|
+
return
|
99
109
|
prefix = self.__get_log_prefix()
|
100
110
|
colored_message = colored(
|
101
111
|
f'{prefix} • {message}', attrs=['dark']
|
@@ -103,6 +113,8 @@ class BaseTaskModel(CommonTaskModel, PidModel, TimeTracker):
|
|
103
113
|
logger.debug(colored_message)
|
104
114
|
|
105
115
|
def log_warn(self, message: Any):
|
116
|
+
if logging_level > logging.WARNING:
|
117
|
+
return
|
106
118
|
prefix = self.__get_log_prefix()
|
107
119
|
colored_message = colored(
|
108
120
|
f'{prefix} • {message}', attrs=['dark']
|
@@ -110,6 +122,8 @@ class BaseTaskModel(CommonTaskModel, PidModel, TimeTracker):
|
|
110
122
|
logger.warning(colored_message)
|
111
123
|
|
112
124
|
def log_info(self, message: Any):
|
125
|
+
if logging_level > logging.INFO:
|
126
|
+
return
|
113
127
|
prefix = self.__get_log_prefix()
|
114
128
|
colored_message = colored(
|
115
129
|
f'{prefix} • {message}', attrs=['dark']
|
@@ -117,6 +131,8 @@ class BaseTaskModel(CommonTaskModel, PidModel, TimeTracker):
|
|
117
131
|
logger.info(colored_message)
|
118
132
|
|
119
133
|
def log_error(self, message: Any):
|
134
|
+
if logging_level > logging.ERROR:
|
135
|
+
return
|
120
136
|
prefix = self.__get_log_prefix()
|
121
137
|
colored_message = colored(
|
122
138
|
f'{prefix} • {message}', color='red', attrs=['bold']
|
@@ -124,6 +140,8 @@ class BaseTaskModel(CommonTaskModel, PidModel, TimeTracker):
|
|
124
140
|
logger.error(colored_message, exc_info=True)
|
125
141
|
|
126
142
|
def log_critical(self, message: Any):
|
143
|
+
if logging_level > logging.CRITICAL:
|
144
|
+
return
|
127
145
|
prefix = self.__get_log_prefix()
|
128
146
|
colored_message = colored(
|
129
147
|
f'{prefix} • {message}', color='red', attrs=['bold']
|
@@ -203,13 +221,15 @@ class BaseTaskModel(CommonTaskModel, PidModel, TimeTracker):
|
|
203
221
|
def __get_print_prefix(self) -> str:
|
204
222
|
common_prefix = self.__get_common_prefix(show_time=show_time)
|
205
223
|
icon = self.get_icon()
|
206
|
-
|
224
|
+
length = LOG_NAME_LENGTH - len(icon)
|
225
|
+
rjust_cli_name = self.__get_rjust_full_cli_name(length)
|
207
226
|
return f'{common_prefix} {icon} {rjust_cli_name}'
|
208
227
|
|
209
228
|
def __get_log_prefix(self) -> str:
|
210
229
|
common_prefix = self.__get_common_prefix(show_time=False)
|
211
230
|
icon = self.get_icon()
|
212
|
-
|
231
|
+
length = LOG_NAME_LENGTH - len(icon)
|
232
|
+
filled_name = self.__get_rjust_full_cli_name(length)
|
213
233
|
return f'{common_prefix} {icon} {filled_name}'
|
214
234
|
|
215
235
|
def __get_common_prefix(self, show_time: bool) -> str:
|
@@ -221,18 +241,21 @@ class BaseTaskModel(CommonTaskModel, PidModel, TimeTracker):
|
|
221
241
|
return f'◷ {now} ❁ {pid} → {attempt}/{max_attempt}'
|
222
242
|
return f'❁ {pid} → {attempt}/{max_attempt}'
|
223
243
|
|
224
|
-
|
244
|
+
@lru_cache
|
245
|
+
def __get_rjust_full_cli_name(self, length: int) -> str:
|
225
246
|
if self.__rjust_full_cli_name is not None:
|
226
247
|
return self.__rjust_full_cli_name
|
227
248
|
complete_name = self._get_full_cli_name()
|
228
|
-
self.__rjust_full_cli_name = complete_name.rjust(
|
249
|
+
self.__rjust_full_cli_name = complete_name.rjust(length, ' ')
|
229
250
|
return self.__rjust_full_cli_name
|
230
251
|
|
252
|
+
@lru_cache
|
231
253
|
def __get_executable_name(self) -> str:
|
232
254
|
if len(sys.argv) > 0 and sys.argv[0] != '':
|
233
255
|
return os.path.basename(sys.argv[0])
|
234
256
|
return 'zrb'
|
235
257
|
|
258
|
+
@lru_cache
|
236
259
|
def _get_full_cli_name(self) -> str:
|
237
260
|
if self.__complete_name is not None:
|
238
261
|
return self.__complete_name
|
@@ -60,7 +60,7 @@ class CommonTaskModel():
|
|
60
60
|
self._retry = retry
|
61
61
|
self._retry_interval = retry_interval
|
62
62
|
self._upstreams = upstreams
|
63
|
-
self._checkers = checkers
|
63
|
+
self._checkers = [checker.copy() for checker in checkers]
|
64
64
|
self._checking_interval = checking_interval
|
65
65
|
self._run_function: Optional[Callable[..., Any]] = run
|
66
66
|
self._on_triggered = on_triggered
|
@@ -77,10 +77,10 @@ class CommonTaskModel():
|
|
77
77
|
self.__allow_add_env_files = True
|
78
78
|
self.__allow_add_inputs = True
|
79
79
|
self.__allow_add_upstreams: bool = True
|
80
|
+
self.__allow_add_checkers: bool = True
|
80
81
|
self.__has_already_inject_env_files: bool = False
|
81
82
|
self.__has_already_inject_envs: bool = False
|
82
83
|
self.__has_already_inject_inputs: bool = False
|
83
|
-
self.__has_already_inject_checkers: bool = False
|
84
84
|
self.__has_already_inject_upstreams: bool = False
|
85
85
|
self.__all_inputs: Optional[List[AnyInput]] = None
|
86
86
|
|
@@ -104,12 +104,15 @@ class CommonTaskModel():
|
|
104
104
|
return self.__execution_id
|
105
105
|
|
106
106
|
def set_name(self, new_name: str):
|
107
|
-
if self._description == self.
|
107
|
+
if self._description == self.get_name():
|
108
108
|
self._description = new_name
|
109
109
|
self._name = new_name
|
110
110
|
|
111
|
+
def get_name(self) -> str:
|
112
|
+
return self._name
|
113
|
+
|
111
114
|
def get_cli_name(self) -> str:
|
112
|
-
return to_cli_name(self.
|
115
|
+
return to_cli_name(self.get_name())
|
113
116
|
|
114
117
|
def set_description(self, new_description: str):
|
115
118
|
self._description = new_description
|
@@ -139,12 +142,12 @@ class CommonTaskModel():
|
|
139
142
|
|
140
143
|
def insert_input(self, *inputs: AnyInput):
|
141
144
|
if not self.__allow_add_inputs:
|
142
|
-
raise Exception(f'Cannot insert inputs for `{self.
|
145
|
+
raise Exception(f'Cannot insert inputs for `{self.get_name()}`')
|
143
146
|
self._inputs = list(inputs) + list(self._inputs)
|
144
147
|
|
145
148
|
def add_input(self, *inputs: AnyInput):
|
146
149
|
if not self.__allow_add_inputs:
|
147
|
-
raise Exception(f'Cannot add inputs for `{self.
|
150
|
+
raise Exception(f'Cannot add inputs for `{self.get_name()}`')
|
148
151
|
self._inputs = list(self._inputs) + list(inputs)
|
149
152
|
|
150
153
|
def inject_inputs(self):
|
@@ -194,12 +197,12 @@ class CommonTaskModel():
|
|
194
197
|
|
195
198
|
def insert_env(self, *envs: Env):
|
196
199
|
if not self.__allow_add_envs:
|
197
|
-
raise Exception(f'Cannot insert envs to `{self.
|
200
|
+
raise Exception(f'Cannot insert envs to `{self.get_name()}`')
|
198
201
|
self._envs = list(envs) + list(self._envs)
|
199
202
|
|
200
203
|
def add_env(self, *envs: Env):
|
201
204
|
if not self.__allow_add_envs:
|
202
|
-
raise Exception(f'Cannot add envs to `{self.
|
205
|
+
raise Exception(f'Cannot add envs to `{self.get_name()}`')
|
203
206
|
self._envs = list(self._envs) + list(envs)
|
204
207
|
|
205
208
|
def inject_envs(self):
|
@@ -230,12 +233,12 @@ class CommonTaskModel():
|
|
230
233
|
|
231
234
|
def insert_env_file(self, *env_files: EnvFile):
|
232
235
|
if not self.__allow_add_env_files:
|
233
|
-
raise Exception(f'Cannot insert env_files to `{self.
|
236
|
+
raise Exception(f'Cannot insert env_files to `{self.get_name()}`')
|
234
237
|
self._env_files = list(env_files) + list(self._env_files)
|
235
238
|
|
236
239
|
def add_env_file(self, *env_files: EnvFile):
|
237
240
|
if not self.__allow_add_env_files:
|
238
|
-
raise Exception(f'Cannot add env_files to `{self.
|
241
|
+
raise Exception(f'Cannot add env_files to `{self.get_name()}`')
|
239
242
|
self._env_files = list(self._env_files) + list(env_files)
|
240
243
|
|
241
244
|
def inject_env_files(self):
|
@@ -243,12 +246,12 @@ class CommonTaskModel():
|
|
243
246
|
|
244
247
|
def insert_upstream(self, *upstreams: AnyTask):
|
245
248
|
if not self.__allow_add_upstreams:
|
246
|
-
raise Exception(f'Cannot insert upstreams to `{self.
|
249
|
+
raise Exception(f'Cannot insert upstreams to `{self.get_name()}`')
|
247
250
|
self._upstreams = list(upstreams) + list(self._upstreams)
|
248
251
|
|
249
252
|
def add_upstream(self, *upstreams: AnyTask):
|
250
253
|
if not self.__allow_add_upstreams:
|
251
|
-
raise Exception(f'Cannot add upstreams to `{self.
|
254
|
+
raise Exception(f'Cannot add upstreams to `{self.get_name()}`')
|
252
255
|
self._upstreams = list(self._upstreams) + list(upstreams)
|
253
256
|
|
254
257
|
def inject_upstreams(self):
|
@@ -272,11 +275,23 @@ class CommonTaskModel():
|
|
272
275
|
self.__has_already_inject_env_files = True
|
273
276
|
return self._env_files
|
274
277
|
|
278
|
+
def insert_checker(self, *checkers: AnyTask):
|
279
|
+
if not self.__allow_add_checkers:
|
280
|
+
raise Exception(f'Cannot insert checkers to `{self.get_name()}`')
|
281
|
+
additional_checkers = [checker.copy() for checker in checkers]
|
282
|
+
self._checkers = additional_checkers + self._checkers
|
283
|
+
|
284
|
+
def add_checker(self, *checkers: AnyTask):
|
285
|
+
if not self.__allow_add_checkers:
|
286
|
+
raise Exception(f'Cannot add checkers to `{self.get_name()}`')
|
287
|
+
additional_checkers = [checker.copy() for checker in checkers]
|
288
|
+
self._checkers = self._checkers + additional_checkers
|
289
|
+
|
275
290
|
def inject_checkers(self):
|
276
291
|
pass
|
277
292
|
|
278
293
|
def _get_checkers(self) -> List[AnyTask]:
|
279
|
-
if not self.
|
294
|
+
if not self.__allow_add_checkers:
|
280
295
|
self.inject_checkers()
|
281
|
-
self.
|
282
|
-
return
|
296
|
+
self.__allow_add_checkers = True
|
297
|
+
return self._checkers
|
@@ -1,3 +1,4 @@
|
|
1
|
+
from zrb.task.any_task import AnyTask
|
1
2
|
from zrb.helper.typing import Any, JinjaTemplate, Mapping, Optional, Union
|
2
3
|
from zrb.helper.typecheck import typechecked
|
3
4
|
from zrb.helper.string.conversion import to_boolean
|
@@ -24,10 +25,14 @@ class Renderer():
|
|
24
25
|
|
25
26
|
def __init__(self):
|
26
27
|
self.__input_map: Mapping[str, Any] = {}
|
28
|
+
self.__task: Optional[AnyTask] = None
|
27
29
|
self.__env_map: Mapping[str, str] = {}
|
28
30
|
self.__render_data: Optional[Mapping[str, Any]] = None
|
29
31
|
self.__rendered_str: Mapping[str, str] = {}
|
30
32
|
|
33
|
+
def _set_task(self, task: AnyTask):
|
34
|
+
self.__task = task
|
35
|
+
|
31
36
|
def get_input_map(self) -> Mapping[str, Any]:
|
32
37
|
# This return reference to input map, so input map can be updated
|
33
38
|
return self.__input_map
|
@@ -126,6 +131,7 @@ class Renderer():
|
|
126
131
|
render_data.update({
|
127
132
|
'env': self.__env_map,
|
128
133
|
'input': self.__input_map,
|
134
|
+
'task': self.__task,
|
129
135
|
})
|
130
136
|
self.__render_data = render_data
|
131
137
|
return render_data
|
zrb/task/cmd_task.py
CHANGED
@@ -48,6 +48,9 @@ class CmdResult():
|
|
48
48
|
self.output = output
|
49
49
|
self.error = error
|
50
50
|
|
51
|
+
def __str__(self) -> str:
|
52
|
+
return self.output
|
53
|
+
|
51
54
|
|
52
55
|
class CmdGlobalState():
|
53
56
|
def __init__(self):
|
@@ -61,30 +64,18 @@ class CmdTask(BaseTask):
|
|
61
64
|
Command Task.
|
62
65
|
You can use this task to run shell command.
|
63
66
|
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
name='
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
runner.register(hello)
|
77
|
-
|
78
|
-
# run a long running process
|
79
|
-
run_server = CmdTask(
|
80
|
-
name='run',
|
81
|
-
inputs=[StrInput(name='dir', default='.')],
|
82
|
-
envs=[Env(name='PORT', os_name='WEB_PORT', default='3000')],
|
83
|
-
cmd='python -m http.server $PORT --directory {{input.dir}}',
|
84
|
-
checkers=[HTTPChecker(port='{{env.PORT}}')]
|
85
|
-
)
|
86
|
-
runner.register(run_server)
|
87
|
-
```
|
67
|
+
Examples:
|
68
|
+
>>> from zrb import runner, CmdTask, StrInput, Env
|
69
|
+
>>> hello = CmdTask(
|
70
|
+
>>> name='hello',
|
71
|
+
>>> inputs=[StrInput(name='name', default='World')],
|
72
|
+
>>> envs=[Env(name='HOME_DIR', os_name='HOME')],
|
73
|
+
>>> cmd=[
|
74
|
+
>>> 'echo Hello {{ input.name }}',
|
75
|
+
>>> 'echo Home directory is: $HOME_DIR',
|
76
|
+
>>> ]
|
77
|
+
>>> )
|
78
|
+
>>> runner.register(hello)
|
88
79
|
'''
|
89
80
|
|
90
81
|
_pids: List[int] = []
|
@@ -246,6 +237,8 @@ class CmdTask(BaseTask):
|
|
246
237
|
raise Exception(
|
247
238
|
f'Process {self._name} exited ({return_code}): {error}'
|
248
239
|
)
|
240
|
+
self.set_task_xcom(key='output', value=output)
|
241
|
+
self.set_task_xcom(key='error', value=error)
|
249
242
|
return CmdResult(output, error)
|
250
243
|
|
251
244
|
def _should_attempt(self) -> bool:
|
zrb/task/docker_compose_task.py
CHANGED
@@ -293,7 +293,6 @@ class DockerComposeTask(CmdTask):
|
|
293
293
|
]:
|
294
294
|
if os.path.exists(os.path.join(self._cwd, _compose_file)):
|
295
295
|
return os.path.join(self._cwd, _compose_file)
|
296
|
-
return
|
297
296
|
raise Exception(f'Cannot find compose file on {self._cwd}')
|
298
297
|
if os.path.isabs(compose_file) and os.path.exists(compose_file):
|
299
298
|
return compose_file
|
zrb/task/path_checker.py
CHANGED
@@ -1,5 +1,8 @@
|
|
1
|
-
from zrb.helper.typing import
|
1
|
+
from zrb.helper.typing import (
|
2
|
+
Any, Callable, Iterable, List, Optional, Union, TypeVar
|
3
|
+
)
|
2
4
|
from zrb.helper.typecheck import typechecked
|
5
|
+
from zrb.helper.file.match import get_file_names
|
3
6
|
from zrb.task.checker import Checker
|
4
7
|
from zrb.task.any_task import AnyTask
|
5
8
|
from zrb.task.any_task_event_handler import (
|
@@ -10,8 +13,6 @@ from zrb.task_env.env_file import EnvFile
|
|
10
13
|
from zrb.task_group.group import Group
|
11
14
|
from zrb.task_input.any_input import AnyInput
|
12
15
|
|
13
|
-
import glob
|
14
|
-
|
15
16
|
TPathChecker = TypeVar('TPathChecker', bound='PathChecker')
|
16
17
|
|
17
18
|
|
@@ -37,6 +38,7 @@ class PathChecker(Checker):
|
|
37
38
|
on_retry: Optional[OnRetry] = None,
|
38
39
|
on_failed: Optional[OnFailed] = None,
|
39
40
|
path: str = '',
|
41
|
+
ignored_path: Union[str, Iterable[str]] = [],
|
40
42
|
checking_interval: Union[int, float] = 0.1,
|
41
43
|
progress_interval: Union[int, float] = 5,
|
42
44
|
expected_result: bool = True,
|
@@ -66,7 +68,9 @@ class PathChecker(Checker):
|
|
66
68
|
should_execute=should_execute,
|
67
69
|
)
|
68
70
|
self._path = path
|
71
|
+
self._ignored_path = ignored_path
|
69
72
|
self._rendered_path: str = ''
|
73
|
+
self._rendered_ignored_paths: List[str] = []
|
70
74
|
|
71
75
|
def copy(self) -> TPathChecker:
|
72
76
|
return super().copy()
|
@@ -84,12 +88,29 @@ class PathChecker(Checker):
|
|
84
88
|
|
85
89
|
async def run(self, *args: Any, **kwargs: Any) -> bool:
|
86
90
|
self._rendered_path = self.render_str(self._path)
|
91
|
+
self._rendered_ignored_paths = [
|
92
|
+
ignored_path
|
93
|
+
for ignored_path in self._get_rendered_ignored_paths()
|
94
|
+
if ignored_path != ''
|
95
|
+
]
|
87
96
|
return await super().run(*args, **kwargs)
|
88
97
|
|
98
|
+
def _get_rendered_ignored_paths(self) -> List[str]:
|
99
|
+
if isinstance(self._ignored_path, str):
|
100
|
+
return [self.render_str(self._ignored_path)]
|
101
|
+
return [
|
102
|
+
self.render_str(ignored_path)
|
103
|
+
for ignored_path in self._ignored_path
|
104
|
+
]
|
105
|
+
|
89
106
|
async def inspect(self, *args: Any, **kwargs: Any) -> bool:
|
90
107
|
label = f'Checking {self._rendered_path}'
|
91
108
|
try:
|
92
|
-
|
109
|
+
matches = get_file_names(
|
110
|
+
glob_path=self._rendered_path,
|
111
|
+
glob_ignored_paths=self._rendered_ignored_paths
|
112
|
+
)
|
113
|
+
if len(matches) > 0:
|
93
114
|
self.print_out(f'{label} (Exist)')
|
94
115
|
return True
|
95
116
|
self.show_progress(f'{label} (Not Exist)')
|
zrb/task/path_watcher.py
CHANGED
@@ -1,7 +1,8 @@
|
|
1
1
|
from zrb.helper.typing import (
|
2
|
-
Any, Callable, Iterable, Mapping, Optional, Union, TypeVar
|
2
|
+
Any, Callable, Iterable, List, Mapping, Optional, Union, TypeVar
|
3
3
|
)
|
4
4
|
from zrb.helper.typecheck import typechecked
|
5
|
+
from zrb.helper.file.match import get_file_names
|
5
6
|
from zrb.task.checker import Checker
|
6
7
|
from zrb.task.any_task import AnyTask
|
7
8
|
from zrb.task.any_task_event_handler import (
|
@@ -12,7 +13,6 @@ from zrb.task_env.env_file import EnvFile
|
|
12
13
|
from zrb.task_group.group import Group
|
13
14
|
from zrb.task_input.any_input import AnyInput
|
14
15
|
|
15
|
-
import glob
|
16
16
|
import os
|
17
17
|
|
18
18
|
TPathWatcher = TypeVar('TPathWatcher', bound='PathWatcher')
|
@@ -20,6 +20,16 @@ TPathWatcher = TypeVar('TPathWatcher', bound='PathWatcher')
|
|
20
20
|
|
21
21
|
@typechecked
|
22
22
|
class PathWatcher(Checker):
|
23
|
+
'''
|
24
|
+
PathWatcher will wait for any changes specified on path.
|
25
|
+
|
26
|
+
Once the changes detected, PathWatcher will be completed
|
27
|
+
and several xcom will be set:
|
28
|
+
- <task-name>.file
|
29
|
+
- <task-name>.new-file
|
30
|
+
- <task-name>.modified-file
|
31
|
+
- <task-name>.deleted-file
|
32
|
+
'''
|
23
33
|
|
24
34
|
def __init__(
|
25
35
|
self,
|
@@ -40,6 +50,7 @@ class PathWatcher(Checker):
|
|
40
50
|
on_retry: Optional[OnRetry] = None,
|
41
51
|
on_failed: Optional[OnFailed] = None,
|
42
52
|
path: str = '',
|
53
|
+
ignored_path: Union[str, Iterable[str]] = [],
|
43
54
|
checking_interval: Union[int, float] = 0.1,
|
44
55
|
progress_interval: Union[int, float] = 30,
|
45
56
|
watch_new_files: bool = True,
|
@@ -70,10 +81,12 @@ class PathWatcher(Checker):
|
|
70
81
|
should_execute=should_execute,
|
71
82
|
)
|
72
83
|
self._path = path
|
84
|
+
self._ignored_path = ignored_path
|
73
85
|
self._watch_new_files = watch_new_files
|
74
86
|
self._watch_modified_files = watch_modified_files
|
75
87
|
self._watch_deleted_files = watch_deleted_files
|
76
88
|
self._rendered_path: str = ''
|
89
|
+
self._rendered_ignored_paths: List[str] = []
|
77
90
|
self._init_times: Mapping[str, float] = {}
|
78
91
|
|
79
92
|
def copy(self) -> TPathWatcher:
|
@@ -92,25 +105,43 @@ class PathWatcher(Checker):
|
|
92
105
|
|
93
106
|
async def run(self, *args: Any, **kwargs: Any) -> bool:
|
94
107
|
self._rendered_path = self.render_str(self._path)
|
108
|
+
self._rendered_ignored_paths = [
|
109
|
+
ignored_path
|
110
|
+
for ignored_path in self._get_rendered_ignored_paths()
|
111
|
+
if ignored_path != ''
|
112
|
+
]
|
95
113
|
self._init_times = self._get_mod_times()
|
96
114
|
return await super().run(*args, **kwargs)
|
97
115
|
|
116
|
+
def _get_rendered_ignored_paths(self) -> List[str]:
|
117
|
+
if isinstance(self._ignored_path, str):
|
118
|
+
return [self.render_str(self._ignored_path)]
|
119
|
+
return [
|
120
|
+
self.render_str(ignored_path)
|
121
|
+
for ignored_path in self._ignored_path
|
122
|
+
]
|
123
|
+
|
98
124
|
async def inspect(self, *args: Any, **kwargs: Any) -> bool:
|
99
125
|
label = f'Watching {self._rendered_path}'
|
100
126
|
try:
|
101
127
|
mod_times = self._get_mod_times()
|
102
|
-
except Exception:
|
128
|
+
except Exception as e:
|
103
129
|
self.show_progress(f'{label} Cannot inspect')
|
130
|
+
raise e
|
104
131
|
# watch changes
|
105
132
|
if self._watch_new_files:
|
106
133
|
new_files = mod_times.keys() - self._init_times.keys()
|
107
134
|
for file in new_files:
|
108
135
|
self.print_out_dark(f'{label} [+] New file detected: {file}')
|
136
|
+
self.set_task_xcom('new-file', file)
|
137
|
+
self.set_task_xcom('file', file)
|
109
138
|
return True
|
110
139
|
if self._watch_deleted_files:
|
111
140
|
deleted_files = self._init_times.keys() - mod_times.keys()
|
112
141
|
for file in deleted_files:
|
113
142
|
self.print_out_dark(f'{label} [-] File deleted: {file}')
|
143
|
+
self.set_task_xcom('deleted-file', file)
|
144
|
+
self.set_task_xcom('file', file)
|
114
145
|
return True
|
115
146
|
if self._watch_modified_files:
|
116
147
|
modified_files = {
|
@@ -119,12 +150,22 @@ class PathWatcher(Checker):
|
|
119
150
|
}
|
120
151
|
for file in modified_files:
|
121
152
|
self.print_out_dark(f'{label} [/] File modified: {file}')
|
153
|
+
self.set_task_xcom('modified-file', file)
|
154
|
+
self.set_task_xcom('file', file)
|
122
155
|
return True
|
123
156
|
self.show_progress(f'{label} (Nothing changed)')
|
124
157
|
return False
|
125
158
|
|
126
159
|
def _get_mod_times(self) -> Mapping[str, float]:
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
160
|
+
matches = get_file_names(
|
161
|
+
glob_path=self._rendered_path,
|
162
|
+
glob_ignored_paths=self._rendered_ignored_paths
|
163
|
+
)
|
164
|
+
mod_times: Mapping[str, float] = {}
|
165
|
+
for file_name in matches:
|
166
|
+
try:
|
167
|
+
mod_time = os.stat(file_name).st_mtime
|
168
|
+
mod_times[file_name] = mod_time
|
169
|
+
except Exception as e:
|
170
|
+
self.print_err(e)
|
171
|
+
return mod_times
|
zrb/task/recurring_task.py
CHANGED
@@ -8,7 +8,6 @@ from zrb.task.any_task import AnyTask
|
|
8
8
|
from zrb.task.any_task_event_handler import (
|
9
9
|
OnTriggered, OnWaiting, OnSkipped, OnStarted, OnReady, OnRetry, OnFailed
|
10
10
|
)
|
11
|
-
from zrb.task.task import Task
|
12
11
|
from zrb.task_env.env import Env
|
13
12
|
from zrb.task_env.env_file import EnvFile
|
14
13
|
from zrb.task_group.group import Group
|
@@ -18,14 +17,40 @@ import asyncio
|
|
18
17
|
import copy
|
19
18
|
|
20
19
|
|
20
|
+
class RunConfig():
|
21
|
+
def __init__(
|
22
|
+
self,
|
23
|
+
fn: Callable[..., Any],
|
24
|
+
args: List[Any],
|
25
|
+
kwargs: Mapping[Any, Any],
|
26
|
+
execution_id: str
|
27
|
+
):
|
28
|
+
self.fn = fn
|
29
|
+
self.args = args
|
30
|
+
self.kwargs = kwargs
|
31
|
+
self.execution_id = execution_id
|
32
|
+
|
33
|
+
async def run(self):
|
34
|
+
return await self.fn(*self.args, **self.kwargs)
|
35
|
+
|
36
|
+
|
21
37
|
@typechecked
|
22
38
|
class RecurringTask(BaseTask):
|
39
|
+
'''
|
40
|
+
A class representing a recurring task that is triggered based on
|
41
|
+
specified conditions.
|
42
|
+
|
43
|
+
Examples:
|
44
|
+
|
45
|
+
>>> from zrb import RecurringTask
|
46
|
+
'''
|
23
47
|
|
24
48
|
def __init__(
|
25
49
|
self,
|
26
50
|
name: str,
|
27
51
|
task: AnyTask,
|
28
52
|
triggers: Iterable[AnyTask] = [],
|
53
|
+
single_execution: bool = False,
|
29
54
|
group: Optional[Group] = None,
|
30
55
|
inputs: Iterable[AnyInput] = [],
|
31
56
|
envs: Iterable[Env] = [],
|
@@ -80,6 +105,8 @@ class RecurringTask(BaseTask):
|
|
80
105
|
self._triggers: List[AnyTask] = [
|
81
106
|
trigger.copy() for trigger in triggers
|
82
107
|
]
|
108
|
+
self._run_configs: List[RunConfig] = []
|
109
|
+
self._single_execution = single_execution
|
83
110
|
|
84
111
|
async def _set_keyval(self, kwargs: Mapping[str, Any], env_prefix: str):
|
85
112
|
await super()._set_keyval(kwargs=kwargs, env_prefix=env_prefix)
|
@@ -97,6 +124,12 @@ class RecurringTask(BaseTask):
|
|
97
124
|
await asyncio.gather(*trigger_coroutines)
|
98
125
|
|
99
126
|
async def run(self, *args: Any, **kwargs: Any):
|
127
|
+
await asyncio.gather(
|
128
|
+
asyncio.create_task(self.__check_trigger(*args, **kwargs)),
|
129
|
+
asyncio.create_task(self.__run_from_queue())
|
130
|
+
)
|
131
|
+
|
132
|
+
async def __check_trigger(self, *args: Any, **kwargs: Any):
|
100
133
|
task_kwargs = {
|
101
134
|
key: kwargs[key]
|
102
135
|
for key in kwargs if key not in ['_task']
|
@@ -138,13 +171,31 @@ class RecurringTask(BaseTask):
|
|
138
171
|
fn = task_copy.to_function(
|
139
172
|
is_async=True, raise_error=False, show_done_info=False
|
140
173
|
)
|
141
|
-
self.print_out_dark('
|
142
|
-
|
143
|
-
|
174
|
+
self.print_out_dark(f'Add execution to the queue: {execution_id}')
|
175
|
+
self._run_configs.append(
|
176
|
+
RunConfig(
|
177
|
+
fn=fn,
|
178
|
+
args=args,
|
179
|
+
kwargs=task_kwargs,
|
180
|
+
execution_id=execution_id
|
181
|
+
)
|
144
182
|
)
|
145
183
|
|
146
|
-
async def
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
184
|
+
async def __run_from_queue(self):
|
185
|
+
while True:
|
186
|
+
if len(self._run_configs) == 0:
|
187
|
+
await asyncio.sleep(0.1)
|
188
|
+
continue
|
189
|
+
if self._single_execution:
|
190
|
+
# Drain the queue, leave only the latest task
|
191
|
+
while len(self._run_configs) > 1:
|
192
|
+
run_config = self._run_configs.pop(0)
|
193
|
+
self.print_out_dark(f'Skipping {run_config.execution_id}')
|
194
|
+
self.clear_xcom(execution_id=run_config.execution_id)
|
195
|
+
# Run task
|
196
|
+
run_config = self._run_configs.pop(0)
|
197
|
+
self.print_out_dark(f'Executing {run_config.execution_id}')
|
198
|
+
self.print_out_dark(f'{len(self._run_configs)} tasks left')
|
199
|
+
await run_config.run()
|
200
|
+
self.clear_xcom(execution_id=run_config.execution_id)
|
201
|
+
self._play_bell()
|
zrb/task/time_watcher.py
CHANGED
@@ -20,6 +20,12 @@ TTimeWatcher = TypeVar('TTimeWatcher', bound='TimeWatcher')
|
|
20
20
|
|
21
21
|
@typechecked
|
22
22
|
class TimeWatcher(Checker):
|
23
|
+
'''
|
24
|
+
TimeWatcher will wait for any changes specified on path.
|
25
|
+
|
26
|
+
Once the changes detected, TimeWatcher will be completed
|
27
|
+
and <task-name>.scheduled-time xcom will be set.
|
28
|
+
'''
|
23
29
|
|
24
30
|
def __init__(
|
25
31
|
self,
|
@@ -92,6 +98,7 @@ class TimeWatcher(Checker):
|
|
92
98
|
self._rendered_schedule, slightly_before_check_time
|
93
99
|
)
|
94
100
|
self._scheduled_time = cron.get_next(datetime.datetime)
|
101
|
+
self.set_task_xcom(key='scheduled-time', value=self._scheduled_time)
|
95
102
|
return await super().run(*args, **kwargs)
|
96
103
|
|
97
104
|
async def inspect(self, *args: Any, **kwargs: Any) -> bool:
|