zrb 0.1.0a0__py3-none-any.whl → 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- zrb/__main__.py +5 -1
- zrb/action/runner.py +15 -8
- zrb/builtin/helper/reccuring_action.py +27 -2
- zrb/builtin/say.py +2 -2
- zrb/builtin/schedule.py +4 -3
- zrb/builtin/watch_changes.py +13 -2
- zrb/config/config.py +1 -0
- zrb/helper/cli.py +26 -11
- zrb/helper/default_env.py +23 -13
- zrb/helper/file/copy_tree.py +4 -1
- zrb/helper/file/match.py +20 -0
- zrb/helper/loader/load_module.py +13 -5
- zrb/task/any_task.py +114 -2
- zrb/task/base_task/base_task.py +39 -3
- zrb/task/base_task/component/base_task_model.py +28 -5
- zrb/task/base_task/component/common_task_model.py +30 -15
- zrb/task/base_task/component/renderer.py +6 -0
- zrb/task/cmd_task.py +17 -24
- zrb/task/docker_compose_task.py +0 -1
- zrb/task/path_checker.py +25 -4
- zrb/task/path_watcher.py +48 -7
- zrb/task/recurring_task.py +68 -14
- zrb/task/task.py +1 -1
- zrb/task/time_watcher.py +7 -0
- zrb/task_group/group.py +2 -2
- {zrb-0.1.0a0.dist-info → zrb-0.2.0.dist-info}/METADATA +25 -5
- {zrb-0.1.0a0.dist-info → zrb-0.2.0.dist-info}/RECORD +30 -29
- {zrb-0.1.0a0.dist-info → zrb-0.2.0.dist-info}/LICENSE +0 -0
- {zrb-0.1.0a0.dist-info → zrb-0.2.0.dist-info}/WHEEL +0 -0
- {zrb-0.1.0a0.dist-info → zrb-0.2.0.dist-info}/entry_points.txt +0 -0
zrb/task/base_task/base_task.py
CHANGED
@@ -37,6 +37,8 @@ class BaseTask(
|
|
37
37
|
Base class for all tasks.
|
38
38
|
Every task definition should be extended from this class.
|
39
39
|
'''
|
40
|
+
__xcom: Mapping[str, Mapping[str, str]] = {}
|
41
|
+
|
40
42
|
def __init__(
|
41
43
|
self,
|
42
44
|
name: str,
|
@@ -104,7 +106,9 @@ class BaseTask(
|
|
104
106
|
self.__is_execution_triggered: bool = False
|
105
107
|
self.__is_execution_started: bool = False
|
106
108
|
|
107
|
-
def __rshift__(
|
109
|
+
def __rshift__(
|
110
|
+
self, operand: Union[AnyParallel, AnyTask]
|
111
|
+
) -> Union[AnyParallel, AnyTask]:
|
108
112
|
if isinstance(operand, AnyTask):
|
109
113
|
operand.add_upstream(self)
|
110
114
|
return operand
|
@@ -114,6 +118,33 @@ class BaseTask(
|
|
114
118
|
other_task.add_upstream(self)
|
115
119
|
return operand
|
116
120
|
|
121
|
+
def set_task_xcom(self, key: str, value: Any) -> str:
|
122
|
+
return self.set_xcom(
|
123
|
+
key='.'.join([self.get_name(), key]),
|
124
|
+
value=value
|
125
|
+
)
|
126
|
+
|
127
|
+
def set_xcom(self, key: str, value: Any) -> str:
|
128
|
+
execution_id = self.get_execution_id()
|
129
|
+
if execution_id not in self.__xcom:
|
130
|
+
self.__xcom[execution_id] = {}
|
131
|
+
execution_id = self.get_execution_id()
|
132
|
+
self.__xcom[execution_id][key] = f'{value}'
|
133
|
+
return ''
|
134
|
+
|
135
|
+
def get_xcom(self, key: str) -> str:
|
136
|
+
execution_id = self.get_execution_id()
|
137
|
+
if execution_id not in self.__xcom:
|
138
|
+
return ''
|
139
|
+
return self.__xcom[execution_id].get(key, '')
|
140
|
+
|
141
|
+
def clear_xcom(self, execution_id: str = '') -> str:
|
142
|
+
if execution_id == '':
|
143
|
+
execution_id = self.get_execution_id()
|
144
|
+
if execution_id in self.__xcom:
|
145
|
+
del self.__xcom[execution_id]
|
146
|
+
return ''
|
147
|
+
|
117
148
|
def copy(self) -> AnyTask:
|
118
149
|
return copy.deepcopy(self)
|
119
150
|
|
@@ -224,6 +255,7 @@ class BaseTask(
|
|
224
255
|
]
|
225
256
|
results = await asyncio.gather(*coroutines)
|
226
257
|
result = results[-1]
|
258
|
+
self.set_xcom(self.get_name(), f'{result}')
|
227
259
|
self._print_result(result)
|
228
260
|
return result
|
229
261
|
except Exception as e:
|
@@ -270,14 +302,14 @@ class BaseTask(
|
|
270
302
|
this will return True once every self.checkers is completed
|
271
303
|
- Otherwise, this will return check method's return value.
|
272
304
|
'''
|
273
|
-
if len(self.
|
305
|
+
if len(self._get_checkers()) == 0:
|
274
306
|
return await self.check()
|
275
307
|
self.log_debug('Waiting execution to be started')
|
276
308
|
while not self.__is_execution_started:
|
277
309
|
# Don't start checking before the execution itself has been started
|
278
310
|
await asyncio.sleep(0.1)
|
279
311
|
check_coroutines: Iterable[asyncio.Task] = []
|
280
|
-
for checker_task in self.
|
312
|
+
for checker_task in self._get_checkers():
|
281
313
|
checker_task._set_execution_id(self.get_execution_id())
|
282
314
|
check_coroutines.append(
|
283
315
|
asyncio.create_task(checker_task._run_all())
|
@@ -397,6 +429,7 @@ class BaseTask(
|
|
397
429
|
if self.__is_keyval_set:
|
398
430
|
return True
|
399
431
|
self.__is_keyval_set = True
|
432
|
+
# Set input_map for rendering
|
400
433
|
self.log_info('Set input map')
|
401
434
|
for task_input in self._get_combined_inputs():
|
402
435
|
input_name = to_variable_name(task_input.get_name())
|
@@ -408,6 +441,7 @@ class BaseTask(
|
|
408
441
|
'Input map:\n' + map_to_str(self.get_input_map(), item_prefix=' ')
|
409
442
|
)
|
410
443
|
self.log_info('Merging task envs, task env files, and native envs')
|
444
|
+
# Set env_map for rendering
|
411
445
|
for env_name, env in self._get_combined_env().items():
|
412
446
|
env_value = env.get(env_prefix)
|
413
447
|
if env.should_render():
|
@@ -417,6 +451,8 @@ class BaseTask(
|
|
417
451
|
self.log_debug(
|
418
452
|
'Env map:\n' + map_to_str(self.get_env_map(), item_prefix=' ')
|
419
453
|
)
|
454
|
+
# set task
|
455
|
+
self._set_task(self)
|
420
456
|
|
421
457
|
def __repr__(self) -> str:
|
422
458
|
cls_name = self.__class__.__name__
|
@@ -2,7 +2,7 @@ from zrb.helper.typing import (
|
|
2
2
|
Any, Callable, Iterable, List, Mapping, Optional, Union
|
3
3
|
)
|
4
4
|
from zrb.helper.typecheck import typechecked
|
5
|
-
from zrb.config.config import show_time
|
5
|
+
from zrb.config.config import show_time, logging_level
|
6
6
|
from zrb.task.any_task import AnyTask
|
7
7
|
from zrb.task.any_task_event_handler import (
|
8
8
|
OnTriggered, OnWaiting, OnSkipped, OnStarted, OnReady, OnRetry, OnFailed
|
@@ -19,8 +19,10 @@ from zrb.task.base_task.component.trackers import TimeTracker
|
|
19
19
|
from zrb.config.config import env_prefix
|
20
20
|
from zrb.helper.string.modification import double_quote
|
21
21
|
from zrb.helper.string.conversion import to_variable_name
|
22
|
+
from functools import lru_cache
|
22
23
|
|
23
24
|
import datetime
|
25
|
+
import logging
|
24
26
|
import os
|
25
27
|
import sys
|
26
28
|
|
@@ -90,12 +92,20 @@ class BaseTaskModel(CommonTaskModel, PidModel, TimeTracker):
|
|
90
92
|
self.__kwargs: Mapping[str, Any] = {}
|
91
93
|
|
92
94
|
def _set_args(self, args: Iterable[Any]):
|
95
|
+
'''
|
96
|
+
Set args that will be shown at the end of the execution
|
97
|
+
'''
|
93
98
|
self.__args = list(args)
|
94
99
|
|
95
100
|
def _set_kwargs(self, kwargs: Mapping[str, Any]):
|
101
|
+
'''
|
102
|
+
Set kwargs that will be shown at the end of the execution
|
103
|
+
'''
|
96
104
|
self.__kwargs = kwargs
|
97
105
|
|
98
106
|
def log_debug(self, message: Any):
|
107
|
+
if logging_level > logging.DEBUG:
|
108
|
+
return
|
99
109
|
prefix = self.__get_log_prefix()
|
100
110
|
colored_message = colored(
|
101
111
|
f'{prefix} • {message}', attrs=['dark']
|
@@ -103,6 +113,8 @@ class BaseTaskModel(CommonTaskModel, PidModel, TimeTracker):
|
|
103
113
|
logger.debug(colored_message)
|
104
114
|
|
105
115
|
def log_warn(self, message: Any):
|
116
|
+
if logging_level > logging.WARNING:
|
117
|
+
return
|
106
118
|
prefix = self.__get_log_prefix()
|
107
119
|
colored_message = colored(
|
108
120
|
f'{prefix} • {message}', attrs=['dark']
|
@@ -110,6 +122,8 @@ class BaseTaskModel(CommonTaskModel, PidModel, TimeTracker):
|
|
110
122
|
logger.warning(colored_message)
|
111
123
|
|
112
124
|
def log_info(self, message: Any):
|
125
|
+
if logging_level > logging.INFO:
|
126
|
+
return
|
113
127
|
prefix = self.__get_log_prefix()
|
114
128
|
colored_message = colored(
|
115
129
|
f'{prefix} • {message}', attrs=['dark']
|
@@ -117,6 +131,8 @@ class BaseTaskModel(CommonTaskModel, PidModel, TimeTracker):
|
|
117
131
|
logger.info(colored_message)
|
118
132
|
|
119
133
|
def log_error(self, message: Any):
|
134
|
+
if logging_level > logging.ERROR:
|
135
|
+
return
|
120
136
|
prefix = self.__get_log_prefix()
|
121
137
|
colored_message = colored(
|
122
138
|
f'{prefix} • {message}', color='red', attrs=['bold']
|
@@ -124,6 +140,8 @@ class BaseTaskModel(CommonTaskModel, PidModel, TimeTracker):
|
|
124
140
|
logger.error(colored_message, exc_info=True)
|
125
141
|
|
126
142
|
def log_critical(self, message: Any):
|
143
|
+
if logging_level > logging.CRITICAL:
|
144
|
+
return
|
127
145
|
prefix = self.__get_log_prefix()
|
128
146
|
colored_message = colored(
|
129
147
|
f'{prefix} • {message}', color='red', attrs=['bold']
|
@@ -203,13 +221,15 @@ class BaseTaskModel(CommonTaskModel, PidModel, TimeTracker):
|
|
203
221
|
def __get_print_prefix(self) -> str:
|
204
222
|
common_prefix = self.__get_common_prefix(show_time=show_time)
|
205
223
|
icon = self.get_icon()
|
206
|
-
|
224
|
+
length = LOG_NAME_LENGTH - len(icon)
|
225
|
+
rjust_cli_name = self.__get_rjust_full_cli_name(length)
|
207
226
|
return f'{common_prefix} {icon} {rjust_cli_name}'
|
208
227
|
|
209
228
|
def __get_log_prefix(self) -> str:
|
210
229
|
common_prefix = self.__get_common_prefix(show_time=False)
|
211
230
|
icon = self.get_icon()
|
212
|
-
|
231
|
+
length = LOG_NAME_LENGTH - len(icon)
|
232
|
+
filled_name = self.__get_rjust_full_cli_name(length)
|
213
233
|
return f'{common_prefix} {icon} {filled_name}'
|
214
234
|
|
215
235
|
def __get_common_prefix(self, show_time: bool) -> str:
|
@@ -221,18 +241,21 @@ class BaseTaskModel(CommonTaskModel, PidModel, TimeTracker):
|
|
221
241
|
return f'◷ {now} ❁ {pid} → {attempt}/{max_attempt}'
|
222
242
|
return f'❁ {pid} → {attempt}/{max_attempt}'
|
223
243
|
|
224
|
-
|
244
|
+
@lru_cache
|
245
|
+
def __get_rjust_full_cli_name(self, length: int) -> str:
|
225
246
|
if self.__rjust_full_cli_name is not None:
|
226
247
|
return self.__rjust_full_cli_name
|
227
248
|
complete_name = self._get_full_cli_name()
|
228
|
-
self.__rjust_full_cli_name = complete_name.rjust(
|
249
|
+
self.__rjust_full_cli_name = complete_name.rjust(length, ' ')
|
229
250
|
return self.__rjust_full_cli_name
|
230
251
|
|
252
|
+
@lru_cache
|
231
253
|
def __get_executable_name(self) -> str:
|
232
254
|
if len(sys.argv) > 0 and sys.argv[0] != '':
|
233
255
|
return os.path.basename(sys.argv[0])
|
234
256
|
return 'zrb'
|
235
257
|
|
258
|
+
@lru_cache
|
236
259
|
def _get_full_cli_name(self) -> str:
|
237
260
|
if self.__complete_name is not None:
|
238
261
|
return self.__complete_name
|
@@ -60,7 +60,7 @@ class CommonTaskModel():
|
|
60
60
|
self._retry = retry
|
61
61
|
self._retry_interval = retry_interval
|
62
62
|
self._upstreams = upstreams
|
63
|
-
self._checkers = checkers
|
63
|
+
self._checkers = [checker.copy() for checker in checkers]
|
64
64
|
self._checking_interval = checking_interval
|
65
65
|
self._run_function: Optional[Callable[..., Any]] = run
|
66
66
|
self._on_triggered = on_triggered
|
@@ -77,10 +77,10 @@ class CommonTaskModel():
|
|
77
77
|
self.__allow_add_env_files = True
|
78
78
|
self.__allow_add_inputs = True
|
79
79
|
self.__allow_add_upstreams: bool = True
|
80
|
+
self.__allow_add_checkers: bool = True
|
80
81
|
self.__has_already_inject_env_files: bool = False
|
81
82
|
self.__has_already_inject_envs: bool = False
|
82
83
|
self.__has_already_inject_inputs: bool = False
|
83
|
-
self.__has_already_inject_checkers: bool = False
|
84
84
|
self.__has_already_inject_upstreams: bool = False
|
85
85
|
self.__all_inputs: Optional[List[AnyInput]] = None
|
86
86
|
|
@@ -104,12 +104,15 @@ class CommonTaskModel():
|
|
104
104
|
return self.__execution_id
|
105
105
|
|
106
106
|
def set_name(self, new_name: str):
|
107
|
-
if self._description == self.
|
107
|
+
if self._description == self.get_name():
|
108
108
|
self._description = new_name
|
109
109
|
self._name = new_name
|
110
110
|
|
111
|
+
def get_name(self) -> str:
|
112
|
+
return self._name
|
113
|
+
|
111
114
|
def get_cli_name(self) -> str:
|
112
|
-
return to_cli_name(self.
|
115
|
+
return to_cli_name(self.get_name())
|
113
116
|
|
114
117
|
def set_description(self, new_description: str):
|
115
118
|
self._description = new_description
|
@@ -139,12 +142,12 @@ class CommonTaskModel():
|
|
139
142
|
|
140
143
|
def insert_input(self, *inputs: AnyInput):
|
141
144
|
if not self.__allow_add_inputs:
|
142
|
-
raise Exception(f'Cannot insert inputs for `{self.
|
145
|
+
raise Exception(f'Cannot insert inputs for `{self.get_name()}`')
|
143
146
|
self._inputs = list(inputs) + list(self._inputs)
|
144
147
|
|
145
148
|
def add_input(self, *inputs: AnyInput):
|
146
149
|
if not self.__allow_add_inputs:
|
147
|
-
raise Exception(f'Cannot add inputs for `{self.
|
150
|
+
raise Exception(f'Cannot add inputs for `{self.get_name()}`')
|
148
151
|
self._inputs = list(self._inputs) + list(inputs)
|
149
152
|
|
150
153
|
def inject_inputs(self):
|
@@ -194,12 +197,12 @@ class CommonTaskModel():
|
|
194
197
|
|
195
198
|
def insert_env(self, *envs: Env):
|
196
199
|
if not self.__allow_add_envs:
|
197
|
-
raise Exception(f'Cannot insert envs to `{self.
|
200
|
+
raise Exception(f'Cannot insert envs to `{self.get_name()}`')
|
198
201
|
self._envs = list(envs) + list(self._envs)
|
199
202
|
|
200
203
|
def add_env(self, *envs: Env):
|
201
204
|
if not self.__allow_add_envs:
|
202
|
-
raise Exception(f'Cannot add envs to `{self.
|
205
|
+
raise Exception(f'Cannot add envs to `{self.get_name()}`')
|
203
206
|
self._envs = list(self._envs) + list(envs)
|
204
207
|
|
205
208
|
def inject_envs(self):
|
@@ -230,12 +233,12 @@ class CommonTaskModel():
|
|
230
233
|
|
231
234
|
def insert_env_file(self, *env_files: EnvFile):
|
232
235
|
if not self.__allow_add_env_files:
|
233
|
-
raise Exception(f'Cannot insert env_files to `{self.
|
236
|
+
raise Exception(f'Cannot insert env_files to `{self.get_name()}`')
|
234
237
|
self._env_files = list(env_files) + list(self._env_files)
|
235
238
|
|
236
239
|
def add_env_file(self, *env_files: EnvFile):
|
237
240
|
if not self.__allow_add_env_files:
|
238
|
-
raise Exception(f'Cannot add env_files to `{self.
|
241
|
+
raise Exception(f'Cannot add env_files to `{self.get_name()}`')
|
239
242
|
self._env_files = list(self._env_files) + list(env_files)
|
240
243
|
|
241
244
|
def inject_env_files(self):
|
@@ -243,12 +246,12 @@ class CommonTaskModel():
|
|
243
246
|
|
244
247
|
def insert_upstream(self, *upstreams: AnyTask):
|
245
248
|
if not self.__allow_add_upstreams:
|
246
|
-
raise Exception(f'Cannot insert upstreams to `{self.
|
249
|
+
raise Exception(f'Cannot insert upstreams to `{self.get_name()}`')
|
247
250
|
self._upstreams = list(upstreams) + list(self._upstreams)
|
248
251
|
|
249
252
|
def add_upstream(self, *upstreams: AnyTask):
|
250
253
|
if not self.__allow_add_upstreams:
|
251
|
-
raise Exception(f'Cannot add upstreams to `{self.
|
254
|
+
raise Exception(f'Cannot add upstreams to `{self.get_name()}`')
|
252
255
|
self._upstreams = list(self._upstreams) + list(upstreams)
|
253
256
|
|
254
257
|
def inject_upstreams(self):
|
@@ -272,11 +275,23 @@ class CommonTaskModel():
|
|
272
275
|
self.__has_already_inject_env_files = True
|
273
276
|
return self._env_files
|
274
277
|
|
278
|
+
def insert_checker(self, *checkers: AnyTask):
|
279
|
+
if not self.__allow_add_checkers:
|
280
|
+
raise Exception(f'Cannot insert checkers to `{self.get_name()}`')
|
281
|
+
additional_checkers = [checker.copy() for checker in checkers]
|
282
|
+
self._checkers = additional_checkers + self._checkers
|
283
|
+
|
284
|
+
def add_checker(self, *checkers: AnyTask):
|
285
|
+
if not self.__allow_add_checkers:
|
286
|
+
raise Exception(f'Cannot add checkers to `{self.get_name()}`')
|
287
|
+
additional_checkers = [checker.copy() for checker in checkers]
|
288
|
+
self._checkers = self._checkers + additional_checkers
|
289
|
+
|
275
290
|
def inject_checkers(self):
|
276
291
|
pass
|
277
292
|
|
278
293
|
def _get_checkers(self) -> List[AnyTask]:
|
279
|
-
if not self.
|
294
|
+
if not self.__allow_add_checkers:
|
280
295
|
self.inject_checkers()
|
281
|
-
self.
|
282
|
-
return
|
296
|
+
self.__allow_add_checkers = True
|
297
|
+
return self._checkers
|
@@ -1,3 +1,4 @@
|
|
1
|
+
from zrb.task.any_task import AnyTask
|
1
2
|
from zrb.helper.typing import Any, JinjaTemplate, Mapping, Optional, Union
|
2
3
|
from zrb.helper.typecheck import typechecked
|
3
4
|
from zrb.helper.string.conversion import to_boolean
|
@@ -24,10 +25,14 @@ class Renderer():
|
|
24
25
|
|
25
26
|
def __init__(self):
|
26
27
|
self.__input_map: Mapping[str, Any] = {}
|
28
|
+
self.__task: Optional[AnyTask] = None
|
27
29
|
self.__env_map: Mapping[str, str] = {}
|
28
30
|
self.__render_data: Optional[Mapping[str, Any]] = None
|
29
31
|
self.__rendered_str: Mapping[str, str] = {}
|
30
32
|
|
33
|
+
def _set_task(self, task: AnyTask):
|
34
|
+
self.__task = task
|
35
|
+
|
31
36
|
def get_input_map(self) -> Mapping[str, Any]:
|
32
37
|
# This return reference to input map, so input map can be updated
|
33
38
|
return self.__input_map
|
@@ -126,6 +131,7 @@ class Renderer():
|
|
126
131
|
render_data.update({
|
127
132
|
'env': self.__env_map,
|
128
133
|
'input': self.__input_map,
|
134
|
+
'task': self.__task,
|
129
135
|
})
|
130
136
|
self.__render_data = render_data
|
131
137
|
return render_data
|
zrb/task/cmd_task.py
CHANGED
@@ -48,6 +48,9 @@ class CmdResult():
|
|
48
48
|
self.output = output
|
49
49
|
self.error = error
|
50
50
|
|
51
|
+
def __str__(self) -> str:
|
52
|
+
return self.output
|
53
|
+
|
51
54
|
|
52
55
|
class CmdGlobalState():
|
53
56
|
def __init__(self):
|
@@ -61,30 +64,18 @@ class CmdTask(BaseTask):
|
|
61
64
|
Command Task.
|
62
65
|
You can use this task to run shell command.
|
63
66
|
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
name='
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
runner.register(hello)
|
77
|
-
|
78
|
-
# run a long running process
|
79
|
-
run_server = CmdTask(
|
80
|
-
name='run',
|
81
|
-
inputs=[StrInput(name='dir', default='.')],
|
82
|
-
envs=[Env(name='PORT', os_name='WEB_PORT', default='3000')],
|
83
|
-
cmd='python -m http.server $PORT --directory {{input.dir}}',
|
84
|
-
checkers=[HTTPChecker(port='{{env.PORT}}')]
|
85
|
-
)
|
86
|
-
runner.register(run_server)
|
87
|
-
```
|
67
|
+
Examples:
|
68
|
+
>>> from zrb import runner, CmdTask, StrInput, Env
|
69
|
+
>>> hello = CmdTask(
|
70
|
+
>>> name='hello',
|
71
|
+
>>> inputs=[StrInput(name='name', default='World')],
|
72
|
+
>>> envs=[Env(name='HOME_DIR', os_name='HOME')],
|
73
|
+
>>> cmd=[
|
74
|
+
>>> 'echo Hello {{ input.name }}',
|
75
|
+
>>> 'echo Home directory is: $HOME_DIR',
|
76
|
+
>>> ]
|
77
|
+
>>> )
|
78
|
+
>>> runner.register(hello)
|
88
79
|
'''
|
89
80
|
|
90
81
|
_pids: List[int] = []
|
@@ -246,6 +237,8 @@ class CmdTask(BaseTask):
|
|
246
237
|
raise Exception(
|
247
238
|
f'Process {self._name} exited ({return_code}): {error}'
|
248
239
|
)
|
240
|
+
self.set_task_xcom(key='output', value=output)
|
241
|
+
self.set_task_xcom(key='error', value=error)
|
249
242
|
return CmdResult(output, error)
|
250
243
|
|
251
244
|
def _should_attempt(self) -> bool:
|
zrb/task/docker_compose_task.py
CHANGED
@@ -293,7 +293,6 @@ class DockerComposeTask(CmdTask):
|
|
293
293
|
]:
|
294
294
|
if os.path.exists(os.path.join(self._cwd, _compose_file)):
|
295
295
|
return os.path.join(self._cwd, _compose_file)
|
296
|
-
return
|
297
296
|
raise Exception(f'Cannot find compose file on {self._cwd}')
|
298
297
|
if os.path.isabs(compose_file) and os.path.exists(compose_file):
|
299
298
|
return compose_file
|
zrb/task/path_checker.py
CHANGED
@@ -1,5 +1,8 @@
|
|
1
|
-
from zrb.helper.typing import
|
1
|
+
from zrb.helper.typing import (
|
2
|
+
Any, Callable, Iterable, List, Optional, Union, TypeVar
|
3
|
+
)
|
2
4
|
from zrb.helper.typecheck import typechecked
|
5
|
+
from zrb.helper.file.match import get_file_names
|
3
6
|
from zrb.task.checker import Checker
|
4
7
|
from zrb.task.any_task import AnyTask
|
5
8
|
from zrb.task.any_task_event_handler import (
|
@@ -10,8 +13,6 @@ from zrb.task_env.env_file import EnvFile
|
|
10
13
|
from zrb.task_group.group import Group
|
11
14
|
from zrb.task_input.any_input import AnyInput
|
12
15
|
|
13
|
-
import glob
|
14
|
-
|
15
16
|
TPathChecker = TypeVar('TPathChecker', bound='PathChecker')
|
16
17
|
|
17
18
|
|
@@ -37,6 +38,7 @@ class PathChecker(Checker):
|
|
37
38
|
on_retry: Optional[OnRetry] = None,
|
38
39
|
on_failed: Optional[OnFailed] = None,
|
39
40
|
path: str = '',
|
41
|
+
ignored_path: Union[str, Iterable[str]] = [],
|
40
42
|
checking_interval: Union[int, float] = 0.1,
|
41
43
|
progress_interval: Union[int, float] = 5,
|
42
44
|
expected_result: bool = True,
|
@@ -66,7 +68,9 @@ class PathChecker(Checker):
|
|
66
68
|
should_execute=should_execute,
|
67
69
|
)
|
68
70
|
self._path = path
|
71
|
+
self._ignored_path = ignored_path
|
69
72
|
self._rendered_path: str = ''
|
73
|
+
self._rendered_ignored_paths: List[str] = []
|
70
74
|
|
71
75
|
def copy(self) -> TPathChecker:
|
72
76
|
return super().copy()
|
@@ -84,12 +88,29 @@ class PathChecker(Checker):
|
|
84
88
|
|
85
89
|
async def run(self, *args: Any, **kwargs: Any) -> bool:
|
86
90
|
self._rendered_path = self.render_str(self._path)
|
91
|
+
self._rendered_ignored_paths = [
|
92
|
+
ignored_path
|
93
|
+
for ignored_path in self._get_rendered_ignored_paths()
|
94
|
+
if ignored_path != ''
|
95
|
+
]
|
87
96
|
return await super().run(*args, **kwargs)
|
88
97
|
|
98
|
+
def _get_rendered_ignored_paths(self) -> List[str]:
|
99
|
+
if isinstance(self._ignored_path, str):
|
100
|
+
return [self.render_str(self._ignored_path)]
|
101
|
+
return [
|
102
|
+
self.render_str(ignored_path)
|
103
|
+
for ignored_path in self._ignored_path
|
104
|
+
]
|
105
|
+
|
89
106
|
async def inspect(self, *args: Any, **kwargs: Any) -> bool:
|
90
107
|
label = f'Checking {self._rendered_path}'
|
91
108
|
try:
|
92
|
-
|
109
|
+
matches = get_file_names(
|
110
|
+
glob_path=self._rendered_path,
|
111
|
+
glob_ignored_paths=self._rendered_ignored_paths
|
112
|
+
)
|
113
|
+
if len(matches) > 0:
|
93
114
|
self.print_out(f'{label} (Exist)')
|
94
115
|
return True
|
95
116
|
self.show_progress(f'{label} (Not Exist)')
|
zrb/task/path_watcher.py
CHANGED
@@ -1,7 +1,8 @@
|
|
1
1
|
from zrb.helper.typing import (
|
2
|
-
Any, Callable, Iterable, Mapping, Optional, Union, TypeVar
|
2
|
+
Any, Callable, Iterable, List, Mapping, Optional, Union, TypeVar
|
3
3
|
)
|
4
4
|
from zrb.helper.typecheck import typechecked
|
5
|
+
from zrb.helper.file.match import get_file_names
|
5
6
|
from zrb.task.checker import Checker
|
6
7
|
from zrb.task.any_task import AnyTask
|
7
8
|
from zrb.task.any_task_event_handler import (
|
@@ -12,7 +13,6 @@ from zrb.task_env.env_file import EnvFile
|
|
12
13
|
from zrb.task_group.group import Group
|
13
14
|
from zrb.task_input.any_input import AnyInput
|
14
15
|
|
15
|
-
import glob
|
16
16
|
import os
|
17
17
|
|
18
18
|
TPathWatcher = TypeVar('TPathWatcher', bound='PathWatcher')
|
@@ -20,6 +20,16 @@ TPathWatcher = TypeVar('TPathWatcher', bound='PathWatcher')
|
|
20
20
|
|
21
21
|
@typechecked
|
22
22
|
class PathWatcher(Checker):
|
23
|
+
'''
|
24
|
+
PathWatcher will wait for any changes specified on path.
|
25
|
+
|
26
|
+
Once the changes detected, PathWatcher will be completed
|
27
|
+
and several xcom will be set:
|
28
|
+
- <task-name>.file
|
29
|
+
- <task-name>.new-file
|
30
|
+
- <task-name>.modified-file
|
31
|
+
- <task-name>.deleted-file
|
32
|
+
'''
|
23
33
|
|
24
34
|
def __init__(
|
25
35
|
self,
|
@@ -40,6 +50,7 @@ class PathWatcher(Checker):
|
|
40
50
|
on_retry: Optional[OnRetry] = None,
|
41
51
|
on_failed: Optional[OnFailed] = None,
|
42
52
|
path: str = '',
|
53
|
+
ignored_path: Union[str, Iterable[str]] = [],
|
43
54
|
checking_interval: Union[int, float] = 0.1,
|
44
55
|
progress_interval: Union[int, float] = 30,
|
45
56
|
watch_new_files: bool = True,
|
@@ -70,10 +81,12 @@ class PathWatcher(Checker):
|
|
70
81
|
should_execute=should_execute,
|
71
82
|
)
|
72
83
|
self._path = path
|
84
|
+
self._ignored_path = ignored_path
|
73
85
|
self._watch_new_files = watch_new_files
|
74
86
|
self._watch_modified_files = watch_modified_files
|
75
87
|
self._watch_deleted_files = watch_deleted_files
|
76
88
|
self._rendered_path: str = ''
|
89
|
+
self._rendered_ignored_paths: List[str] = []
|
77
90
|
self._init_times: Mapping[str, float] = {}
|
78
91
|
|
79
92
|
def copy(self) -> TPathWatcher:
|
@@ -92,25 +105,43 @@ class PathWatcher(Checker):
|
|
92
105
|
|
93
106
|
async def run(self, *args: Any, **kwargs: Any) -> bool:
|
94
107
|
self._rendered_path = self.render_str(self._path)
|
108
|
+
self._rendered_ignored_paths = [
|
109
|
+
ignored_path
|
110
|
+
for ignored_path in self._get_rendered_ignored_paths()
|
111
|
+
if ignored_path != ''
|
112
|
+
]
|
95
113
|
self._init_times = self._get_mod_times()
|
96
114
|
return await super().run(*args, **kwargs)
|
97
115
|
|
116
|
+
def _get_rendered_ignored_paths(self) -> List[str]:
|
117
|
+
if isinstance(self._ignored_path, str):
|
118
|
+
return [self.render_str(self._ignored_path)]
|
119
|
+
return [
|
120
|
+
self.render_str(ignored_path)
|
121
|
+
for ignored_path in self._ignored_path
|
122
|
+
]
|
123
|
+
|
98
124
|
async def inspect(self, *args: Any, **kwargs: Any) -> bool:
|
99
125
|
label = f'Watching {self._rendered_path}'
|
100
126
|
try:
|
101
127
|
mod_times = self._get_mod_times()
|
102
|
-
except Exception:
|
128
|
+
except Exception as e:
|
103
129
|
self.show_progress(f'{label} Cannot inspect')
|
130
|
+
raise e
|
104
131
|
# watch changes
|
105
132
|
if self._watch_new_files:
|
106
133
|
new_files = mod_times.keys() - self._init_times.keys()
|
107
134
|
for file in new_files:
|
108
135
|
self.print_out_dark(f'{label} [+] New file detected: {file}')
|
136
|
+
self.set_task_xcom('new-file', file)
|
137
|
+
self.set_task_xcom('file', file)
|
109
138
|
return True
|
110
139
|
if self._watch_deleted_files:
|
111
140
|
deleted_files = self._init_times.keys() - mod_times.keys()
|
112
141
|
for file in deleted_files:
|
113
142
|
self.print_out_dark(f'{label} [-] File deleted: {file}')
|
143
|
+
self.set_task_xcom('deleted-file', file)
|
144
|
+
self.set_task_xcom('file', file)
|
114
145
|
return True
|
115
146
|
if self._watch_modified_files:
|
116
147
|
modified_files = {
|
@@ -119,12 +150,22 @@ class PathWatcher(Checker):
|
|
119
150
|
}
|
120
151
|
for file in modified_files:
|
121
152
|
self.print_out_dark(f'{label} [/] File modified: {file}')
|
153
|
+
self.set_task_xcom('modified-file', file)
|
154
|
+
self.set_task_xcom('file', file)
|
122
155
|
return True
|
123
156
|
self.show_progress(f'{label} (Nothing changed)')
|
124
157
|
return False
|
125
158
|
|
126
159
|
def _get_mod_times(self) -> Mapping[str, float]:
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
160
|
+
matches = get_file_names(
|
161
|
+
glob_path=self._rendered_path,
|
162
|
+
glob_ignored_paths=self._rendered_ignored_paths
|
163
|
+
)
|
164
|
+
mod_times: Mapping[str, float] = {}
|
165
|
+
for file_name in matches:
|
166
|
+
try:
|
167
|
+
mod_time = os.stat(file_name).st_mtime
|
168
|
+
mod_times[file_name] = mod_time
|
169
|
+
except Exception as e:
|
170
|
+
self.print_err(e)
|
171
|
+
return mod_times
|