zrb 0.1.0a0__py3-none-any.whl → 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -37,6 +37,8 @@ class BaseTask(
37
37
  Base class for all tasks.
38
38
  Every task definition should be extended from this class.
39
39
  '''
40
+ __xcom: Mapping[str, Mapping[str, str]] = {}
41
+
40
42
  def __init__(
41
43
  self,
42
44
  name: str,
@@ -104,7 +106,9 @@ class BaseTask(
104
106
  self.__is_execution_triggered: bool = False
105
107
  self.__is_execution_started: bool = False
106
108
 
107
- def __rshift__(self, operand: Union[AnyParallel, AnyTask]):
109
+ def __rshift__(
110
+ self, operand: Union[AnyParallel, AnyTask]
111
+ ) -> Union[AnyParallel, AnyTask]:
108
112
  if isinstance(operand, AnyTask):
109
113
  operand.add_upstream(self)
110
114
  return operand
@@ -114,6 +118,33 @@ class BaseTask(
114
118
  other_task.add_upstream(self)
115
119
  return operand
116
120
 
121
+ def set_task_xcom(self, key: str, value: Any) -> str:
122
+ return self.set_xcom(
123
+ key='.'.join([self.get_name(), key]),
124
+ value=value
125
+ )
126
+
127
+ def set_xcom(self, key: str, value: Any) -> str:
128
+ execution_id = self.get_execution_id()
129
+ if execution_id not in self.__xcom:
130
+ self.__xcom[execution_id] = {}
131
+ execution_id = self.get_execution_id()
132
+ self.__xcom[execution_id][key] = f'{value}'
133
+ return ''
134
+
135
+ def get_xcom(self, key: str) -> str:
136
+ execution_id = self.get_execution_id()
137
+ if execution_id not in self.__xcom:
138
+ return ''
139
+ return self.__xcom[execution_id].get(key, '')
140
+
141
+ def clear_xcom(self, execution_id: str = '') -> str:
142
+ if execution_id == '':
143
+ execution_id = self.get_execution_id()
144
+ if execution_id in self.__xcom:
145
+ del self.__xcom[execution_id]
146
+ return ''
147
+
117
148
  def copy(self) -> AnyTask:
118
149
  return copy.deepcopy(self)
119
150
 
@@ -224,6 +255,7 @@ class BaseTask(
224
255
  ]
225
256
  results = await asyncio.gather(*coroutines)
226
257
  result = results[-1]
258
+ self.set_xcom(self.get_name(), f'{result}')
227
259
  self._print_result(result)
228
260
  return result
229
261
  except Exception as e:
@@ -270,14 +302,14 @@ class BaseTask(
270
302
  this will return True once every self.checkers is completed
271
303
  - Otherwise, this will return check method's return value.
272
304
  '''
273
- if len(self._checkers) == 0:
305
+ if len(self._get_checkers()) == 0:
274
306
  return await self.check()
275
307
  self.log_debug('Waiting execution to be started')
276
308
  while not self.__is_execution_started:
277
309
  # Don't start checking before the execution itself has been started
278
310
  await asyncio.sleep(0.1)
279
311
  check_coroutines: Iterable[asyncio.Task] = []
280
- for checker_task in self._checkers:
312
+ for checker_task in self._get_checkers():
281
313
  checker_task._set_execution_id(self.get_execution_id())
282
314
  check_coroutines.append(
283
315
  asyncio.create_task(checker_task._run_all())
@@ -397,6 +429,7 @@ class BaseTask(
397
429
  if self.__is_keyval_set:
398
430
  return True
399
431
  self.__is_keyval_set = True
432
+ # Set input_map for rendering
400
433
  self.log_info('Set input map')
401
434
  for task_input in self._get_combined_inputs():
402
435
  input_name = to_variable_name(task_input.get_name())
@@ -408,6 +441,7 @@ class BaseTask(
408
441
  'Input map:\n' + map_to_str(self.get_input_map(), item_prefix=' ')
409
442
  )
410
443
  self.log_info('Merging task envs, task env files, and native envs')
444
+ # Set env_map for rendering
411
445
  for env_name, env in self._get_combined_env().items():
412
446
  env_value = env.get(env_prefix)
413
447
  if env.should_render():
@@ -417,6 +451,8 @@ class BaseTask(
417
451
  self.log_debug(
418
452
  'Env map:\n' + map_to_str(self.get_env_map(), item_prefix=' ')
419
453
  )
454
+ # set task
455
+ self._set_task(self)
420
456
 
421
457
  def __repr__(self) -> str:
422
458
  cls_name = self.__class__.__name__
@@ -2,7 +2,7 @@ from zrb.helper.typing import (
2
2
  Any, Callable, Iterable, List, Mapping, Optional, Union
3
3
  )
4
4
  from zrb.helper.typecheck import typechecked
5
- from zrb.config.config import show_time
5
+ from zrb.config.config import show_time, logging_level
6
6
  from zrb.task.any_task import AnyTask
7
7
  from zrb.task.any_task_event_handler import (
8
8
  OnTriggered, OnWaiting, OnSkipped, OnStarted, OnReady, OnRetry, OnFailed
@@ -19,8 +19,10 @@ from zrb.task.base_task.component.trackers import TimeTracker
19
19
  from zrb.config.config import env_prefix
20
20
  from zrb.helper.string.modification import double_quote
21
21
  from zrb.helper.string.conversion import to_variable_name
22
+ from functools import lru_cache
22
23
 
23
24
  import datetime
25
+ import logging
24
26
  import os
25
27
  import sys
26
28
 
@@ -90,12 +92,20 @@ class BaseTaskModel(CommonTaskModel, PidModel, TimeTracker):
90
92
  self.__kwargs: Mapping[str, Any] = {}
91
93
 
92
94
  def _set_args(self, args: Iterable[Any]):
95
+ '''
96
+ Set args that will be shown at the end of the execution
97
+ '''
93
98
  self.__args = list(args)
94
99
 
95
100
  def _set_kwargs(self, kwargs: Mapping[str, Any]):
101
+ '''
102
+ Set kwargs that will be shown at the end of the execution
103
+ '''
96
104
  self.__kwargs = kwargs
97
105
 
98
106
  def log_debug(self, message: Any):
107
+ if logging_level > logging.DEBUG:
108
+ return
99
109
  prefix = self.__get_log_prefix()
100
110
  colored_message = colored(
101
111
  f'{prefix} • {message}', attrs=['dark']
@@ -103,6 +113,8 @@ class BaseTaskModel(CommonTaskModel, PidModel, TimeTracker):
103
113
  logger.debug(colored_message)
104
114
 
105
115
  def log_warn(self, message: Any):
116
+ if logging_level > logging.WARNING:
117
+ return
106
118
  prefix = self.__get_log_prefix()
107
119
  colored_message = colored(
108
120
  f'{prefix} • {message}', attrs=['dark']
@@ -110,6 +122,8 @@ class BaseTaskModel(CommonTaskModel, PidModel, TimeTracker):
110
122
  logger.warning(colored_message)
111
123
 
112
124
  def log_info(self, message: Any):
125
+ if logging_level > logging.INFO:
126
+ return
113
127
  prefix = self.__get_log_prefix()
114
128
  colored_message = colored(
115
129
  f'{prefix} • {message}', attrs=['dark']
@@ -117,6 +131,8 @@ class BaseTaskModel(CommonTaskModel, PidModel, TimeTracker):
117
131
  logger.info(colored_message)
118
132
 
119
133
  def log_error(self, message: Any):
134
+ if logging_level > logging.ERROR:
135
+ return
120
136
  prefix = self.__get_log_prefix()
121
137
  colored_message = colored(
122
138
  f'{prefix} • {message}', color='red', attrs=['bold']
@@ -124,6 +140,8 @@ class BaseTaskModel(CommonTaskModel, PidModel, TimeTracker):
124
140
  logger.error(colored_message, exc_info=True)
125
141
 
126
142
  def log_critical(self, message: Any):
143
+ if logging_level > logging.CRITICAL:
144
+ return
127
145
  prefix = self.__get_log_prefix()
128
146
  colored_message = colored(
129
147
  f'{prefix} • {message}', color='red', attrs=['bold']
@@ -203,13 +221,15 @@ class BaseTaskModel(CommonTaskModel, PidModel, TimeTracker):
203
221
  def __get_print_prefix(self) -> str:
204
222
  common_prefix = self.__get_common_prefix(show_time=show_time)
205
223
  icon = self.get_icon()
206
- rjust_cli_name = self.__get_rjust_full_cli_name()
224
+ length = LOG_NAME_LENGTH - len(icon)
225
+ rjust_cli_name = self.__get_rjust_full_cli_name(length)
207
226
  return f'{common_prefix} {icon} {rjust_cli_name}'
208
227
 
209
228
  def __get_log_prefix(self) -> str:
210
229
  common_prefix = self.__get_common_prefix(show_time=False)
211
230
  icon = self.get_icon()
212
- filled_name = self.__get_rjust_full_cli_name()
231
+ length = LOG_NAME_LENGTH - len(icon)
232
+ filled_name = self.__get_rjust_full_cli_name(length)
213
233
  return f'{common_prefix} {icon} {filled_name}'
214
234
 
215
235
  def __get_common_prefix(self, show_time: bool) -> str:
@@ -221,18 +241,21 @@ class BaseTaskModel(CommonTaskModel, PidModel, TimeTracker):
221
241
  return f'◷ {now} ❁ {pid} → {attempt}/{max_attempt}'
222
242
  return f'❁ {pid} → {attempt}/{max_attempt}'
223
243
 
224
- def __get_rjust_full_cli_name(self) -> str:
244
+ @lru_cache
245
+ def __get_rjust_full_cli_name(self, length: int) -> str:
225
246
  if self.__rjust_full_cli_name is not None:
226
247
  return self.__rjust_full_cli_name
227
248
  complete_name = self._get_full_cli_name()
228
- self.__rjust_full_cli_name = complete_name.rjust(LOG_NAME_LENGTH, ' ')
249
+ self.__rjust_full_cli_name = complete_name.rjust(length, ' ')
229
250
  return self.__rjust_full_cli_name
230
251
 
252
+ @lru_cache
231
253
  def __get_executable_name(self) -> str:
232
254
  if len(sys.argv) > 0 and sys.argv[0] != '':
233
255
  return os.path.basename(sys.argv[0])
234
256
  return 'zrb'
235
257
 
258
+ @lru_cache
236
259
  def _get_full_cli_name(self) -> str:
237
260
  if self.__complete_name is not None:
238
261
  return self.__complete_name
@@ -60,7 +60,7 @@ class CommonTaskModel():
60
60
  self._retry = retry
61
61
  self._retry_interval = retry_interval
62
62
  self._upstreams = upstreams
63
- self._checkers = checkers
63
+ self._checkers = [checker.copy() for checker in checkers]
64
64
  self._checking_interval = checking_interval
65
65
  self._run_function: Optional[Callable[..., Any]] = run
66
66
  self._on_triggered = on_triggered
@@ -77,10 +77,10 @@ class CommonTaskModel():
77
77
  self.__allow_add_env_files = True
78
78
  self.__allow_add_inputs = True
79
79
  self.__allow_add_upstreams: bool = True
80
+ self.__allow_add_checkers: bool = True
80
81
  self.__has_already_inject_env_files: bool = False
81
82
  self.__has_already_inject_envs: bool = False
82
83
  self.__has_already_inject_inputs: bool = False
83
- self.__has_already_inject_checkers: bool = False
84
84
  self.__has_already_inject_upstreams: bool = False
85
85
  self.__all_inputs: Optional[List[AnyInput]] = None
86
86
 
@@ -104,12 +104,15 @@ class CommonTaskModel():
104
104
  return self.__execution_id
105
105
 
106
106
  def set_name(self, new_name: str):
107
- if self._description == self._name:
107
+ if self._description == self.get_name():
108
108
  self._description = new_name
109
109
  self._name = new_name
110
110
 
111
+ def get_name(self) -> str:
112
+ return self._name
113
+
111
114
  def get_cli_name(self) -> str:
112
- return to_cli_name(self._name)
115
+ return to_cli_name(self.get_name())
113
116
 
114
117
  def set_description(self, new_description: str):
115
118
  self._description = new_description
@@ -139,12 +142,12 @@ class CommonTaskModel():
139
142
 
140
143
  def insert_input(self, *inputs: AnyInput):
141
144
  if not self.__allow_add_inputs:
142
- raise Exception(f'Cannot insert inputs for `{self._name}`')
145
+ raise Exception(f'Cannot insert inputs for `{self.get_name()}`')
143
146
  self._inputs = list(inputs) + list(self._inputs)
144
147
 
145
148
  def add_input(self, *inputs: AnyInput):
146
149
  if not self.__allow_add_inputs:
147
- raise Exception(f'Cannot add inputs for `{self._name}`')
150
+ raise Exception(f'Cannot add inputs for `{self.get_name()}`')
148
151
  self._inputs = list(self._inputs) + list(inputs)
149
152
 
150
153
  def inject_inputs(self):
@@ -194,12 +197,12 @@ class CommonTaskModel():
194
197
 
195
198
  def insert_env(self, *envs: Env):
196
199
  if not self.__allow_add_envs:
197
- raise Exception(f'Cannot insert envs to `{self._name}`')
200
+ raise Exception(f'Cannot insert envs to `{self.get_name()}`')
198
201
  self._envs = list(envs) + list(self._envs)
199
202
 
200
203
  def add_env(self, *envs: Env):
201
204
  if not self.__allow_add_envs:
202
- raise Exception(f'Cannot add envs to `{self._name}`')
205
+ raise Exception(f'Cannot add envs to `{self.get_name()}`')
203
206
  self._envs = list(self._envs) + list(envs)
204
207
 
205
208
  def inject_envs(self):
@@ -230,12 +233,12 @@ class CommonTaskModel():
230
233
 
231
234
  def insert_env_file(self, *env_files: EnvFile):
232
235
  if not self.__allow_add_env_files:
233
- raise Exception(f'Cannot insert env_files to `{self._name}`')
236
+ raise Exception(f'Cannot insert env_files to `{self.get_name()}`')
234
237
  self._env_files = list(env_files) + list(self._env_files)
235
238
 
236
239
  def add_env_file(self, *env_files: EnvFile):
237
240
  if not self.__allow_add_env_files:
238
- raise Exception(f'Cannot add env_files to `{self._name}`')
241
+ raise Exception(f'Cannot add env_files to `{self.get_name()}`')
239
242
  self._env_files = list(self._env_files) + list(env_files)
240
243
 
241
244
  def inject_env_files(self):
@@ -243,12 +246,12 @@ class CommonTaskModel():
243
246
 
244
247
  def insert_upstream(self, *upstreams: AnyTask):
245
248
  if not self.__allow_add_upstreams:
246
- raise Exception(f'Cannot insert upstreams to `{self._name}`')
249
+ raise Exception(f'Cannot insert upstreams to `{self.get_name()}`')
247
250
  self._upstreams = list(upstreams) + list(self._upstreams)
248
251
 
249
252
  def add_upstream(self, *upstreams: AnyTask):
250
253
  if not self.__allow_add_upstreams:
251
- raise Exception(f'Cannot add upstreams to `{self._name}`')
254
+ raise Exception(f'Cannot add upstreams to `{self.get_name()}`')
252
255
  self._upstreams = list(self._upstreams) + list(upstreams)
253
256
 
254
257
  def inject_upstreams(self):
@@ -272,11 +275,23 @@ class CommonTaskModel():
272
275
  self.__has_already_inject_env_files = True
273
276
  return self._env_files
274
277
 
278
+ def insert_checker(self, *checkers: AnyTask):
279
+ if not self.__allow_add_checkers:
280
+ raise Exception(f'Cannot insert checkers to `{self.get_name()}`')
281
+ additional_checkers = [checker.copy() for checker in checkers]
282
+ self._checkers = additional_checkers + self._checkers
283
+
284
+ def add_checker(self, *checkers: AnyTask):
285
+ if not self.__allow_add_checkers:
286
+ raise Exception(f'Cannot add checkers to `{self.get_name()}`')
287
+ additional_checkers = [checker.copy() for checker in checkers]
288
+ self._checkers = self._checkers + additional_checkers
289
+
275
290
  def inject_checkers(self):
276
291
  pass
277
292
 
278
293
  def _get_checkers(self) -> List[AnyTask]:
279
- if not self.__has_already_inject_checkers:
294
+ if not self.__allow_add_checkers:
280
295
  self.inject_checkers()
281
- self.__has_already_inject_checkers = True
282
- return list(self._checkers)
296
+ self.__allow_add_checkers = True
297
+ return self._checkers
@@ -1,3 +1,4 @@
1
+ from zrb.task.any_task import AnyTask
1
2
  from zrb.helper.typing import Any, JinjaTemplate, Mapping, Optional, Union
2
3
  from zrb.helper.typecheck import typechecked
3
4
  from zrb.helper.string.conversion import to_boolean
@@ -24,10 +25,14 @@ class Renderer():
24
25
 
25
26
  def __init__(self):
26
27
  self.__input_map: Mapping[str, Any] = {}
28
+ self.__task: Optional[AnyTask] = None
27
29
  self.__env_map: Mapping[str, str] = {}
28
30
  self.__render_data: Optional[Mapping[str, Any]] = None
29
31
  self.__rendered_str: Mapping[str, str] = {}
30
32
 
33
+ def _set_task(self, task: AnyTask):
34
+ self.__task = task
35
+
31
36
  def get_input_map(self) -> Mapping[str, Any]:
32
37
  # This return reference to input map, so input map can be updated
33
38
  return self.__input_map
@@ -126,6 +131,7 @@ class Renderer():
126
131
  render_data.update({
127
132
  'env': self.__env_map,
128
133
  'input': self.__input_map,
134
+ 'task': self.__task,
129
135
  })
130
136
  self.__render_data = render_data
131
137
  return render_data
zrb/task/cmd_task.py CHANGED
@@ -48,6 +48,9 @@ class CmdResult():
48
48
  self.output = output
49
49
  self.error = error
50
50
 
51
+ def __str__(self) -> str:
52
+ return self.output
53
+
51
54
 
52
55
  class CmdGlobalState():
53
56
  def __init__(self):
@@ -61,30 +64,18 @@ class CmdTask(BaseTask):
61
64
  Command Task.
62
65
  You can use this task to run shell command.
63
66
 
64
- For example:
65
- ```python
66
- # run a simple task
67
- hello = CmdTask(
68
- name='hello',
69
- inputs=[StrInput(name='name', default='World')],
70
- envs=[Env(name='HOME_DIR', os_name='HOME')],
71
- cmd=[
72
- 'echo Hello {{ input.name }}',
73
- 'echo Home directory is: $HOME_DIR',
74
- ]
75
- )
76
- runner.register(hello)
77
-
78
- # run a long running process
79
- run_server = CmdTask(
80
- name='run',
81
- inputs=[StrInput(name='dir', default='.')],
82
- envs=[Env(name='PORT', os_name='WEB_PORT', default='3000')],
83
- cmd='python -m http.server $PORT --directory {{input.dir}}',
84
- checkers=[HTTPChecker(port='{{env.PORT}}')]
85
- )
86
- runner.register(run_server)
87
- ```
67
+ Examples:
68
+ >>> from zrb import runner, CmdTask, StrInput, Env
69
+ >>> hello = CmdTask(
70
+ >>> name='hello',
71
+ >>> inputs=[StrInput(name='name', default='World')],
72
+ >>> envs=[Env(name='HOME_DIR', os_name='HOME')],
73
+ >>> cmd=[
74
+ >>> 'echo Hello {{ input.name }}',
75
+ >>> 'echo Home directory is: $HOME_DIR',
76
+ >>> ]
77
+ >>> )
78
+ >>> runner.register(hello)
88
79
  '''
89
80
 
90
81
  _pids: List[int] = []
@@ -246,6 +237,8 @@ class CmdTask(BaseTask):
246
237
  raise Exception(
247
238
  f'Process {self._name} exited ({return_code}): {error}'
248
239
  )
240
+ self.set_task_xcom(key='output', value=output)
241
+ self.set_task_xcom(key='error', value=error)
249
242
  return CmdResult(output, error)
250
243
 
251
244
  def _should_attempt(self) -> bool:
@@ -293,7 +293,6 @@ class DockerComposeTask(CmdTask):
293
293
  ]:
294
294
  if os.path.exists(os.path.join(self._cwd, _compose_file)):
295
295
  return os.path.join(self._cwd, _compose_file)
296
- return
297
296
  raise Exception(f'Cannot find compose file on {self._cwd}')
298
297
  if os.path.isabs(compose_file) and os.path.exists(compose_file):
299
298
  return compose_file
zrb/task/path_checker.py CHANGED
@@ -1,5 +1,8 @@
1
- from zrb.helper.typing import Any, Callable, Iterable, Optional, Union, TypeVar
1
+ from zrb.helper.typing import (
2
+ Any, Callable, Iterable, List, Optional, Union, TypeVar
3
+ )
2
4
  from zrb.helper.typecheck import typechecked
5
+ from zrb.helper.file.match import get_file_names
3
6
  from zrb.task.checker import Checker
4
7
  from zrb.task.any_task import AnyTask
5
8
  from zrb.task.any_task_event_handler import (
@@ -10,8 +13,6 @@ from zrb.task_env.env_file import EnvFile
10
13
  from zrb.task_group.group import Group
11
14
  from zrb.task_input.any_input import AnyInput
12
15
 
13
- import glob
14
-
15
16
  TPathChecker = TypeVar('TPathChecker', bound='PathChecker')
16
17
 
17
18
 
@@ -37,6 +38,7 @@ class PathChecker(Checker):
37
38
  on_retry: Optional[OnRetry] = None,
38
39
  on_failed: Optional[OnFailed] = None,
39
40
  path: str = '',
41
+ ignored_path: Union[str, Iterable[str]] = [],
40
42
  checking_interval: Union[int, float] = 0.1,
41
43
  progress_interval: Union[int, float] = 5,
42
44
  expected_result: bool = True,
@@ -66,7 +68,9 @@ class PathChecker(Checker):
66
68
  should_execute=should_execute,
67
69
  )
68
70
  self._path = path
71
+ self._ignored_path = ignored_path
69
72
  self._rendered_path: str = ''
73
+ self._rendered_ignored_paths: List[str] = []
70
74
 
71
75
  def copy(self) -> TPathChecker:
72
76
  return super().copy()
@@ -84,12 +88,29 @@ class PathChecker(Checker):
84
88
 
85
89
  async def run(self, *args: Any, **kwargs: Any) -> bool:
86
90
  self._rendered_path = self.render_str(self._path)
91
+ self._rendered_ignored_paths = [
92
+ ignored_path
93
+ for ignored_path in self._get_rendered_ignored_paths()
94
+ if ignored_path != ''
95
+ ]
87
96
  return await super().run(*args, **kwargs)
88
97
 
98
+ def _get_rendered_ignored_paths(self) -> List[str]:
99
+ if isinstance(self._ignored_path, str):
100
+ return [self.render_str(self._ignored_path)]
101
+ return [
102
+ self.render_str(ignored_path)
103
+ for ignored_path in self._ignored_path
104
+ ]
105
+
89
106
  async def inspect(self, *args: Any, **kwargs: Any) -> bool:
90
107
  label = f'Checking {self._rendered_path}'
91
108
  try:
92
- if len(glob.glob(self._rendered_path, recursive=True)) > 0:
109
+ matches = get_file_names(
110
+ glob_path=self._rendered_path,
111
+ glob_ignored_paths=self._rendered_ignored_paths
112
+ )
113
+ if len(matches) > 0:
93
114
  self.print_out(f'{label} (Exist)')
94
115
  return True
95
116
  self.show_progress(f'{label} (Not Exist)')
zrb/task/path_watcher.py CHANGED
@@ -1,7 +1,8 @@
1
1
  from zrb.helper.typing import (
2
- Any, Callable, Iterable, Mapping, Optional, Union, TypeVar
2
+ Any, Callable, Iterable, List, Mapping, Optional, Union, TypeVar
3
3
  )
4
4
  from zrb.helper.typecheck import typechecked
5
+ from zrb.helper.file.match import get_file_names
5
6
  from zrb.task.checker import Checker
6
7
  from zrb.task.any_task import AnyTask
7
8
  from zrb.task.any_task_event_handler import (
@@ -12,7 +13,6 @@ from zrb.task_env.env_file import EnvFile
12
13
  from zrb.task_group.group import Group
13
14
  from zrb.task_input.any_input import AnyInput
14
15
 
15
- import glob
16
16
  import os
17
17
 
18
18
  TPathWatcher = TypeVar('TPathWatcher', bound='PathWatcher')
@@ -20,6 +20,16 @@ TPathWatcher = TypeVar('TPathWatcher', bound='PathWatcher')
20
20
 
21
21
  @typechecked
22
22
  class PathWatcher(Checker):
23
+ '''
24
+ PathWatcher will wait for any changes specified on path.
25
+
26
+ Once the changes detected, PathWatcher will be completed
27
+ and several xcom will be set:
28
+ - <task-name>.file
29
+ - <task-name>.new-file
30
+ - <task-name>.modified-file
31
+ - <task-name>.deleted-file
32
+ '''
23
33
 
24
34
  def __init__(
25
35
  self,
@@ -40,6 +50,7 @@ class PathWatcher(Checker):
40
50
  on_retry: Optional[OnRetry] = None,
41
51
  on_failed: Optional[OnFailed] = None,
42
52
  path: str = '',
53
+ ignored_path: Union[str, Iterable[str]] = [],
43
54
  checking_interval: Union[int, float] = 0.1,
44
55
  progress_interval: Union[int, float] = 30,
45
56
  watch_new_files: bool = True,
@@ -70,10 +81,12 @@ class PathWatcher(Checker):
70
81
  should_execute=should_execute,
71
82
  )
72
83
  self._path = path
84
+ self._ignored_path = ignored_path
73
85
  self._watch_new_files = watch_new_files
74
86
  self._watch_modified_files = watch_modified_files
75
87
  self._watch_deleted_files = watch_deleted_files
76
88
  self._rendered_path: str = ''
89
+ self._rendered_ignored_paths: List[str] = []
77
90
  self._init_times: Mapping[str, float] = {}
78
91
 
79
92
  def copy(self) -> TPathWatcher:
@@ -92,25 +105,43 @@ class PathWatcher(Checker):
92
105
 
93
106
  async def run(self, *args: Any, **kwargs: Any) -> bool:
94
107
  self._rendered_path = self.render_str(self._path)
108
+ self._rendered_ignored_paths = [
109
+ ignored_path
110
+ for ignored_path in self._get_rendered_ignored_paths()
111
+ if ignored_path != ''
112
+ ]
95
113
  self._init_times = self._get_mod_times()
96
114
  return await super().run(*args, **kwargs)
97
115
 
116
+ def _get_rendered_ignored_paths(self) -> List[str]:
117
+ if isinstance(self._ignored_path, str):
118
+ return [self.render_str(self._ignored_path)]
119
+ return [
120
+ self.render_str(ignored_path)
121
+ for ignored_path in self._ignored_path
122
+ ]
123
+
98
124
  async def inspect(self, *args: Any, **kwargs: Any) -> bool:
99
125
  label = f'Watching {self._rendered_path}'
100
126
  try:
101
127
  mod_times = self._get_mod_times()
102
- except Exception:
128
+ except Exception as e:
103
129
  self.show_progress(f'{label} Cannot inspect')
130
+ raise e
104
131
  # watch changes
105
132
  if self._watch_new_files:
106
133
  new_files = mod_times.keys() - self._init_times.keys()
107
134
  for file in new_files:
108
135
  self.print_out_dark(f'{label} [+] New file detected: {file}')
136
+ self.set_task_xcom('new-file', file)
137
+ self.set_task_xcom('file', file)
109
138
  return True
110
139
  if self._watch_deleted_files:
111
140
  deleted_files = self._init_times.keys() - mod_times.keys()
112
141
  for file in deleted_files:
113
142
  self.print_out_dark(f'{label} [-] File deleted: {file}')
143
+ self.set_task_xcom('deleted-file', file)
144
+ self.set_task_xcom('file', file)
114
145
  return True
115
146
  if self._watch_modified_files:
116
147
  modified_files = {
@@ -119,12 +150,22 @@ class PathWatcher(Checker):
119
150
  }
120
151
  for file in modified_files:
121
152
  self.print_out_dark(f'{label} [/] File modified: {file}')
153
+ self.set_task_xcom('modified-file', file)
154
+ self.set_task_xcom('file', file)
122
155
  return True
123
156
  self.show_progress(f'{label} (Nothing changed)')
124
157
  return False
125
158
 
126
159
  def _get_mod_times(self) -> Mapping[str, float]:
127
- return {
128
- file_name: os.stat(file_name).st_mtime
129
- for file_name in glob.glob(self._rendered_path, recursive=True)
130
- }
160
+ matches = get_file_names(
161
+ glob_path=self._rendered_path,
162
+ glob_ignored_paths=self._rendered_ignored_paths
163
+ )
164
+ mod_times: Mapping[str, float] = {}
165
+ for file_name in matches:
166
+ try:
167
+ mod_time = os.stat(file_name).st_mtime
168
+ mod_times[file_name] = mod_time
169
+ except Exception as e:
170
+ self.print_err(e)
171
+ return mod_times