meerschaum 2.0.0rc7__py3-none-any.whl → 2.0.0rc8__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- meerschaum/actions/__init__.py +97 -48
- meerschaum/actions/bootstrap.py +1 -1
- meerschaum/actions/clear.py +1 -1
- meerschaum/actions/deduplicate.py +1 -1
- meerschaum/actions/delete.py +8 -7
- meerschaum/actions/drop.py +1 -10
- meerschaum/actions/edit.py +1 -1
- meerschaum/actions/install.py +1 -1
- meerschaum/actions/pause.py +1 -1
- meerschaum/actions/register.py +1 -1
- meerschaum/actions/setup.py +1 -1
- meerschaum/actions/show.py +1 -1
- meerschaum/actions/start.py +18 -7
- meerschaum/actions/stop.py +5 -4
- meerschaum/actions/sync.py +3 -1
- meerschaum/actions/uninstall.py +1 -1
- meerschaum/actions/upgrade.py +1 -1
- meerschaum/actions/verify.py +54 -3
- meerschaum/config/_formatting.py +26 -0
- meerschaum/config/_jobs.py +28 -5
- meerschaum/config/_paths.py +21 -5
- meerschaum/config/_version.py +1 -1
- meerschaum/connectors/api/_fetch.py +1 -1
- meerschaum/connectors/api/_pipes.py +6 -11
- meerschaum/connectors/sql/_fetch.py +29 -11
- meerschaum/core/Pipe/_deduplicate.py +39 -23
- meerschaum/core/Pipe/_dtypes.py +2 -1
- meerschaum/core/Pipe/_verify.py +59 -24
- meerschaum/plugins/__init__.py +3 -0
- meerschaum/utils/daemon/Daemon.py +108 -27
- meerschaum/utils/daemon/__init__.py +35 -1
- meerschaum/utils/formatting/__init__.py +144 -1
- meerschaum/utils/formatting/_pipes.py +28 -5
- meerschaum/utils/misc.py +183 -187
- meerschaum/utils/packages/__init__.py +1 -1
- meerschaum/utils/packages/_packages.py +1 -0
- {meerschaum-2.0.0rc7.dist-info → meerschaum-2.0.0rc8.dist-info}/METADATA +4 -1
- {meerschaum-2.0.0rc7.dist-info → meerschaum-2.0.0rc8.dist-info}/RECORD +44 -44
- {meerschaum-2.0.0rc7.dist-info → meerschaum-2.0.0rc8.dist-info}/LICENSE +0 -0
- {meerschaum-2.0.0rc7.dist-info → meerschaum-2.0.0rc8.dist-info}/NOTICE +0 -0
- {meerschaum-2.0.0rc7.dist-info → meerschaum-2.0.0rc8.dist-info}/WHEEL +0 -0
- {meerschaum-2.0.0rc7.dist-info → meerschaum-2.0.0rc8.dist-info}/entry_points.txt +0 -0
- {meerschaum-2.0.0rc7.dist-info → meerschaum-2.0.0rc8.dist-info}/top_level.txt +0 -0
- {meerschaum-2.0.0rc7.dist-info → meerschaum-2.0.0rc8.dist-info}/zip-safe +0 -0
meerschaum/actions/__init__.py
CHANGED
@@ -11,51 +11,6 @@ from meerschaum.utils.typing import Callable, Any, Optional, Union, List, Dict,
|
|
11
11
|
from meerschaum.utils.packages import get_modules_from_package
|
12
12
|
_custom_actions = []
|
13
13
|
|
14
|
-
### build __all__ from other .py files in this package
|
15
|
-
import sys
|
16
|
-
modules = get_modules_from_package(
|
17
|
-
sys.modules[__name__],
|
18
|
-
names = False,
|
19
|
-
)
|
20
|
-
__all__ = ['actions', 'get_subactions', 'get_action', 'get_main_action_name', 'get_completer']
|
21
|
-
|
22
|
-
### Build the actions dictionary by importing all
|
23
|
-
### functions that do not begin with '_' from all submodules.
|
24
|
-
from inspect import getmembers, isfunction
|
25
|
-
actions = {}
|
26
|
-
"This docstring will be replaced in __pdoc__ at the end of this file."
|
27
|
-
|
28
|
-
for module in modules:
|
29
|
-
### A couple important things happening here:
|
30
|
-
### 1. Find all functions in all modules in `actions` package
|
31
|
-
### (skip functions that begin with '_')
|
32
|
-
### 2. Add them as members to the Shell class
|
33
|
-
### - Original definition : meerschaum._internal.shell.Shell
|
34
|
-
### - New definition : meerschaum._internal.Shell
|
35
|
-
### 3. Populate the actions dictionary with function names and functions
|
36
|
-
###
|
37
|
-
### UPDATE:
|
38
|
-
### Shell modifications have been deferred to get_shell in order to improve lazy loading.
|
39
|
-
|
40
|
-
actions.update(
|
41
|
-
dict(
|
42
|
-
[
|
43
|
-
### __name__ and new function pointer
|
44
|
-
(ob[0], ob[1])
|
45
|
-
for ob in getmembers(module)
|
46
|
-
if isfunction(ob[1])
|
47
|
-
### check that the function belongs to the module
|
48
|
-
and ob[0] == module.__name__.replace('_', '').split('.')[-1]
|
49
|
-
### skip functions that start with '_'
|
50
|
-
and ob[0][0] != '_'
|
51
|
-
]
|
52
|
-
)
|
53
|
-
)
|
54
|
-
|
55
|
-
original_actions = actions.copy()
|
56
|
-
from meerschaum._internal.entry import entry
|
57
|
-
|
58
|
-
|
59
14
|
def get_subactions(
|
60
15
|
action: Union[str, List[str]],
|
61
16
|
_actions: Optional[Dict[str, Callable[[Any], Any]]] = None,
|
@@ -273,14 +228,108 @@ def choose_subaction(
|
|
273
228
|
return options[parsed_choice](**kw)
|
274
229
|
|
275
230
|
|
276
|
-
|
231
|
+
def _get_subaction_names(action: str, globs: dict = None) -> List[str]:
|
232
|
+
"""Use `meerschaum.actions.get_subactions()` instead."""
|
233
|
+
if globs is None:
|
234
|
+
import importlib
|
235
|
+
module = importlib.import_module(f'meerschaum.actions.{action}')
|
236
|
+
globs = vars(module)
|
237
|
+
subactions = []
|
238
|
+
for item in globs:
|
239
|
+
if f'_{action}' in item and 'complete' not in item.lstrip('_'):
|
240
|
+
subactions.append(globs[item])
|
241
|
+
return subactions
|
242
|
+
|
243
|
+
|
244
|
+
def choices_docstring(action: str, globs : Optional[Dict[str, Any]] = None) -> str:
|
245
|
+
"""
|
246
|
+
Append the an action's available options to the module docstring.
|
247
|
+
This function is to be placed at the bottom of each action module.
|
248
|
+
|
249
|
+
Parameters
|
250
|
+
----------
|
251
|
+
action: str
|
252
|
+
The name of the action module (e.g. 'install').
|
253
|
+
|
254
|
+
globs: Optional[Dict[str, Any]], default None
|
255
|
+
An optional dictionary of global variables.
|
256
|
+
|
257
|
+
Returns
|
258
|
+
-------
|
259
|
+
The generated docstring for the module.
|
260
|
+
|
261
|
+
Examples
|
262
|
+
--------
|
263
|
+
>>> from meerschaum.utils.misc import choices_docstring as _choices_docstring
|
264
|
+
>>> install.__doc__ += _choices_docstring('install')
|
265
|
+
|
266
|
+
"""
|
267
|
+
options_str = f"\n Options:\n `{action} "
|
268
|
+
subactions = _get_subaction_names(action, globs=globs)
|
269
|
+
options_str += "["
|
270
|
+
sa_names = []
|
271
|
+
for sa in subactions:
|
272
|
+
try:
|
273
|
+
sa_names.append(sa.__name__[len(f"_{action}") + 1:])
|
274
|
+
except Exception as e:
|
275
|
+
print(e)
|
276
|
+
return ""
|
277
|
+
for sa_name in sorted(sa_names):
|
278
|
+
options_str += f"{sa_name}, "
|
279
|
+
options_str = options_str[:-2] + "]`"
|
280
|
+
return options_str
|
281
|
+
|
282
|
+
### build __all__ from other .py files in this package
|
283
|
+
import sys
|
284
|
+
modules = get_modules_from_package(
|
285
|
+
sys.modules[__name__],
|
286
|
+
names = False,
|
287
|
+
)
|
288
|
+
__all__ = ['actions', 'get_subactions', 'get_action', 'get_main_action_name', 'get_completer']
|
289
|
+
|
290
|
+
### Build the actions dictionary by importing all
|
291
|
+
### functions that do not begin with '_' from all submodules.
|
292
|
+
from inspect import getmembers, isfunction
|
293
|
+
actions = {}
|
294
|
+
"This docstring will be replaced in __pdoc__ at the end of this file."
|
295
|
+
|
296
|
+
for module in modules:
|
297
|
+
### A couple important things happening here:
|
298
|
+
### 1. Find all functions in all modules in `actions` package
|
299
|
+
### (skip functions that begin with '_')
|
300
|
+
### 2. Add them as members to the Shell class
|
301
|
+
### - Original definition : meerschaum._internal.shell.Shell
|
302
|
+
### - New definition : meerschaum._internal.Shell
|
303
|
+
### 3. Populate the actions dictionary with function names and functions
|
304
|
+
###
|
305
|
+
### UPDATE:
|
306
|
+
### Shell modifications have been deferred to get_shell in order to improve lazy loading.
|
307
|
+
|
308
|
+
actions.update(
|
309
|
+
dict(
|
310
|
+
[
|
311
|
+
### __name__ and new function pointer
|
312
|
+
(ob[0], ob[1])
|
313
|
+
for ob in getmembers(module)
|
314
|
+
if isfunction(ob[1])
|
315
|
+
### check that the function belongs to the module
|
316
|
+
and ob[0] == module.__name__.replace('_', '').split('.')[-1]
|
317
|
+
### skip functions that start with '_'
|
318
|
+
and ob[0][0] != '_'
|
319
|
+
]
|
320
|
+
)
|
321
|
+
)
|
322
|
+
|
323
|
+
original_actions = actions.copy()
|
324
|
+
from meerschaum._internal.entry import entry, get_shell
|
277
325
|
import meerschaum.plugins
|
278
|
-
# meerschaum.plugins.load_plugins()
|
279
326
|
make_action = meerschaum.plugins.make_action
|
280
327
|
|
281
328
|
### Instruct pdoc to skip the `meerschaum.actions.plugins` subdirectory.
|
282
329
|
__pdoc__ = {
|
283
|
-
'plugins'
|
330
|
+
'plugins': False,
|
331
|
+
'arguments': True,
|
332
|
+
'shell': False,
|
284
333
|
'actions': (
|
285
334
|
"Access functions of the standard Meerschaum actions.\n\n"
|
286
335
|
+ "Visit the [actions reference page](https://meerschaum.io/reference/actions/) "
|
meerschaum/actions/bootstrap.py
CHANGED
@@ -416,5 +416,5 @@ def _bootstrap_config(
|
|
416
416
|
### NOTE: This must be the final statement of the module.
|
417
417
|
### Any subactions added below these lines will not
|
418
418
|
### be added to the `help` docstring.
|
419
|
-
from meerschaum.
|
419
|
+
from meerschaum.actions import choices_docstring as _choices_docstring
|
420
420
|
bootstrap.__doc__ += _choices_docstring('bootstrap')
|
meerschaum/actions/clear.py
CHANGED
@@ -141,5 +141,5 @@ def _ask_with_rowcounts(
|
|
141
141
|
### NOTE: This must be the final statement of the module.
|
142
142
|
### Any subactions added below these lines will not
|
143
143
|
### be added to the `help` docstring.
|
144
|
-
from meerschaum.
|
144
|
+
from meerschaum.actions import choices_docstring as _choices_docstring
|
145
145
|
clear.__doc__ += _choices_docstring('clear')
|
@@ -38,5 +38,5 @@ def _deduplicate_pipes(**kwargs) -> SuccessTuple:
|
|
38
38
|
### NOTE: This must be the final statement of the module.
|
39
39
|
### Any subactions added below these lines will not
|
40
40
|
### be added to the `help` docstring.
|
41
|
-
from meerschaum.
|
41
|
+
from meerschaum.actions import choices_docstring as _choices_docstring
|
42
42
|
deduplicate.__doc__ += _choices_docstring('deduplicate')
|
meerschaum/actions/delete.py
CHANGED
@@ -374,12 +374,12 @@ def _complete_delete_connectors(
|
|
374
374
|
return get_connector_labels(*types, search_term=search_term)
|
375
375
|
|
376
376
|
def _delete_jobs(
|
377
|
-
action
|
378
|
-
noask
|
379
|
-
nopretty
|
380
|
-
force
|
381
|
-
yes
|
382
|
-
debug
|
377
|
+
action: Optional[List[str]] = None,
|
378
|
+
noask: bool = False,
|
379
|
+
nopretty: bool = False,
|
380
|
+
force: bool = False,
|
381
|
+
yes: bool = False,
|
382
|
+
debug: bool = False,
|
383
383
|
**kw
|
384
384
|
) -> SuccessTuple:
|
385
385
|
"""
|
@@ -431,6 +431,7 @@ def _delete_jobs(
|
|
431
431
|
force = force,
|
432
432
|
noask = noask,
|
433
433
|
debug = debug,
|
434
|
+
**kw
|
434
435
|
)
|
435
436
|
### Ensure the running jobs are dead.
|
436
437
|
if get_running_daemons(daemons):
|
@@ -473,5 +474,5 @@ def _delete_jobs(
|
|
473
474
|
### NOTE: This must be the final statement of the module.
|
474
475
|
### Any subactions added below these lines will not
|
475
476
|
### be added to the `help` docstring.
|
476
|
-
from meerschaum.
|
477
|
+
from meerschaum.actions import choices_docstring as _choices_docstring
|
477
478
|
delete.__doc__ += _choices_docstring('delete')
|
meerschaum/actions/drop.py
CHANGED
@@ -18,7 +18,6 @@ def drop(
|
|
18
18
|
from meerschaum.actions import choose_subaction
|
19
19
|
options = {
|
20
20
|
'pipes' : _drop_pipes,
|
21
|
-
'tables' : _drop_tables,
|
22
21
|
}
|
23
22
|
return choose_subaction(action, options, **kw)
|
24
23
|
|
@@ -79,17 +78,9 @@ def _drop_pipes(
|
|
79
78
|
)
|
80
79
|
return successes > 0, msg
|
81
80
|
|
82
|
-
def _drop_tables(
|
83
|
-
action: Optional[List[str]] = None,
|
84
|
-
**kw: Any
|
85
|
-
) -> SuccessTuple:
|
86
|
-
"""
|
87
|
-
Drop SQL tables. WARNING: This is very dangerous!
|
88
|
-
"""
|
89
|
-
return False, "Not implemented"
|
90
81
|
|
91
82
|
### NOTE: This must be the final statement of the module.
|
92
83
|
### Any subactions added below these lines will not
|
93
84
|
### be added to the `help` docstring.
|
94
|
-
from meerschaum.
|
85
|
+
from meerschaum.actions import choices_docstring as _choices_docstring
|
95
86
|
drop.__doc__ += _choices_docstring('drop')
|
meerschaum/actions/edit.py
CHANGED
@@ -292,5 +292,5 @@ def _edit_users(
|
|
292
292
|
### NOTE: This must be the final statement of the module.
|
293
293
|
### Any subactions added below these lines will not
|
294
294
|
### be added to the `help` docstring.
|
295
|
-
from meerschaum.
|
295
|
+
from meerschaum.actions import choices_docstring as _choices_docstring
|
296
296
|
edit.__doc__ += _choices_docstring('edit')
|
meerschaum/actions/install.py
CHANGED
@@ -236,5 +236,5 @@ def _complete_install_required(*args, **kw) -> List[str]:
|
|
236
236
|
### NOTE: This must be the final statement of the module.
|
237
237
|
### Any subactions added below these lines will not
|
238
238
|
### be added to the `help` docstring.
|
239
|
-
from meerschaum.
|
239
|
+
from meerschaum.actions import choices_docstring as _choices_docstring
|
240
240
|
install.__doc__ += _choices_docstring('install')
|
meerschaum/actions/pause.py
CHANGED
@@ -124,5 +124,5 @@ def _pause_jobs(
|
|
124
124
|
### NOTE: This must be the final statement of the module.
|
125
125
|
### Any subactions added below these lines will not
|
126
126
|
### be added to the `help` docstring.
|
127
|
-
from meerschaum.
|
127
|
+
from meerschaum.actions import choices_docstring as _choices_docstring
|
128
128
|
pause.__doc__ += _choices_docstring('pause')
|
meerschaum/actions/register.py
CHANGED
@@ -336,5 +336,5 @@ def _register_users(
|
|
336
336
|
### NOTE: This must be the final statement of the module.
|
337
337
|
### Any subactions added below these lines will not
|
338
338
|
### be added to the `help` docstring.
|
339
|
-
from meerschaum.
|
339
|
+
from meerschaum.actions import choices_docstring as _choices_docstring
|
340
340
|
register.__doc__ += _choices_docstring('register')
|
meerschaum/actions/setup.py
CHANGED
@@ -108,5 +108,5 @@ def _complete_setup_plugins(*args, **kw) -> List[str]:
|
|
108
108
|
### NOTE: This must be the final statement of the module.
|
109
109
|
### Any subactions added below these lines will not
|
110
110
|
### be added to the `help` docstring.
|
111
|
-
from meerschaum.
|
111
|
+
from meerschaum.actions import choices_docstring as _choices_docstring
|
112
112
|
setup.__doc__ += _choices_docstring('setup')
|
meerschaum/actions/show.py
CHANGED
@@ -733,5 +733,5 @@ def _show_environment(
|
|
733
733
|
### NOTE: This must be the final statement of the module.
|
734
734
|
### Any subactions added below these lines will not
|
735
735
|
### be added to the `help` docstring.
|
736
|
-
from meerschaum.
|
736
|
+
from meerschaum.actions import choices_docstring as _choices_docstring
|
737
737
|
show.__doc__ += _choices_docstring('show')
|
meerschaum/actions/start.py
CHANGED
@@ -128,12 +128,16 @@ def _start_jobs(
|
|
128
128
|
daemon_ids = get_daemon_ids()
|
129
129
|
|
130
130
|
new_job = len(list(action)) > 0
|
131
|
-
_potential_jobs = {'known'
|
131
|
+
_potential_jobs = {'known': [], 'unknown': []}
|
132
132
|
|
133
133
|
if action:
|
134
134
|
for a in action:
|
135
|
-
_potential_jobs[(
|
136
|
-
|
135
|
+
_potential_jobs[(
|
136
|
+
'known'
|
137
|
+
if a in daemon_ids
|
138
|
+
else 'unknown'
|
139
|
+
)].append(a)
|
140
|
+
|
137
141
|
### Check if the job is named after an action.
|
138
142
|
if (
|
139
143
|
_potential_jobs['known']
|
@@ -144,7 +148,7 @@ def _start_jobs(
|
|
144
148
|
_potential_jobs['unknown'].insert(0, _potential_jobs['known'][0])
|
145
149
|
del _potential_jobs['known'][0]
|
146
150
|
|
147
|
-
### Only spawn a new job if we don't
|
151
|
+
### Only spawn a new job if we don't find any jobs.
|
148
152
|
new_job = (len(_potential_jobs['known']) == 0)
|
149
153
|
if not new_job and _potential_jobs['unknown']:
|
150
154
|
if not kw.get('nopretty', False):
|
@@ -171,7 +175,7 @@ def _start_jobs(
|
|
171
175
|
return False, msg
|
172
176
|
|
173
177
|
### No action provided but a --name was. Start job if possible.
|
174
|
-
### E.g. `start job --myjob`
|
178
|
+
### E.g. `start job --name myjob`
|
175
179
|
elif name is not None:
|
176
180
|
new_job = False
|
177
181
|
names = [name]
|
@@ -215,7 +219,10 @@ def _start_jobs(
|
|
215
219
|
try:
|
216
220
|
_daemon_sysargs = daemon.properties['target']['args'][0]
|
217
221
|
except KeyError:
|
218
|
-
return
|
222
|
+
return (
|
223
|
+
(False, f"Failed to get arguments for daemon '{daemon.daemon_id}'."),
|
224
|
+
daemon.daemon_id
|
225
|
+
)
|
219
226
|
_daemon_kw = parse_arguments(_daemon_sysargs)
|
220
227
|
_daemon_kw['name'] = daemon.daemon_id
|
221
228
|
_action_success_tuple = daemon_action(**_daemon_kw)
|
@@ -277,7 +284,11 @@ def _start_jobs(
|
|
277
284
|
|
278
285
|
_successes, _failures = [], []
|
279
286
|
for _name in names:
|
280
|
-
success_tuple, __name =
|
287
|
+
success_tuple, __name = (
|
288
|
+
_run_new_job(_name)
|
289
|
+
if new_job
|
290
|
+
else _run_existing_job(_name)
|
291
|
+
)
|
281
292
|
if not kw.get('nopretty', False):
|
282
293
|
print_tuple(success_tuple)
|
283
294
|
_successes.append(_name) if success_tuple[0] else _failures.append(_name)
|
meerschaum/actions/stop.py
CHANGED
@@ -50,6 +50,7 @@ def _complete_stop(
|
|
50
50
|
|
51
51
|
def _stop_jobs(
|
52
52
|
action: Optional[List[str]] = None,
|
53
|
+
timeout_seconds: Optional[int] = None,
|
53
54
|
noask: bool = False,
|
54
55
|
force: bool = False,
|
55
56
|
yes: bool = False,
|
@@ -93,18 +94,18 @@ def _stop_jobs(
|
|
93
94
|
|
94
95
|
_quit_daemons, _kill_daemons = [], []
|
95
96
|
for d in daemons_to_stop:
|
96
|
-
quit_success, quit_msg = d.quit()
|
97
|
+
quit_success, quit_msg = d.quit(timeout=timeout_seconds)
|
97
98
|
if quit_success:
|
98
99
|
_quit_daemons.append(d)
|
99
100
|
continue
|
100
101
|
else:
|
101
102
|
warn(
|
102
|
-
f"Failed to gracefully quit job '{d.daemon_id}', attempting to terminate:\n"
|
103
|
+
f"Failed to gracefully quit job '{d.daemon_id}', attempting to terminate:\n "
|
103
104
|
+ f"{quit_msg}",
|
104
105
|
stack = False,
|
105
106
|
)
|
106
107
|
|
107
|
-
kill_success, kill_msg = d.kill()
|
108
|
+
kill_success, kill_msg = d.kill(timeout=timeout_seconds)
|
108
109
|
if kill_success:
|
109
110
|
_kill_daemons.append(d)
|
110
111
|
continue
|
@@ -126,5 +127,5 @@ def _stop_jobs(
|
|
126
127
|
### NOTE: This must be the final statement of the module.
|
127
128
|
### Any subactions added below these lines will not
|
128
129
|
### be added to the `help` docstring.
|
129
|
-
from meerschaum.
|
130
|
+
from meerschaum.actions import choices_docstring as _choices_docstring
|
130
131
|
stop.__doc__ += _choices_docstring('stop')
|
meerschaum/actions/sync.py
CHANGED
@@ -154,6 +154,7 @@ def _pipes_lap(
|
|
154
154
|
) + msg + '\n'
|
155
155
|
print_tuple(
|
156
156
|
(success, msg),
|
157
|
+
calm = True,
|
157
158
|
_progress = _progress,
|
158
159
|
)
|
159
160
|
_checkpoint(_progress=_progress, _task=_task)
|
@@ -287,6 +288,7 @@ def _sync_pipes(
|
|
287
288
|
lap_begin = time.perf_counter()
|
288
289
|
|
289
290
|
try:
|
291
|
+
results_dict = {}
|
290
292
|
with cm:
|
291
293
|
results_dict = _pipes_lap(
|
292
294
|
min_seconds = min_seconds,
|
@@ -423,5 +425,5 @@ def _wrap_sync_pipe(
|
|
423
425
|
### NOTE: This must be the final statement of the module.
|
424
426
|
### Any subactions added below these lines will not
|
425
427
|
### be added to the `help` docstring.
|
426
|
-
from meerschaum.
|
428
|
+
from meerschaum.actions import choices_docstring as _choices_docstring
|
427
429
|
sync.__doc__ += _choices_docstring('sync')
|
meerschaum/actions/uninstall.py
CHANGED
@@ -192,5 +192,5 @@ def _uninstall_packages(
|
|
192
192
|
### NOTE: This must be the final statement of the module.
|
193
193
|
### Any subactions added below these lines will not
|
194
194
|
### be added to the `help` docstring.
|
195
|
-
from meerschaum.
|
195
|
+
from meerschaum.actions import choices_docstring as _choices_docstring
|
196
196
|
uninstall.__doc__ += _choices_docstring('uninstall')
|
meerschaum/actions/upgrade.py
CHANGED
@@ -213,5 +213,5 @@ def _upgrade_plugins(
|
|
213
213
|
### NOTE: This must be the final statement of the module.
|
214
214
|
### Any subactions added below these lines will not
|
215
215
|
### be added to the `help` docstring.
|
216
|
-
from meerschaum.
|
216
|
+
from meerschaum.actions import choices_docstring as _choices_docstring
|
217
217
|
upgrade.__doc__ += _choices_docstring('upgrade')
|
meerschaum/actions/verify.py
CHANGED
@@ -11,7 +11,7 @@ from meerschaum.utils.typing import Union, Any, Sequence, SuccessTuple, Optional
|
|
11
11
|
|
12
12
|
def verify(
|
13
13
|
action: Optional[List[str]] = None,
|
14
|
-
**
|
14
|
+
**kwargs: Any
|
15
15
|
) -> SuccessTuple:
|
16
16
|
"""
|
17
17
|
Verify the states of pipes, packages, and more.
|
@@ -21,8 +21,9 @@ def verify(
|
|
21
21
|
'pipes': _verify_pipes,
|
22
22
|
'packages': _verify_packages,
|
23
23
|
'venvs': _verify_venvs,
|
24
|
+
'plugins': _verify_plugins,
|
24
25
|
}
|
25
|
-
return choose_subaction(action, options, **
|
26
|
+
return choose_subaction(action, options, **kwargs)
|
26
27
|
|
27
28
|
|
28
29
|
def _verify_pipes(**kwargs) -> SuccessTuple:
|
@@ -92,8 +93,58 @@ def _verify_venvs(
|
|
92
93
|
return True, "Success"
|
93
94
|
|
94
95
|
|
96
|
+
def _verify_plugins(
|
97
|
+
action: Optional[List[str]] = None,
|
98
|
+
**kwargs: Any
|
99
|
+
) -> SuccessTuple:
|
100
|
+
"""
|
101
|
+
Verify that all of the available plugins are able to be imported as expected.
|
102
|
+
"""
|
103
|
+
from meerschaum.utils.formatting import print_options, UNICODE, print_tuple
|
104
|
+
from meerschaum.plugins import import_plugins, get_plugins_names, Plugin
|
105
|
+
from meerschaum.config import get_config
|
106
|
+
from meerschaum.utils.misc import items_str
|
107
|
+
|
108
|
+
plugins_names_to_verify = action or get_plugins_names()
|
109
|
+
if not plugins_names_to_verify:
|
110
|
+
if not action:
|
111
|
+
return True, "There are no installed plugins; nothing to do."
|
112
|
+
return False, f"Unable to verify plugins {items_str(action)}."
|
113
|
+
|
114
|
+
header = f"Verifying {len(plugins_names_to_verify)} plugins"
|
115
|
+
print_options(
|
116
|
+
plugins_names_to_verify,
|
117
|
+
header = header,
|
118
|
+
)
|
119
|
+
|
120
|
+
failed_to_import = []
|
121
|
+
for plugin_name in plugins_names_to_verify:
|
122
|
+
plugin = Plugin(plugin_name)
|
123
|
+
plugin_success = plugin.module is not None
|
124
|
+
if not plugin_success:
|
125
|
+
failed_to_import.append(plugin)
|
126
|
+
plugin_msg = (
|
127
|
+
(f"Imported plugin '{plugin}'.")
|
128
|
+
if plugin_success
|
129
|
+
else (f"Failed to import plugin '{plugin}'.")
|
130
|
+
)
|
131
|
+
print_tuple((plugin_success, plugin_msg), calm=True)
|
132
|
+
|
133
|
+
success = len(failed_to_import) == 0
|
134
|
+
message = (
|
135
|
+
f"Successfully imported {len(plugins_names_to_verify)} plugins."
|
136
|
+
if success
|
137
|
+
else (
|
138
|
+
f"Failed to import plugin"
|
139
|
+
+ ('s' if len(failed_to_import) != 1 else '')
|
140
|
+
+ f" {items_str(failed_to_import)}."
|
141
|
+
)
|
142
|
+
)
|
143
|
+
return success, message
|
144
|
+
|
145
|
+
|
95
146
|
### NOTE: This must be the final statement of the module.
|
96
147
|
### Any subactions added below these lines will not
|
97
148
|
### be added to the `help` docstring.
|
98
|
-
from meerschaum.
|
149
|
+
from meerschaum.actions import choices_docstring as _choices_docstring
|
99
150
|
verify.__doc__ += _choices_docstring('verify')
|
meerschaum/config/_formatting.py
CHANGED
@@ -115,6 +115,32 @@ default_formatting_config = {
|
|
115
115
|
},
|
116
116
|
},
|
117
117
|
},
|
118
|
+
'success_calm' : {
|
119
|
+
'unicode' : {
|
120
|
+
'icon' : 'MRSM{formatting:emoji:success_calm}',
|
121
|
+
},
|
122
|
+
'ascii' : {
|
123
|
+
'icon' : '+',
|
124
|
+
},
|
125
|
+
'ansi' : {
|
126
|
+
'rich' : {
|
127
|
+
'style' : 'pale_green3',
|
128
|
+
},
|
129
|
+
},
|
130
|
+
},
|
131
|
+
'failure_calm' : {
|
132
|
+
'unicode' : {
|
133
|
+
'icon' : 'MRSM{formatting:emoji:failure_calm}',
|
134
|
+
},
|
135
|
+
'ascii' : {
|
136
|
+
'icon' : 'x',
|
137
|
+
},
|
138
|
+
'ansi' : {
|
139
|
+
'rich' : {
|
140
|
+
'style' : 'indian red',
|
141
|
+
},
|
142
|
+
},
|
143
|
+
},
|
118
144
|
'errors' : {
|
119
145
|
'unicode' : {
|
120
146
|
'icon' : 'MRSM{formatting:emoji:error}',
|
meerschaum/config/_jobs.py
CHANGED
@@ -7,6 +7,8 @@ Default configuration for jobs.
|
|
7
7
|
"""
|
8
8
|
|
9
9
|
default_jobs_config = {
|
10
|
+
'timeout_seconds': 8,
|
11
|
+
'check_timeout_interval_seconds': 0.1,
|
10
12
|
'logs' : {
|
11
13
|
'num_files_to_keep': 5,
|
12
14
|
'max_file_size': 100_000,
|
@@ -14,11 +16,32 @@ default_jobs_config = {
|
|
14
16
|
'refresh_files_seconds': 5.0,
|
15
17
|
'min_buffer_len': 15,
|
16
18
|
'colors' : [
|
17
|
-
'cyan',
|
18
|
-
'
|
19
|
-
'
|
20
|
-
'
|
21
|
-
'
|
19
|
+
'cyan',
|
20
|
+
'magenta',
|
21
|
+
'orange3',
|
22
|
+
'green',
|
23
|
+
'blue',
|
24
|
+
'red',
|
25
|
+
'spring_green3',
|
26
|
+
'medium_purple3',
|
27
|
+
'medium_violet_red',
|
28
|
+
'slate_blue1',
|
29
|
+
'bright_red',
|
30
|
+
'steel_blue3',
|
31
|
+
'aquamarine1',
|
32
|
+
'dark_khaki',
|
33
|
+
'pink3',
|
34
|
+
'gold3',
|
35
|
+
'pale_green1',
|
36
|
+
'light coral',
|
37
|
+
'light_goldenrod2',
|
38
|
+
'cornsilk1',
|
39
|
+
'orange_red1',
|
40
|
+
'deep_pink1',
|
41
|
+
'aquamarine3',
|
42
|
+
'sky_blue2',
|
43
|
+
'tan',
|
44
|
+
'honeydew2',
|
22
45
|
],
|
23
46
|
},
|
24
47
|
}
|
meerschaum/config/_paths.py
CHANGED
@@ -48,22 +48,38 @@ if ENVIRONMENT_PLUGINS_DIR in os.environ:
|
|
48
48
|
Path(path).resolve()
|
49
49
|
for path in json.loads(os.environ[ENVIRONMENT_PLUGINS_DIR])
|
50
50
|
] if os.environ[ENVIRONMENT_PLUGINS_DIR].lstrip().startswith('[')
|
51
|
-
else [
|
51
|
+
else [
|
52
|
+
Path(path_str).resolve()
|
53
|
+
for path_str in os.environ[ENVIRONMENT_PLUGINS_DIR].split(':')
|
54
|
+
if path_str
|
55
|
+
]
|
52
56
|
)
|
53
57
|
except Exception as e:
|
54
58
|
PLUGINS_DIR_PATHS = []
|
55
59
|
|
56
60
|
if not PLUGINS_DIR_PATHS:
|
57
61
|
print(
|
58
|
-
"Invalid plugins directories set for "
|
59
|
-
f"environment variable '{ENVIRONMENT_PLUGINS_DIR}'.\n"
|
60
|
-
f"
|
61
|
-
|
62
|
+
"Invalid plugins directories set for "
|
63
|
+
f"environment variable '{ENVIRONMENT_PLUGINS_DIR}'.\n\n"
|
64
|
+
f"Set this to a colon-separated path string:\n\n"
|
65
|
+
f"`export {ENVIRONMENT_PLUGINS_DIR}=./plugins:/another/path/to/plugins`\n\n"
|
66
|
+
"or a JSON-encoded path list:\n\n"
|
67
|
+
f"`export {ENVIRONMENT_PLUGINS_DIR}=" + "'[\"./plugins\", \"/another/path/to/plugins\"]'`"
|
68
|
+
f"",
|
62
69
|
)
|
63
70
|
sys.exit(1)
|
64
71
|
else:
|
65
72
|
PLUGINS_DIR_PATHS = [_ROOT_DIR_PATH / 'plugins']
|
66
73
|
|
74
|
+
### Remove duplicate plugins paths.
|
75
|
+
_seen_plugins_paths, _plugins_paths_to_remove = set(), set()
|
76
|
+
for _plugin_path in PLUGINS_DIR_PATHS:
|
77
|
+
if _plugin_path in _seen_plugins_paths:
|
78
|
+
_plugins_paths_to_remove.add(_plugin_path)
|
79
|
+
_seen_plugins_paths.add(_plugin_path)
|
80
|
+
for _plugin_path in _plugins_paths_to_remove:
|
81
|
+
PLUGINS_DIR_PATHS.remove(_plugin_path)
|
82
|
+
|
67
83
|
|
68
84
|
paths = {
|
69
85
|
'PACKAGE_ROOT_PATH' : str(Path(__file__).parent.parent.resolve()),
|
meerschaum/config/_version.py
CHANGED
@@ -42,7 +42,7 @@ def fetch(
|
|
42
42
|
remote_metric_key = instructions.get('metric_key', None)
|
43
43
|
remote_location_key = instructions.get('location_key', None)
|
44
44
|
if begin is None:
|
45
|
-
begin = pipe.
|
45
|
+
begin = pipe.get_sync_time(debug=debug)
|
46
46
|
|
47
47
|
_params = copy.deepcopy(params) if params is not None else {}
|
48
48
|
_params = apply_patch_to_config(_params, instructions.get('params', {}))
|