meerschaum 2.4.0.dev1__py3-none-any.whl → 2.4.0rc2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- meerschaum/_internal/arguments/_parse_arguments.py +2 -5
- meerschaum/_internal/docs/index.py +3 -2
- meerschaum/_internal/entry.py +13 -7
- meerschaum/_internal/shell/Shell.py +38 -44
- meerschaum/_internal/term/TermPageHandler.py +2 -3
- meerschaum/_internal/term/__init__.py +13 -11
- meerschaum/actions/api.py +10 -7
- meerschaum/actions/bootstrap.py +2 -1
- meerschaum/actions/delete.py +4 -1
- meerschaum/actions/register.py +1 -3
- meerschaum/actions/stack.py +24 -19
- meerschaum/actions/start.py +25 -26
- meerschaum/actions/sync.py +53 -52
- meerschaum/api/__init__.py +48 -14
- meerschaum/api/_events.py +15 -10
- meerschaum/api/_oauth2.py +2 -2
- meerschaum/api/_websockets.py +5 -4
- meerschaum/api/dash/__init__.py +1 -11
- meerschaum/api/dash/callbacks/dashboard.py +47 -55
- meerschaum/api/dash/callbacks/jobs.py +15 -16
- meerschaum/api/dash/callbacks/login.py +16 -10
- meerschaum/api/dash/callbacks/pipes.py +3 -4
- meerschaum/api/dash/callbacks/plugins.py +1 -1
- meerschaum/api/dash/callbacks/register.py +15 -11
- meerschaum/api/dash/components.py +54 -59
- meerschaum/api/dash/jobs.py +5 -9
- meerschaum/api/dash/pages/pipes.py +4 -1
- meerschaum/api/dash/pipes.py +13 -17
- meerschaum/api/dash/plugins.py +6 -4
- meerschaum/api/dash/sessions.py +176 -0
- meerschaum/api/dash/users.py +2 -53
- meerschaum/api/dash/webterm.py +12 -17
- meerschaum/api/resources/static/js/terminado.js +1 -1
- meerschaum/api/routes/_actions.py +4 -20
- meerschaum/api/routes/_jobs.py +8 -7
- meerschaum/api/routes/_webterm.py +5 -6
- meerschaum/config/_default.py +6 -1
- meerschaum/config/_version.py +1 -1
- meerschaum/config/stack/__init__.py +9 -7
- meerschaum/config/static/__init__.py +4 -0
- meerschaum/connectors/__init__.py +15 -9
- meerschaum/connectors/api/{APIConnector.py → _APIConnector.py} +3 -1
- meerschaum/connectors/api/__init__.py +2 -1
- meerschaum/connectors/parse.py +18 -16
- meerschaum/connectors/sql/__init__.py +3 -1
- meerschaum/connectors/sql/_pipes.py +39 -39
- meerschaum/connectors/valkey/{ValkeyConnector.py → _ValkeyConnector.py} +5 -5
- meerschaum/connectors/valkey/__init__.py +3 -1
- meerschaum/connectors/valkey/_pipes.py +13 -8
- meerschaum/core/Pipe/_data.py +155 -100
- meerschaum/jobs/_Job.py +1 -6
- meerschaum/jobs/__init__.py +7 -2
- meerschaum/utils/dataframe.py +4 -1
- meerschaum/utils/formatting/_shell.py +5 -6
- meerschaum/utils/packages/__init__.py +14 -9
- {meerschaum-2.4.0.dev1.dist-info → meerschaum-2.4.0rc2.dist-info}/METADATA +1 -1
- {meerschaum-2.4.0.dev1.dist-info → meerschaum-2.4.0rc2.dist-info}/RECORD +65 -65
- {meerschaum-2.4.0.dev1.dist-info → meerschaum-2.4.0rc2.dist-info}/WHEEL +1 -1
- meerschaum/api/dash/actions.py +0 -255
- /meerschaum/connectors/{Connector.py → _Connector.py} +0 -0
- /meerschaum/connectors/sql/{SQLConnector.py → _SQLConnector.py} +0 -0
- {meerschaum-2.4.0.dev1.dist-info → meerschaum-2.4.0rc2.dist-info}/LICENSE +0 -0
- {meerschaum-2.4.0.dev1.dist-info → meerschaum-2.4.0rc2.dist-info}/NOTICE +0 -0
- {meerschaum-2.4.0.dev1.dist-info → meerschaum-2.4.0rc2.dist-info}/entry_points.txt +0 -0
- {meerschaum-2.4.0.dev1.dist-info → meerschaum-2.4.0rc2.dist-info}/top_level.txt +0 -0
- {meerschaum-2.4.0.dev1.dist-info → meerschaum-2.4.0rc2.dist-info}/zip-safe +0 -0
@@ -18,6 +18,7 @@ _locks = {
|
|
18
18
|
}
|
19
19
|
_loaded_plugins_args: bool = False
|
20
20
|
|
21
|
+
|
21
22
|
def split_pipeline_sysargs(sysargs: List[str]) -> Tuple[List[str], List[str]]:
|
22
23
|
"""
|
23
24
|
Split `sysargs` into the main pipeline and the flags following the pipeline separator (`:`).
|
@@ -89,7 +90,6 @@ def parse_arguments(sysargs: List[str]) -> Dict[str, Any]:
|
|
89
90
|
|
90
91
|
"""
|
91
92
|
import shlex
|
92
|
-
import copy
|
93
93
|
from meerschaum.config.static import STATIC_CONFIG
|
94
94
|
from meerschaum._internal.arguments._parser import parser
|
95
95
|
|
@@ -261,8 +261,6 @@ def parse_dict_to_sysargs(
|
|
261
261
|
"""Revert an arguments dictionary back to a command line list."""
|
262
262
|
import shlex
|
263
263
|
from meerschaum._internal.arguments._parser import get_arguments_triggers
|
264
|
-
from meerschaum.config.static import STATIC_CONFIG
|
265
|
-
from meerschaum.utils.warnings import warn
|
266
264
|
|
267
265
|
action = args_dict.get('action', None)
|
268
266
|
sysargs: List[str] = []
|
@@ -284,7 +282,6 @@ def parse_dict_to_sysargs(
|
|
284
282
|
if isinstance(args_dict[a], (list, tuple)):
|
285
283
|
if len(args_dict[a]) > 0:
|
286
284
|
if a == 'sub_args' and args_dict[a] != ['']:
|
287
|
-
print(f"{args_dict[a]=}")
|
288
285
|
sysargs.extend(
|
289
286
|
[
|
290
287
|
'-A',
|
@@ -386,7 +383,7 @@ def remove_leading_action(
|
|
386
383
|
if not action_str.replace(UNDERSCORE_STANDIN, '_').startswith(action_name):
|
387
384
|
warn(f"Unable to parse '{action_str}' for action '{action_name}'.")
|
388
385
|
return action
|
389
|
-
|
386
|
+
|
390
387
|
parsed_action = action_str[len(action_name)+1:].split('_')
|
391
388
|
|
392
389
|
### Substitute the underscores back in.
|
@@ -436,8 +436,9 @@ def init_dash(dash_app):
|
|
436
436
|
- `meerschaum.connectors.is_connected()`
|
437
437
|
- `meerschaum.connectors.poll.retry_connect()`
|
438
438
|
- `meerschaum.connectors.Connector`
|
439
|
-
- `meerschaum.connectors.SQLConnector`
|
440
|
-
- `meerschaum.connectors.APIConnector`
|
439
|
+
- `meerschaum.connectors.sql.SQLConnector`
|
440
|
+
- `meerschaum.connectors.api.APIConnector`
|
441
|
+
- `meerschaum.connectors.valkey.ValkeyConnector`
|
441
442
|
|
442
443
|
</details>
|
443
444
|
|
meerschaum/_internal/entry.py
CHANGED
@@ -199,13 +199,13 @@ def entry_with_args(
|
|
199
199
|
"""Execute a Meerschaum action with keyword arguments.
|
200
200
|
Use `_entry()` for parsing sysargs before executing.
|
201
201
|
"""
|
202
|
-
import sys
|
203
202
|
import functools
|
204
203
|
import inspect
|
205
|
-
from meerschaum.actions import get_action
|
204
|
+
from meerschaum.actions import get_action
|
206
205
|
from meerschaum._internal.arguments import remove_leading_action
|
207
206
|
from meerschaum.utils.venv import active_venvs, deactivate_venv
|
208
207
|
from meerschaum.config.static import STATIC_CONFIG
|
208
|
+
from meerschaum.utils.typing import is_success_tuple
|
209
209
|
|
210
210
|
if _patch_args:
|
211
211
|
kw.update(_patch_args)
|
@@ -225,7 +225,8 @@ def entry_with_args(
|
|
225
225
|
or
|
226
226
|
(kw['action'][0] == 'mrsm' and len(kw['action'][1:]) == 0)
|
227
227
|
):
|
228
|
-
|
228
|
+
_ = get_shell(**kw).cmdloop()
|
229
|
+
return True, "Success"
|
229
230
|
|
230
231
|
skip_schedule = False
|
231
232
|
|
@@ -296,12 +297,15 @@ def entry_with_args(
|
|
296
297
|
for venv in [venv for venv in active_venvs]:
|
297
298
|
deactivate_venv(venv, debug=kw.get('debug', False), force=True)
|
298
299
|
|
300
|
+
if not is_success_tuple(result):
|
301
|
+
return True, str(result)
|
302
|
+
|
299
303
|
return result
|
300
304
|
|
301
305
|
|
302
306
|
def _do_action_wrapper(action_function, plugin_name, **kw):
|
303
307
|
from meerschaum.plugins import Plugin
|
304
|
-
from meerschaum.utils.venv import Venv
|
308
|
+
from meerschaum.utils.venv import Venv
|
305
309
|
from meerschaum.utils.misc import filter_keywords
|
306
310
|
plugin = Plugin(plugin_name) if plugin_name else None
|
307
311
|
with Venv(plugin, debug=kw.get('debug', False)):
|
@@ -330,7 +334,9 @@ _shell = None
|
|
330
334
|
def get_shell(
|
331
335
|
sysargs: Optional[List[str]] = None,
|
332
336
|
reload: bool = False,
|
333
|
-
debug: bool = False
|
337
|
+
debug: bool = False,
|
338
|
+
mrsm_instance: Optional[str] = None,
|
339
|
+
**kwargs: Any
|
334
340
|
):
|
335
341
|
"""Initialize and return the Meerschaum shell object."""
|
336
342
|
global _shell
|
@@ -346,9 +352,9 @@ def get_shell(
|
|
346
352
|
|
347
353
|
if _shell is None:
|
348
354
|
shell_pkg._insert_shell_actions()
|
349
|
-
_shell = shell_pkg.Shell(actions, sysargs=sysargs)
|
355
|
+
_shell = shell_pkg.Shell(actions, sysargs=sysargs, instance_keys=mrsm_instance)
|
350
356
|
elif reload:
|
351
|
-
_shell.__init__()
|
357
|
+
_shell.__init__(instance_keys=mrsm_instance)
|
352
358
|
|
353
359
|
_shells.append(_shell)
|
354
360
|
return _shell
|
@@ -81,10 +81,10 @@ PIPELINE_KEY: str = STATIC_CONFIG['system']['arguments']['pipeline_key']
|
|
81
81
|
ESCAPED_PIPELINE_KEY: str = STATIC_CONFIG['system']['arguments']['escaped_pipeline_key']
|
82
82
|
|
83
83
|
def _insert_shell_actions(
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
84
|
+
_shell: Optional['Shell'] = None,
|
85
|
+
actions: Optional[Dict[str, Callable[[Any], SuccessTuple]]] = None,
|
86
|
+
keep_self: bool = False,
|
87
|
+
) -> None:
|
88
88
|
"""
|
89
89
|
Update the Shell with Meerschaum actions.
|
90
90
|
"""
|
@@ -111,9 +111,10 @@ def _insert_shell_actions(
|
|
111
111
|
completer = _completer_wrapper(_completer)
|
112
112
|
setattr(_shell_class, 'complete_' + a, completer)
|
113
113
|
|
114
|
+
|
114
115
|
def _completer_wrapper(
|
115
116
|
target: Callable[[Any], List[str]]
|
116
|
-
) -> Callable[['
|
117
|
+
) -> Callable[['mrsm._internal.shell.Shell', str, str, int, int], Any]:
|
117
118
|
"""
|
118
119
|
Wrapper for `complete_` functions so they can instead use Meerschaum arguments.
|
119
120
|
"""
|
@@ -162,21 +163,21 @@ def default_action_completer(
|
|
162
163
|
possibilities.append(sa)
|
163
164
|
return sorted(possibilities)
|
164
165
|
|
166
|
+
|
165
167
|
def _check_complete_keys(line: str) -> Optional[List[str]]:
|
166
|
-
from meerschaum._internal.arguments._parser import
|
168
|
+
from meerschaum._internal.arguments._parser import get_arguments_triggers
|
167
169
|
|
168
170
|
### TODO Add all triggers
|
169
171
|
trigger_args = {
|
170
|
-
'-c'
|
171
|
-
'--connector-keys'
|
172
|
-
'-r'
|
173
|
-
'--repository'
|
174
|
-
'-i'
|
175
|
-
'--instance'
|
176
|
-
'--mrsm-instance'
|
172
|
+
'-c': 'connector_keys',
|
173
|
+
'--connector-keys': 'connector_keys',
|
174
|
+
'-r': 'repository',
|
175
|
+
'--repository': 'repository',
|
176
|
+
'-i': 'mrsm_instance',
|
177
|
+
'--instance': 'mrsm_instance',
|
178
|
+
'--mrsm-instance': 'mrsm_instance',
|
177
179
|
}
|
178
180
|
|
179
|
-
|
180
181
|
### TODO Find out arg possibilities
|
181
182
|
possibilities = []
|
182
183
|
last_word = line.rstrip(' ').split(' ')[-1]
|
@@ -236,10 +237,14 @@ def get_shell_intro(with_color: bool = True) -> str:
|
|
236
237
|
|
237
238
|
|
238
239
|
class Shell(cmd.Cmd):
|
240
|
+
"""
|
241
|
+
The interactive Meerschaum shell.
|
242
|
+
"""
|
239
243
|
def __init__(
|
240
244
|
self,
|
241
245
|
actions: Optional[Dict[str, Any]] = None,
|
242
|
-
sysargs: Optional[List[str]] = None
|
246
|
+
sysargs: Optional[List[str]] = None,
|
247
|
+
instance_keys: Optional[str] = None,
|
243
248
|
):
|
244
249
|
"""
|
245
250
|
Customize the CLI from configuration
|
@@ -268,26 +273,13 @@ class Shell(cmd.Cmd):
|
|
268
273
|
reserve_space_for_menu=False,
|
269
274
|
)
|
270
275
|
|
271
|
-
|
272
|
-
super().__init__(
|
273
|
-
allow_cli_args = False,
|
274
|
-
auto_load_commands = False,
|
275
|
-
persistent_history_length = 1000,
|
276
|
-
persistent_history_file = None,
|
277
|
-
)
|
278
|
-
_init = True
|
279
|
-
except Exception as e:
|
280
|
-
### fall back to default init (cmd)
|
281
|
-
_init = False
|
282
|
-
|
283
|
-
if not _init:
|
284
|
-
super().__init__()
|
276
|
+
super().__init__()
|
285
277
|
|
286
278
|
### remove default commands from the Cmd class
|
287
279
|
for command in commands_to_remove:
|
288
280
|
try:
|
289
281
|
delattr(cmd.Cmd, f'do_{command}')
|
290
|
-
except Exception
|
282
|
+
except Exception:
|
291
283
|
pass
|
292
284
|
|
293
285
|
### NOTE: custom actions must be added to the self._actions dictionary
|
@@ -301,7 +293,7 @@ class Shell(cmd.Cmd):
|
|
301
293
|
shell_attrs['_old_bottom_toolbar'] = ''
|
302
294
|
shell_attrs['debug'] = False
|
303
295
|
shell_attrs['_reload'] = True
|
304
|
-
self.load_config()
|
296
|
+
self.load_config(instance=instance_keys)
|
305
297
|
self.hidden_commands = []
|
306
298
|
### update hidden commands list (cmd2 only)
|
307
299
|
try:
|
@@ -314,7 +306,6 @@ class Shell(cmd.Cmd):
|
|
314
306
|
from meerschaum._internal.shell.updates import run_version_check_thread
|
315
307
|
self._update_thread = run_version_check_thread(debug=shell_attrs.get('debug', False))
|
316
308
|
|
317
|
-
|
318
309
|
def load_config(self, instance: Optional[str] = None):
|
319
310
|
"""
|
320
311
|
Set attributes from the shell configuration.
|
@@ -478,13 +469,13 @@ class Shell(cmd.Cmd):
|
|
478
469
|
"""
|
479
470
|
Pass line string to parent actions.
|
480
471
|
Pass parsed arguments to custom actions
|
481
|
-
|
472
|
+
|
482
473
|
Overrides `default`. If an action does not exist, assume the action is `shell`
|
483
474
|
"""
|
484
475
|
### Preserve the working directory.
|
485
476
|
old_cwd = os.getcwd()
|
486
477
|
|
487
|
-
from meerschaum._internal.entry import
|
478
|
+
from meerschaum._internal.entry import get_shell
|
488
479
|
self = get_shell(sysargs=shell_attrs['_sysargs'], debug=shell_attrs.get('debug', False))
|
489
480
|
|
490
481
|
### make a backup of line for later
|
@@ -522,7 +513,6 @@ class Shell(cmd.Cmd):
|
|
522
513
|
if line.startswith(help_token):
|
523
514
|
return "help " + line[len(help_token):]
|
524
515
|
|
525
|
-
from meerschaum._internal.arguments import parse_line
|
526
516
|
try:
|
527
517
|
sysargs = shlex.split(line)
|
528
518
|
except ValueError as e:
|
@@ -628,18 +618,24 @@ class Shell(cmd.Cmd):
|
|
628
618
|
for i, kwargs in enumerate([_ for _ in chained_kwargs]):
|
629
619
|
kwargs.update(patches[i])
|
630
620
|
|
631
|
-
from meerschaum._internal.entry import
|
621
|
+
from meerschaum._internal.entry import entry
|
632
622
|
sysargs_to_execute = []
|
633
623
|
for i, kwargs in enumerate(chained_kwargs):
|
634
624
|
step_kwargs = {k: v for k, v in kwargs.items() if k != 'line'}
|
635
|
-
|
625
|
+
step_action = kwargs.get('action', None)
|
626
|
+
step_action_name = step_action[0] if step_action else None
|
627
|
+
### NOTE: For `stack`, revert argument parsing.
|
628
|
+
step_sysargs = (
|
629
|
+
parse_dict_to_sysargs(step_kwargs)
|
630
|
+
if step_action_name != 'stack'
|
631
|
+
else chained_sysargs[i]
|
632
|
+
)
|
636
633
|
sysargs_to_execute.extend(step_sysargs)
|
637
634
|
sysargs_to_execute.append(AND_KEY)
|
638
|
-
|
635
|
+
|
639
636
|
sysargs_to_execute = sysargs_to_execute[:-1] + (
|
640
637
|
([':'] + pipeline_args) if pipeline_args else []
|
641
638
|
)
|
642
|
-
|
643
639
|
try:
|
644
640
|
success_tuple = entry(sysargs_to_execute, _patch_args=patch_args)
|
645
641
|
except Exception as e:
|
@@ -763,7 +759,6 @@ class Shell(cmd.Cmd):
|
|
763
759
|
|
764
760
|
return True, "Success"
|
765
761
|
|
766
|
-
|
767
762
|
def complete_instance(
|
768
763
|
self,
|
769
764
|
text: str,
|
@@ -776,9 +771,10 @@ class Shell(cmd.Cmd):
|
|
776
771
|
from meerschaum.utils.misc import get_connector_labels
|
777
772
|
from meerschaum._internal.arguments._parse_arguments import parse_line
|
778
773
|
from meerschaum.connectors import instance_types, _load_builtin_custom_connectors
|
779
|
-
if
|
774
|
+
if not self.__dict__.get('_loaded_custom_connectors', None):
|
780
775
|
_load_builtin_custom_connectors()
|
781
|
-
|
776
|
+
self.__dict__['_loaded_custom_connectors'] = True
|
777
|
+
from meerschaum.jobs import executor_types
|
782
778
|
|
783
779
|
conn_types = instance_types if not _executor else executor_types
|
784
780
|
|
@@ -792,7 +788,6 @@ class Shell(cmd.Cmd):
|
|
792
788
|
_additional_options=_additional_options,
|
793
789
|
)
|
794
790
|
|
795
|
-
|
796
791
|
def do_repo(
|
797
792
|
self,
|
798
793
|
action: Optional[List[str]] = None,
|
@@ -992,7 +987,6 @@ class Shell(cmd.Cmd):
|
|
992
987
|
"""
|
993
988
|
Patch builtin cmdloop with my own input (defined below).
|
994
989
|
"""
|
995
|
-
import signal, os
|
996
990
|
cmd.__builtins__['input'] = input_with_sigint(
|
997
991
|
_old_input,
|
998
992
|
shell_attrs['session'],
|
@@ -12,10 +12,9 @@ tornado_web = attempt_import('tornado.web', lazy=False)
|
|
12
12
|
|
13
13
|
class TermPageHandler(tornado_web.RequestHandler):
|
14
14
|
def get(self):
|
15
|
-
from meerschaum.api import endpoints
|
16
15
|
return self.render(
|
17
16
|
"termpage.html",
|
18
|
-
static
|
19
|
-
ws_url_path
|
17
|
+
static=self.static_url,
|
18
|
+
ws_url_path="/websocket",
|
20
19
|
)
|
21
20
|
|
@@ -7,10 +7,8 @@ Build the web console virtual terminal using Tornado and xterm.js.
|
|
7
7
|
"""
|
8
8
|
|
9
9
|
from __future__ import annotations
|
10
|
-
|
11
|
-
import
|
12
|
-
import sys
|
13
|
-
from typing import List, Tuple
|
10
|
+
|
11
|
+
from typing import Optional, Tuple
|
14
12
|
from meerschaum.utils.packages import attempt_import
|
15
13
|
from meerschaum._internal.term.TermPageHandler import TermPageHandler
|
16
14
|
from meerschaum.config._paths import API_TEMPLATES_PATH, API_STATIC_PATH
|
@@ -20,10 +18,13 @@ tornado, tornado_ioloop, terminado = attempt_import(
|
|
20
18
|
'tornado', 'tornado.ioloop', 'terminado', lazy=False,
|
21
19
|
)
|
22
20
|
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
21
|
+
|
22
|
+
def get_webterm_app_and_manager(
|
23
|
+
instance_keys: Optional[str] = None,
|
24
|
+
) -> Tuple[
|
25
|
+
tornado.web.Application,
|
26
|
+
terminado.UniqueTermManager,
|
27
|
+
]:
|
27
28
|
"""
|
28
29
|
Construct the Tornado web app and term manager from the provided sysargs.
|
29
30
|
|
@@ -31,12 +32,13 @@ def get_webterm_app_and_manager() -> Tuple[
|
|
31
32
|
-------
|
32
33
|
A tuple of the Tornado web application and term manager.
|
33
34
|
"""
|
35
|
+
shell_kwargs_str = f"mrsm_instance='{instance_keys}'" if instance_keys else ""
|
34
36
|
commands = [
|
35
37
|
venv_executable(None),
|
36
38
|
'-c',
|
37
39
|
"import os; _ = os.environ.pop('COLUMNS', None); _ = os.environ.pop('LINES', None); "
|
38
40
|
"from meerschaum._internal.entry import get_shell; "
|
39
|
-
"get_shell(
|
41
|
+
f"get_shell({shell_kwargs_str}).cmdloop()"
|
40
42
|
]
|
41
43
|
|
42
44
|
term_manager = terminado.UniqueTermManager(shell_command=commands)
|
@@ -53,7 +55,7 @@ def get_webterm_app_and_manager() -> Tuple[
|
|
53
55
|
]
|
54
56
|
tornado_app = tornado.web.Application(
|
55
57
|
handlers,
|
56
|
-
static_path
|
57
|
-
template_path
|
58
|
+
static_path=API_STATIC_PATH,
|
59
|
+
template_path=API_TEMPLATES_PATH,
|
58
60
|
)
|
59
61
|
return tornado_app, term_manager
|
meerschaum/actions/api.py
CHANGED
@@ -167,7 +167,7 @@ def _api_start(
|
|
167
167
|
|
168
168
|
### Uvicorn must be installed on the host because of multiprocessing reasons.
|
169
169
|
### `check_update` must be False, because otherwise Uvicorn's hidden imports will break things.
|
170
|
-
|
170
|
+
_ = attempt_import('dotenv', lazy=False)
|
171
171
|
uvicorn, gunicorn = attempt_import(
|
172
172
|
'uvicorn', 'gunicorn', venv=None, lazy=False, check_update=False,
|
173
173
|
)
|
@@ -235,6 +235,7 @@ def _api_start(
|
|
235
235
|
'no_dash': no_dash,
|
236
236
|
'no_auth': no_auth,
|
237
237
|
'private': private,
|
238
|
+
'production': production,
|
238
239
|
})
|
239
240
|
if debug:
|
240
241
|
uvicorn_config['reload'] = debug
|
@@ -251,7 +252,7 @@ def _api_start(
|
|
251
252
|
if secure:
|
252
253
|
cf['system']['api']['permissions']['actions']['non_admin'] = False
|
253
254
|
|
254
|
-
custom_keys = ['mrsm_instance', 'no_dash', 'no_auth', 'private', 'debug']
|
255
|
+
custom_keys = ['mrsm_instance', 'no_dash', 'no_auth', 'private', 'debug', 'production']
|
255
256
|
|
256
257
|
### write config to a temporary file to communicate with uvicorn threads
|
257
258
|
import json, sys
|
@@ -342,7 +343,7 @@ def _api_start(
|
|
342
343
|
run_python_package(
|
343
344
|
'gunicorn',
|
344
345
|
gunicorn_args,
|
345
|
-
env
|
346
|
+
env={
|
346
347
|
k: (
|
347
348
|
json.dumps(v)
|
348
349
|
if isinstance(v, (dict, list))
|
@@ -350,14 +351,16 @@ def _api_start(
|
|
350
351
|
)
|
351
352
|
for k, v in env_dict.items()
|
352
353
|
},
|
353
|
-
venv
|
354
|
-
debug
|
354
|
+
venv=None,
|
355
|
+
debug=debug,
|
355
356
|
)
|
356
357
|
except KeyboardInterrupt:
|
357
358
|
pass
|
358
359
|
|
359
|
-
|
360
|
-
|
360
|
+
if production:
|
361
|
+
_run_gunicorn()
|
362
|
+
else:
|
363
|
+
_run_uvicorn()
|
361
364
|
|
362
365
|
### Cleanup
|
363
366
|
if uvicorn_config_path.parent.exists():
|
meerschaum/actions/bootstrap.py
CHANGED
@@ -301,7 +301,7 @@ def _bootstrap_connectors(
|
|
301
301
|
)
|
302
302
|
if not overwrite and not force:
|
303
303
|
return False, "No changes made to connector configuration."
|
304
|
-
|
304
|
+
break
|
305
305
|
elif _label == "":
|
306
306
|
warn("Please enter a label.", stack=False)
|
307
307
|
else:
|
@@ -336,6 +336,7 @@ def _bootstrap_connectors(
|
|
336
336
|
return abort_tuple
|
337
337
|
new_attributes['flavor'] = flavor
|
338
338
|
required = sorted(list(connector_attributes[_type]['flavors'][flavor]['requirements']))
|
339
|
+
required = sorted(list(connector_attributes[_type]['flavors'][flavor]['optional']))
|
339
340
|
default = type_attributes['flavors'][flavor].get('defaults', {})
|
340
341
|
else:
|
341
342
|
required = sorted(list(type_attributes.get('required', {})))
|
meerschaum/actions/delete.py
CHANGED
@@ -521,7 +521,10 @@ def _complete_delete_jobs(
|
|
521
521
|
)
|
522
522
|
)
|
523
523
|
|
524
|
-
if
|
524
|
+
if (
|
525
|
+
executor_keys != 'systemd'
|
526
|
+
and parse_executor_keys(executor_keys, construct=False) is None
|
527
|
+
):
|
525
528
|
return []
|
526
529
|
|
527
530
|
jobs = get_jobs(executor_keys, include_hidden=False)
|
meerschaum/actions/register.py
CHANGED
@@ -118,7 +118,6 @@ def _register_pipes(
|
|
118
118
|
)
|
119
119
|
|
120
120
|
success, message = True, "Success"
|
121
|
-
failed_message = ""
|
122
121
|
failed_pipes = []
|
123
122
|
success_pipes = []
|
124
123
|
for p in pipes:
|
@@ -127,7 +126,6 @@ def _register_pipes(
|
|
127
126
|
ss, msg = p.register(debug=debug)
|
128
127
|
if not ss:
|
129
128
|
warn(f"{msg}", stack=False)
|
130
|
-
success = False
|
131
129
|
failed_pipes.append(p)
|
132
130
|
else:
|
133
131
|
success_pipes.append(p)
|
@@ -356,7 +354,7 @@ def _register_users(
|
|
356
354
|
|
357
355
|
msg = (
|
358
356
|
f"Finished registering {succeeded + failed} users." + '\n' +
|
359
|
-
f"
|
357
|
+
f" ({succeeded} succeeded, {failed} failed)"
|
360
358
|
)
|
361
359
|
return succeeded > 0, msg
|
362
360
|
|
meerschaum/actions/stack.py
CHANGED
@@ -9,34 +9,32 @@ Functions for running the Docker Compose stack
|
|
9
9
|
from __future__ import annotations
|
10
10
|
from meerschaum.utils.typing import SuccessTuple, Any, List, Optional, Union
|
11
11
|
|
12
|
+
|
12
13
|
def stack(
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
14
|
+
action: Optional[List[str]] = None,
|
15
|
+
sysargs: Optional[List[str]] = None,
|
16
|
+
sub_args: Optional[List[str]] = None,
|
17
|
+
yes: bool = False,
|
18
|
+
noask: bool = False,
|
19
|
+
force: bool = False,
|
20
|
+
debug: bool = False,
|
21
|
+
_capture_output: bool = False,
|
22
|
+
**kw: Any
|
23
|
+
) -> Union[SuccessTuple, 'subprocess.Popen']:
|
23
24
|
"""
|
24
25
|
Control the Meerschaum stack with Docker Compose.
|
25
26
|
Usage: `stack {command}`
|
26
|
-
|
27
|
+
|
27
28
|
Command: action[0]: default 'up'
|
28
29
|
Docker Compose command to run. E.g. 'config' will print Docker Compose configuration
|
29
30
|
"""
|
30
31
|
import subprocess
|
31
|
-
import contextlib
|
32
|
-
import io
|
33
32
|
import os
|
34
33
|
import sys
|
35
34
|
import pathlib
|
36
35
|
import meerschaum.config.stack
|
37
36
|
from meerschaum.config.stack import NECESSARY_FILES, write_stack
|
38
37
|
from meerschaum.config._paths import STACK_COMPOSE_PATH
|
39
|
-
from meerschaum.utils.prompt import yes_no
|
40
38
|
import meerschaum.config
|
41
39
|
from meerschaum.config._patch import apply_patch_to_config
|
42
40
|
from meerschaum.utils.packages import (
|
@@ -47,9 +45,9 @@ def stack(
|
|
47
45
|
from meerschaum.config import get_config
|
48
46
|
from meerschaum.utils.debug import dprint
|
49
47
|
from meerschaum.utils.warnings import warn
|
50
|
-
from meerschaum.utils.formatting import ANSI
|
51
48
|
from meerschaum.utils.misc import is_docker_available
|
52
49
|
from meerschaum.config._read_config import search_and_substitute_config
|
50
|
+
from meerschaum.utils.prompt import yes_no
|
53
51
|
|
54
52
|
stack_env_dict = apply_patch_to_config(
|
55
53
|
os.environ.copy(),
|
@@ -81,7 +79,7 @@ def stack(
|
|
81
79
|
break
|
82
80
|
if bootstrap:
|
83
81
|
write_stack(debug=debug)
|
84
|
-
else:
|
82
|
+
else:
|
85
83
|
sync_files(['stack'])
|
86
84
|
|
87
85
|
### define project name when starting containers
|
@@ -91,7 +89,7 @@ def stack(
|
|
91
89
|
'stack', 'project_name', patch=True, substitute=True,
|
92
90
|
)
|
93
91
|
]
|
94
|
-
|
92
|
+
|
95
93
|
### Debug list used to include --log-level DEBUG, but the flag is not supported on Windows (?)
|
96
94
|
debug_list = []
|
97
95
|
|
@@ -102,7 +100,7 @@ def stack(
|
|
102
100
|
print(
|
103
101
|
"To start the Docker service, run `sudo systemctl start docker` or `sudo dockerd`.\n"
|
104
102
|
+ "On Windows or MacOS, make sure Docker Desktop is running.",
|
105
|
-
file
|
103
|
+
file=sys.stderr,
|
106
104
|
)
|
107
105
|
return False, "Failed to connect to the Docker engine."
|
108
106
|
|
@@ -115,7 +113,7 @@ def stack(
|
|
115
113
|
|
116
114
|
if not has_builtin_compose:
|
117
115
|
_compose_venv = 'mrsm'
|
118
|
-
|
116
|
+
_ = attempt_import('compose', lazy=False, venv=_compose_venv, debug=debug)
|
119
117
|
|
120
118
|
### If docker-compose is installed globally, don't use the `mrsm` venv.
|
121
119
|
if not venv_contains_package('compose', _compose_venv):
|
@@ -129,6 +127,13 @@ def stack(
|
|
129
127
|
if not pip_install('pyyaml', venv=_compose_venv, debug=debug):
|
130
128
|
warn(f"Unable to install `pyyaml` into venv '{_compose_venv}'.")
|
131
129
|
|
130
|
+
if 'down' in sysargs and '-v' in sysargs:
|
131
|
+
if not yes_no(
|
132
|
+
"Are you sure you want to drop volumes?\n This cannot be undone!",
|
133
|
+
default='n',
|
134
|
+
):
|
135
|
+
return False, "Nothing was dropped."
|
136
|
+
|
132
137
|
cmd_list = [
|
133
138
|
_arg
|
134
139
|
for _arg in (settings_list + sysargs[1:])
|