meerschaum 2.4.0.dev1__py3-none-any.whl → 2.4.0rc2__py3-none-any.whl

meerschaum/_internal/arguments/_parse_arguments.py

@@ -18,6 +18,7 @@ _locks = {
 }
 _loaded_plugins_args: bool = False
 
+
 def split_pipeline_sysargs(sysargs: List[str]) -> Tuple[List[str], List[str]]:
     """
     Split `sysargs` into the main pipeline and the flags following the pipeline separator (`:`).
@@ -89,7 +90,6 @@ def parse_arguments(sysargs: List[str]) -> Dict[str, Any]:
 
     """
     import shlex
-    import copy
     from meerschaum.config.static import STATIC_CONFIG
     from meerschaum._internal.arguments._parser import parser
 
@@ -261,8 +261,6 @@ def parse_dict_to_sysargs(
     """Revert an arguments dictionary back to a command line list."""
     import shlex
     from meerschaum._internal.arguments._parser import get_arguments_triggers
-    from meerschaum.config.static import STATIC_CONFIG
-    from meerschaum.utils.warnings import warn
 
     action = args_dict.get('action', None)
     sysargs: List[str] = []
@@ -284,7 +282,6 @@ def parse_dict_to_sysargs(
             if isinstance(args_dict[a], (list, tuple)):
                 if len(args_dict[a]) > 0:
                     if a == 'sub_args' and args_dict[a] != ['']:
-                        print(f"{args_dict[a]=}")
                         sysargs.extend(
                             [
                                 '-A',
@@ -386,7 +383,7 @@ def remove_leading_action(
     if not action_str.replace(UNDERSCORE_STANDIN, '_').startswith(action_name):
         warn(f"Unable to parse '{action_str}' for action '{action_name}'.")
         return action
-    
+
     parsed_action = action_str[len(action_name)+1:].split('_')
 
     ### Substitute the underscores back in.

meerschaum/_internal/docs/index.py

@@ -436,8 +436,9 @@ def init_dash(dash_app):
   - `meerschaum.connectors.is_connected()`
   - `meerschaum.connectors.poll.retry_connect()`
   - `meerschaum.connectors.Connector`
-  - `meerschaum.connectors.SQLConnector`
-  - `meerschaum.connectors.APIConnector`
+  - `meerschaum.connectors.sql.SQLConnector`
+  - `meerschaum.connectors.api.APIConnector`
+  - `meerschaum.connectors.valkey.ValkeyConnector`
 
 </details>
 

meerschaum/_internal/entry.py

@@ -199,13 +199,13 @@ def entry_with_args(
     """Execute a Meerschaum action with keyword arguments.
     Use `_entry()` for parsing sysargs before executing.
     """
-    import sys
     import functools
     import inspect
-    from meerschaum.actions import get_action, get_main_action_name
+    from meerschaum.actions import get_action
     from meerschaum._internal.arguments import remove_leading_action
     from meerschaum.utils.venv import active_venvs, deactivate_venv
     from meerschaum.config.static import STATIC_CONFIG
+    from meerschaum.utils.typing import is_success_tuple
 
     if _patch_args:
         kw.update(_patch_args)
@@ -225,7 +225,8 @@ def entry_with_args(
         or
         (kw['action'][0] == 'mrsm' and len(kw['action'][1:]) == 0)
     ):
-        return get_shell().cmdloop()
+        _ = get_shell(**kw).cmdloop()
+        return True, "Success"
 
     skip_schedule = False
 
@@ -296,12 +297,15 @@ def entry_with_args(
     for venv in [venv for venv in active_venvs]:
         deactivate_venv(venv, debug=kw.get('debug', False), force=True)
 
+    if not is_success_tuple(result):
+        return True, str(result)
+
     return result
 
 
 def _do_action_wrapper(action_function, plugin_name, **kw):
     from meerschaum.plugins import Plugin
-    from meerschaum.utils.venv import Venv, active_venvs, deactivate_venv
+    from meerschaum.utils.venv import Venv
     from meerschaum.utils.misc import filter_keywords
     plugin = Plugin(plugin_name) if plugin_name else None
     with Venv(plugin, debug=kw.get('debug', False)):
@@ -330,7 +334,9 @@ _shell = None
 def get_shell(
     sysargs: Optional[List[str]] = None,
     reload: bool = False,
-    debug: bool = False
+    debug: bool = False,
+    mrsm_instance: Optional[str] = None,
+    **kwargs: Any
 ):
     """Initialize and return the Meerschaum shell object."""
     global _shell
@@ -346,9 +352,9 @@ def get_shell(
 
         if _shell is None:
             shell_pkg._insert_shell_actions()
-            _shell = shell_pkg.Shell(actions, sysargs=sysargs)
+            _shell = shell_pkg.Shell(actions, sysargs=sysargs, instance_keys=mrsm_instance)
         elif reload:
-            _shell.__init__()
+            _shell.__init__(instance_keys=mrsm_instance)
 
         _shells.append(_shell)
     return _shell

meerschaum/_internal/shell/Shell.py

@@ -81,10 +81,10 @@ PIPELINE_KEY: str = STATIC_CONFIG['system']['arguments']['pipeline_key']
 ESCAPED_PIPELINE_KEY: str = STATIC_CONFIG['system']['arguments']['escaped_pipeline_key']
 
 def _insert_shell_actions(
-        _shell: Optional['Shell'] = None,
-        actions: Optional[Dict[str, Callable[[Any], SuccessTuple]]] = None,
-        keep_self: bool = False,
-    ) -> None:
+    _shell: Optional['Shell'] = None,
+    actions: Optional[Dict[str, Callable[[Any], SuccessTuple]]] = None,
+    keep_self: bool = False,
+) -> None:
     """
     Update the Shell with Meerschaum actions.
     """
@@ -111,9 +111,10 @@ def _insert_shell_actions(
         completer = _completer_wrapper(_completer)
         setattr(_shell_class, 'complete_' + a, completer)
 
+
 def _completer_wrapper(
     target: Callable[[Any], List[str]]
-) -> Callable[['meerschaum._internal.shell.Shell', str, str, int, int], Any]:
+) -> Callable[['mrsm._internal.shell.Shell', str, str, int, int], Any]:
     """
     Wrapper for `complete_` functions so they can instead use Meerschaum arguments.
     """
@@ -162,21 +163,21 @@ def default_action_completer(
             possibilities.append(sa)
     return sorted(possibilities)
 
+
 def _check_complete_keys(line: str) -> Optional[List[str]]:
-    from meerschaum._internal.arguments._parser import parser, get_arguments_triggers
+    from meerschaum._internal.arguments._parser import get_arguments_triggers
 
     ### TODO Add all triggers
     trigger_args = {
-        '-c' : 'connector_keys',
-        '--connector-keys' : 'connector_keys',
-        '-r' : 'repository',
-        '--repository' : 'repository',
-        '-i' : 'mrsm_instance',
-        '--instance' : 'mrsm_instance',
-        '--mrsm-instance' : 'mrsm_instance',
+        '-c': 'connector_keys',
+        '--connector-keys': 'connector_keys',
+        '-r': 'repository',
+        '--repository': 'repository',
+        '-i': 'mrsm_instance',
+        '--instance': 'mrsm_instance',
+        '--mrsm-instance': 'mrsm_instance',
     }
 
-
     ### TODO Find out arg possibilities
     possibilities = []
     last_word = line.rstrip(' ').split(' ')[-1]
@@ -236,10 +237,14 @@ def get_shell_intro(with_color: bool = True) -> str:
 
 
 class Shell(cmd.Cmd):
+    """
+    The interactive Meerschaum shell.
+    """
     def __init__(
         self,
         actions: Optional[Dict[str, Any]] = None,
-        sysargs: Optional[List[str]] = None
+        sysargs: Optional[List[str]] = None,
+        instance_keys: Optional[str] = None,
     ):
         """
         Customize the CLI from configuration
@@ -268,26 +273,13 @@ class Shell(cmd.Cmd):
             reserve_space_for_menu=False,
         )
 
-        try: ### try cmd2 arguments first
-            super().__init__(
-                allow_cli_args = False,
-                auto_load_commands = False,
-                persistent_history_length = 1000,
-                persistent_history_file = None,
-            )
-            _init = True
-        except Exception as e:
-             ### fall back to default init (cmd)
-            _init = False
-        
-        if not _init:
-            super().__init__()
+        super().__init__()
 
         ### remove default commands from the Cmd class
         for command in commands_to_remove:
             try:
                 delattr(cmd.Cmd, f'do_{command}')
-            except Exception as e:
+            except Exception:
                 pass
 
         ### NOTE: custom actions must be added to the self._actions dictionary
@@ -301,7 +293,7 @@ class Shell(cmd.Cmd):
         shell_attrs['_old_bottom_toolbar'] = ''
         shell_attrs['debug'] = False
         shell_attrs['_reload'] = True
-        self.load_config()
+        self.load_config(instance=instance_keys)
         self.hidden_commands = []
         ### update hidden commands list (cmd2 only)
         try:
@@ -314,7 +306,6 @@ class Shell(cmd.Cmd):
         from meerschaum._internal.shell.updates import run_version_check_thread
         self._update_thread = run_version_check_thread(debug=shell_attrs.get('debug', False))
 
-
     def load_config(self, instance: Optional[str] = None):
         """
         Set attributes from the shell configuration.
@@ -478,13 +469,13 @@ class Shell(cmd.Cmd):
         """
         Pass line string to parent actions.
         Pass parsed arguments to custom actions
-        
+
         Overrides `default`. If an action does not exist, assume the action is `shell`
         """
         ### Preserve the working directory.
         old_cwd = os.getcwd()
 
-        from meerschaum._internal.entry import _shell, get_shell
+        from meerschaum._internal.entry import get_shell
         self = get_shell(sysargs=shell_attrs['_sysargs'], debug=shell_attrs.get('debug', False))
 
         ### make a backup of line for later
@@ -522,7 +513,6 @@ class Shell(cmd.Cmd):
         if line.startswith(help_token):
             return "help " + line[len(help_token):]
 
-        from meerschaum._internal.arguments import parse_line
         try:
             sysargs = shlex.split(line)
         except ValueError as e:
@@ -628,18 +618,24 @@ class Shell(cmd.Cmd):
         for i, kwargs in enumerate([_ for _ in chained_kwargs]):
             kwargs.update(patches[i])
 
-        from meerschaum._internal.entry import entry_with_args, entry
+        from meerschaum._internal.entry import entry
         sysargs_to_execute = []
         for i, kwargs in enumerate(chained_kwargs):
             step_kwargs = {k: v for k, v in kwargs.items() if k != 'line'}
-            step_sysargs = parse_dict_to_sysargs(step_kwargs)
+            step_action = kwargs.get('action', None)
+            step_action_name = step_action[0] if step_action else None
+            ### NOTE: For `stack`, revert argument parsing.
+            step_sysargs = (
+                parse_dict_to_sysargs(step_kwargs)
+                if step_action_name != 'stack'
+                else chained_sysargs[i]
+            )
             sysargs_to_execute.extend(step_sysargs)
             sysargs_to_execute.append(AND_KEY)
-        
+
         sysargs_to_execute = sysargs_to_execute[:-1] + (
             ([':'] + pipeline_args) if pipeline_args else []
         )
-
         try:
             success_tuple = entry(sysargs_to_execute, _patch_args=patch_args)
         except Exception as e:
@@ -763,7 +759,6 @@ class Shell(cmd.Cmd):
 
         return True, "Success"
 
-
     def complete_instance(
         self,
         text: str,
@@ -776,9 +771,10 @@ class Shell(cmd.Cmd):
         from meerschaum.utils.misc import get_connector_labels
         from meerschaum._internal.arguments._parse_arguments import parse_line
         from meerschaum.connectors import instance_types, _load_builtin_custom_connectors
-        if _executor:
+        if not self.__dict__.get('_loaded_custom_connectors', None):
             _load_builtin_custom_connectors()
-            from meerschaum.jobs import executor_types
+            self.__dict__['_loaded_custom_connectors'] = True
+        from meerschaum.jobs import executor_types
 
         conn_types = instance_types if not _executor else executor_types
 
@@ -792,7 +788,6 @@ class Shell(cmd.Cmd):
             _additional_options=_additional_options,
         )
 
-
     def do_repo(
         self,
         action: Optional[List[str]] = None,
@@ -992,7 +987,6 @@ class Shell(cmd.Cmd):
         """
         Patch builtin cmdloop with my own input (defined below).
         """
-        import signal, os
         cmd.__builtins__['input'] = input_with_sigint(
             _old_input,
             shell_attrs['session'],

meerschaum/_internal/term/TermPageHandler.py

@@ -12,10 +12,9 @@ tornado_web = attempt_import('tornado.web', lazy=False)
 
 class TermPageHandler(tornado_web.RequestHandler):
     def get(self):
-        from meerschaum.api import endpoints
         return self.render(
             "termpage.html",
-            static = self.static_url,
-            ws_url_path = "/websocket"
+            static=self.static_url,
+            ws_url_path="/websocket",
         )
 

meerschaum/_internal/term/__init__.py

@@ -7,10 +7,8 @@ Build the web console virtual terminal using Tornado and xterm.js.
 """
 
 from __future__ import annotations
-import os
-import platform
-import sys
-from typing import List, Tuple
+
+from typing import Optional, Tuple
 from meerschaum.utils.packages import attempt_import
 from meerschaum._internal.term.TermPageHandler import TermPageHandler
 from meerschaum.config._paths import API_TEMPLATES_PATH, API_STATIC_PATH
@@ -20,10 +18,13 @@ tornado, tornado_ioloop, terminado = attempt_import(
     'tornado', 'tornado.ioloop', 'terminado', lazy=False,
 )
 
-def get_webterm_app_and_manager() -> Tuple[
-        tornado.web.Application,
-        terminado.UniqueTermManager,
-    ]:
+
+def get_webterm_app_and_manager(
+    instance_keys: Optional[str] = None,
+) -> Tuple[
+    tornado.web.Application,
+    terminado.UniqueTermManager,
+]:
     """
     Construct the Tornado web app and term manager from the provided sysargs.
 
@@ -31,12 +32,13 @@ def get_webterm_app_and_manager() -> Tuple[
     -------
     A tuple of the Tornado web application and term manager.
     """
+    shell_kwargs_str = f"mrsm_instance='{instance_keys}'" if instance_keys else ""
     commands = [
         venv_executable(None),
         '-c',
         "import os; _ = os.environ.pop('COLUMNS', None); _ = os.environ.pop('LINES', None); "
         "from meerschaum._internal.entry import get_shell; "
-        "get_shell([]).cmdloop()"
+        f"get_shell({shell_kwargs_str}).cmdloop()"
     ]
 
     term_manager = terminado.UniqueTermManager(shell_command=commands)
@@ -53,7 +55,7 @@ def get_webterm_app_and_manager() -> Tuple[
     ]
     tornado_app = tornado.web.Application(
         handlers,
-        static_path = API_STATIC_PATH,
-        template_path = API_TEMPLATES_PATH,
+        static_path=API_STATIC_PATH,
+        template_path=API_TEMPLATES_PATH,
     )
     return tornado_app, term_manager

meerschaum/actions/api.py

@@ -167,7 +167,7 @@ def _api_start(
 
     ### Uvicorn must be installed on the host because of multiprocessing reasons.
     ### `check_update` must be False, because otherwise Uvicorn's hidden imports will break things.
-    dotenv = attempt_import('dotenv', lazy=False)
+    _ = attempt_import('dotenv', lazy=False)
     uvicorn, gunicorn = attempt_import(
         'uvicorn', 'gunicorn', venv=None, lazy=False, check_update=False,
     )
@@ -235,6 +235,7 @@ def _api_start(
         'no_dash': no_dash,
         'no_auth': no_auth,
         'private': private,
+        'production': production,
     })
     if debug:
         uvicorn_config['reload'] = debug
@@ -251,7 +252,7 @@ def _api_start(
     if secure:
         cf['system']['api']['permissions']['actions']['non_admin'] = False
 
-    custom_keys = ['mrsm_instance', 'no_dash', 'no_auth', 'private', 'debug']
+    custom_keys = ['mrsm_instance', 'no_dash', 'no_auth', 'private', 'debug', 'production']
 
     ### write config to a temporary file to communicate with uvicorn threads
     import json, sys
@@ -342,7 +343,7 @@ def _api_start(
             run_python_package(
                 'gunicorn',
                 gunicorn_args,
-                env = {
+                env={
                     k: (
                         json.dumps(v)
                         if isinstance(v, (dict, list))
@@ -350,14 +351,16 @@ def _api_start(
                     )
                     for k, v in env_dict.items()
                 },
-                venv = None,
-                debug = debug,
+                venv=None,
+                debug=debug,
             )
         except KeyboardInterrupt:
             pass
 
-
-    _run_uvicorn() if not production else _run_gunicorn()
+    if production:
+        _run_gunicorn()
+    else:
+        _run_uvicorn()
 
     ### Cleanup
     if uvicorn_config_path.parent.exists():

meerschaum/actions/bootstrap.py

@@ -301,7 +301,7 @@ def _bootstrap_connectors(
             )
             if not overwrite and not force:
                 return False, "No changes made to connector configuration."
-                break
+            break
         elif _label == "":
             warn("Please enter a label.", stack=False)
         else:
@@ -336,6 +336,7 @@ def _bootstrap_connectors(
             return abort_tuple
         new_attributes['flavor'] = flavor
         required = sorted(list(connector_attributes[_type]['flavors'][flavor]['requirements']))
+        required = sorted(list(connector_attributes[_type]['flavors'][flavor]['optional']))
         default = type_attributes['flavors'][flavor].get('defaults', {})
     else:
         required = sorted(list(type_attributes.get('required', {})))

meerschaum/actions/delete.py

@@ -521,7 +521,10 @@ def _complete_delete_jobs(
         )
     )
 
-    if parse_executor_keys(executor_keys, construct=False) is None:
+    if (
+        executor_keys != 'systemd'
+        and parse_executor_keys(executor_keys, construct=False) is None
+    ):
         return []
 
     jobs = get_jobs(executor_keys, include_hidden=False)

meerschaum/actions/register.py

@@ -118,7 +118,6 @@ def _register_pipes(
             )
 
     success, message = True, "Success"
-    failed_message = ""
     failed_pipes = []
     success_pipes = []
     for p in pipes:
@@ -127,7 +126,6 @@ def _register_pipes(
         ss, msg = p.register(debug=debug)
         if not ss:
             warn(f"{msg}", stack=False)
-            success = False
             failed_pipes.append(p)
         else:
             success_pipes.append(p)
@@ -356,7 +354,7 @@ def _register_users(
 
     msg = (
         f"Finished registering {succeeded + failed} users." + '\n' +
-        f"  ({succeeded} succeeded, {failed} failed)"
+        f"    ({succeeded} succeeded, {failed} failed)"
     )
     return succeeded > 0, msg
 

meerschaum/actions/stack.py

@@ -9,34 +9,32 @@ Functions for running the Docker Compose stack
 from __future__ import annotations
 from meerschaum.utils.typing import SuccessTuple, Any, List, Optional, Union
 
+
 def stack(
-        action: Optional[List[str]] = None,
-        sysargs: Optional[List[str]] = None,
-        sub_args: Optional[List[str]] = None,
-        yes: bool = False,
-        noask: bool = False,
-        force: bool = False,
-        debug: bool = False,
-        _capture_output: bool = False,
-        **kw: Any
-    ) -> Union[SuccessTuple, 'subprocess.Popen']:
+    action: Optional[List[str]] = None,
+    sysargs: Optional[List[str]] = None,
+    sub_args: Optional[List[str]] = None,
+    yes: bool = False,
+    noask: bool = False,
+    force: bool = False,
+    debug: bool = False,
+    _capture_output: bool = False,
+    **kw: Any
+) -> Union[SuccessTuple, 'subprocess.Popen']:
     """
     Control the Meerschaum stack with Docker Compose.
     Usage: `stack {command}`
-    
+
     Command: action[0]: default 'up'
         Docker Compose command to run. E.g. 'config' will print Docker Compose configuration
     """
     import subprocess
-    import contextlib
-    import io
     import os
     import sys
     import pathlib
     import meerschaum.config.stack
     from meerschaum.config.stack import NECESSARY_FILES, write_stack
     from meerschaum.config._paths import STACK_COMPOSE_PATH
-    from meerschaum.utils.prompt import yes_no
     import meerschaum.config
     from meerschaum.config._patch import apply_patch_to_config
     from meerschaum.utils.packages import (
@@ -47,9 +45,9 @@ def stack(
     from meerschaum.config import get_config
     from meerschaum.utils.debug import dprint
     from meerschaum.utils.warnings import warn
-    from meerschaum.utils.formatting import ANSI
     from meerschaum.utils.misc import is_docker_available
     from meerschaum.config._read_config import search_and_substitute_config
+    from meerschaum.utils.prompt import yes_no
 
     stack_env_dict = apply_patch_to_config(
         os.environ.copy(),
@@ -81,7 +79,7 @@ def stack(
             break
     if bootstrap:
         write_stack(debug=debug)
-    else: 
+    else:
         sync_files(['stack'])
 
     ### define project name when starting containers
@@ -91,7 +89,7 @@ def stack(
             'stack', 'project_name', patch=True, substitute=True,
         )
     ]
-    
+
     ### Debug list used to include --log-level DEBUG, but the flag is not supported on Windows (?)
     debug_list = []
 
@@ -102,7 +100,7 @@ def stack(
         print(
             "To start the Docker service, run `sudo systemctl start docker` or `sudo dockerd`.\n"
             + "On Windows or MacOS, make sure Docker Desktop is running.",
-            file = sys.stderr,
+            file=sys.stderr,
         )
         return False, "Failed to connect to the Docker engine."
 
@@ -115,7 +113,7 @@ def stack(
 
     if not has_builtin_compose:
         _compose_venv = 'mrsm'
-        compose = attempt_import('compose', lazy=False, venv=_compose_venv, debug=debug)
+        _ = attempt_import('compose', lazy=False, venv=_compose_venv, debug=debug)
 
         ### If docker-compose is installed globally, don't use the `mrsm` venv.
         if not venv_contains_package('compose', _compose_venv):
@@ -129,6 +127,13 @@ def stack(
             if not pip_install('pyyaml', venv=_compose_venv, debug=debug):
                 warn(f"Unable to install `pyyaml` into venv '{_compose_venv}'.")
 
+    if 'down' in sysargs and '-v' in sysargs:
+        if not yes_no(
+            "Are you sure you want to drop volumes?\n    This cannot be undone!",
+            default='n',
+        ):
+            return False, "Nothing was dropped."
+
     cmd_list = [
         _arg
         for _arg in (settings_list + sysargs[1:])