meerschaum 3.0.0rc4__py3-none-any.whl → 3.0.0rc8__py3-none-any.whl

meerschaum/_internal/shell/Shell.py

@@ -31,11 +31,13 @@ prompt_toolkit = attempt_import('prompt_toolkit', lazy=False, warn=False, instal
 )
 from meerschaum._internal.shell.ValidAutoSuggest import ValidAutoSuggest
 from meerschaum._internal.shell.ShellCompleter import ShellCompleter
+from meerschaum._internal.cli.daemons import get_cli_daemon, get_cli_session_id
 _clear_screen = get_config('shell', 'clear_screen', patch=True)
 from meerschaum.utils.misc import string_width, remove_ansi
 from meerschaum.utils.warnings import warn
 from meerschaum.jobs import get_executor_keys_from_context
 from meerschaum._internal.static import STATIC_CONFIG
+from meerschaum.utils.formatting._shell import clear_screen, flush_stdout
 from meerschaum._internal.arguments._parse_arguments import (
     split_chained_sysargs,
     split_pipeline_sysargs,
@@ -81,6 +83,7 @@ ESCAPED_AND_KEY: str = STATIC_CONFIG['system']['arguments']['escaped_and_key']
 PIPELINE_KEY: str = STATIC_CONFIG['system']['arguments']['pipeline_key']
 ESCAPED_PIPELINE_KEY: str = STATIC_CONFIG['system']['arguments']['escaped_pipeline_key']
 
+
 def _insert_shell_actions(
     _shell: Optional['Shell'] = None,
     actions: Optional[Dict[str, Callable[[Any], SuccessTuple]]] = None,
@@ -99,6 +102,10 @@ def _insert_shell_actions(
     _shell_class = _shell if _shell is not None else shell_pkg.Shell
 
     for a, f in actions.items():
+        existing_method = getattr(_shell_class, 'do_' + a, None)
+        if existing_method == f:
+            continue
+
         add_method_to_class(
             func = f,
             class_def = _shell_class,
@@ -113,6 +120,35 @@ def _insert_shell_actions(
         setattr(_shell_class, 'complete_' + a, completer)
 
 
+def _remove_shell_actions(
+    _shell: Optional['Shell'] = None,
+    actions: Optional[Dict[str, Callable[[Any], SuccessTuple]]] = None,
+) -> None:
+    """
+    Remove the actions added to the shell.
+    """
+    import meerschaum._internal.shell as shell_pkg
+    if actions is None:
+        from meerschaum.actions import actions as _actions
+        actions = _actions
+
+    _shell_class = _shell if _shell is not None else shell_pkg.Shell
+
+    for a, f in actions.items():
+        try:
+            delattr(_shell_class, 'do_' + a)
+        except AttributeError:
+            pass
+
+        if a in reserved_completers:
+            continue
+
+        try:
+            delattr(_shell_class, 'complete_' + a)
+        except AttributeError:
+            pass
+
+
 def _completer_wrapper(
     target: Callable[[Any], List[str]]
 ) -> Callable[['mrsm._internal.shell.Shell', str, str, int, int], Any]:
@@ -294,16 +330,19 @@ class Shell(cmd.Cmd):
                 pass
 
         ### NOTE: custom actions must be added to the self._actions dictionary
+        shell_attrs['_session_id'] = get_cli_session_id()
         shell_attrs['_actions'] = actions
         shell_attrs['_sysargs'] = sysargs
         shell_attrs['_actions']['instance'] = self.do_instance
         shell_attrs['_actions']['repo'] = self.do_repo
         shell_attrs['_actions']['executor'] = self.do_executor
         shell_attrs['_actions']['debug'] = self.do_debug
+        shell_attrs['_actions']['daemon'] = self.do_daemon
         shell_attrs['_update_bottom_toolbar'] = True
         shell_attrs['_old_bottom_toolbar'] = ''
         shell_attrs['debug'] = False
         shell_attrs['_reload'] = True
+        shell_attrs['daemon'] = get_config('system', 'experimental', 'cli_daemon')
         self.load_config(instance=instance_keys)
         self.hidden_commands = []
         ### update hidden commands list (cmd2 only)
@@ -353,7 +392,7 @@ class Shell(cmd.Cmd):
             shell_attrs['instance'] = instance
             shell_attrs['instance_keys'] = remove_ansi(str(instance))
         if shell_attrs.get('repo_keys', None) is None:
-            shell_attrs['repo_keys'] = get_config('meerschaum', 'default_repository', patch=patch)
+            shell_attrs['repo_keys'] = get_config('meerschaum', 'repository', patch=patch)
         if shell_attrs.get('executor_keys', None) is None:
             shell_attrs['executor_keys'] = get_executor_keys_from_context()
 
@@ -496,8 +535,7 @@ class Shell(cmd.Cmd):
             )
         )
         shell_attrs['prompt'] = self.prompt
-        ### flush stdout
-        print("", end="", flush=True)
+        flush_stdout()
 
 
     def precmd(self, line: str):
@@ -530,7 +568,6 @@ class Shell(cmd.Cmd):
 
         ### if the user specifies, clear the screen before executing any commands
         if _clear_screen:
-            from meerschaum.utils.formatting._shell import clear_screen
             clear_screen(debug=shell_attrs['debug'])
 
         ### return blank commands (spaces break argparse)
@@ -618,7 +655,7 @@ class Shell(cmd.Cmd):
             if key == 'mrsm_instance':
                 default_value = get_config('meerschaum', 'instance')
             elif key == 'repository':
-                default_value = get_config('meerschaum', 'default_repository')
+                default_value = get_config('meerschaum', 'repository')
             elif key == 'executor_keys':
                 default_value = get_executor_keys_from_context()
             else:
@@ -672,7 +709,12 @@ class Shell(cmd.Cmd):
             ([':'] + pipeline_args) if pipeline_args else []
         )
         try:
-            success_tuple = entry(sysargs_to_execute, _patch_args=patch_args)
+            success_tuple = entry(
+                sysargs_to_execute,
+                _patch_args=patch_args,
+                _session_id=shell_attrs.get('session_id', None),
+                _use_cli_daemon=shell_attrs.get('daemon', False),
+            )
         except Exception as e:
             success_tuple = False, str(e)
 
@@ -732,6 +774,35 @@ class Shell(cmd.Cmd):
 
         info(f"Debug mode is {'on' if shell_attrs['debug'] else 'off'}.")
 
+    def do_daemon(self, action: Optional[List[str]] = None, executor_keys=None, **kw):
+        """
+        Toggle whether to route commands through the CLI daemon.
+        
+        Command:
+            `debug {on/true | off/false}`
+            Ommitting on / off will toggle the existing value.
+        """
+        from meerschaum.utils.warnings import info
+        on_commands = {'on', 'true', 'True'}
+        off_commands = {'off', 'false', 'False'}
+        if action is None:
+            action = []
+        try:
+            state = action[0]
+        except (IndexError, AttributeError):
+            state = ''
+        if state == '':
+            shell_attrs['daemon'] = not shell_attrs['daemon']
+        elif state.lower() in on_commands:
+            shell_attrs['daemon'] = True
+        elif state.lower() in off_commands:
+            shell_attrs['daemon'] = False
+        else:
+            info(f"Unknown state '{state}'. Ignoring...")
+
+        info(f"CLI daemon mode is {'on' if shell_attrs['daemon'] else 'off'}.")
+        return True, "Success"
+
     def do_instance(
         self,
         action: Optional[List[str]] = None,
@@ -833,7 +904,7 @@ class Shell(cmd.Cmd):
         """
         Temporarily set a default Meerschaum repository for the duration of the shell.
         The default repository (mrsm.io) is loaded from the Meerschaum configuraton file
-          (at keys 'meerschaum:default_repository').
+          (at keys 'meerschaum.repository').
         
         You can change the default repository with `edit config`.
         
@@ -864,7 +935,7 @@ class Shell(cmd.Cmd):
         except (IndexError, AttributeError):
             repo_keys = ''
         if repo_keys == '':
-            repo_keys = get_config('meerschaum', 'default_repository', patch=True)
+            repo_keys = get_config('meerschaum', 'repository', patch=True)
 
         conn = parse_repo_keys(repo_keys, debug=debug)
         if conn is None or not conn:
@@ -947,7 +1018,7 @@ class Shell(cmd.Cmd):
         show pipes -h
         ```
         """
-        from meerschaum.actions import actions
+        from meerschaum.actions import get_action
         from meerschaum._internal.arguments._parser import parse_help
         from meerschaum._internal.arguments._parse_arguments import parse_line
         import textwrap
@@ -956,12 +1027,15 @@ class Shell(cmd.Cmd):
             del args['action']
             shell_attrs['_actions']['show'](['actions'], **args)
             return ""
+
         if args['action'][0] not in shell_attrs['_actions']:
+            action_func = get_action(args['action'])
             try:
-                print(textwrap.dedent(getattr(self, f"do_{args['action'][0]}").__doc__))
+                print(textwrap.dedent(action_func.__doc__))
             except Exception:
-                print(f"No help on '{args['action'][0]}'.")
+                print(f"No help on '{shlex.join(args['action'])}'.")
             return ""
+
         parse_help(args)
         return ""
 
@@ -1029,7 +1103,6 @@ class Shell(cmd.Cmd):
 
         ### if the user specifies, clear the screen before initializing the shell
         if _clear_screen:
-            from meerschaum.utils.formatting._shell import clear_screen
             clear_screen(debug=shell_attrs['debug'])
 
         ### if sysargs are provided, skip printing the intro and execute instead
@@ -1045,9 +1118,10 @@ def input_with_sigint(_input, session, shell: Optional[Shell] = None):
     Replace built-in `input()` with prompt_toolkit.prompt.
     """
     from meerschaum.utils.formatting import CHARSET, ANSI, colored, UNICODE
-    from meerschaum.connectors import is_connected, connectors
+    from meerschaum.connectors import connectors
     from meerschaum.utils.misc import remove_ansi, truncate_text_for_display
     from meerschaum.config import get_config
+
     import platform
     if shell is None:
         from meerschaum.actions import get_shell
@@ -1062,8 +1136,16 @@ def input_with_sigint(_input, session, shell: Optional[Shell] = None):
             return None
         if not shell_attrs['_update_bottom_toolbar'] and platform.system() == 'Windows':
             return shell_attrs['_old_bottom_toolbar']
-        size = os.get_terminal_size()
-        num_cols, num_lines = size.columns, size.lines
+        try:
+            size = os.get_terminal_size()
+            num_cols, num_lines = size.columns, size.lines
+        except Exception:
+            from meerschaum.utils.misc import is_int
+            num_cols, num_lines = os.environ.get('COLUMNS', 80), os.environ.get('LINES', 120)
+            if is_int(str(num_cols)):
+                num_cols = int(num_cols)
+            if is_int(str(num_lines)):
+                num_lines = int(num_lines)
         truncation_suffix = (
             '…'
             if UNICODE
@@ -1112,24 +1194,34 @@ def input_with_sigint(_input, session, shell: Optional[Shell] = None):
 
         try:
             typ, label = shell_attrs['instance_keys'].split(':', maxsplit=1)
-            connected = typ in connectors and label in connectors[typ]
+            cli_worker = (
+                get_cli_daemon()
+                if shell_attrs.get('daemon', False)
+                else None
+            )
+            connected = (
+                cli_worker.job.status == 'running'
+                if cli_worker is not None
+                else (typ in connectors and label in connectors[typ])
+            )
         except Exception:
             connected = False
         last_connected = connected
         connected_str = truncate_text_for_display(
-            ('dis' if not connected else '') + 'connected',
+            ('daemon' if cli_worker is not None else remove_ansi(shell_attrs['instance_keys'])),
             **truncation_kwargs
         )
+        connected_status_str = 'disconnected' if not connected else 'connected'
         connected_icon = get_config(
-            'formatting', connected_str, CHARSET, 'icon', warn=False,
+            'formatting', connected_status_str, CHARSET, 'icon', warn=False,
         ) or ''
         connection_text = (
-            connected_icon + ' ' + (
-                colored(connected_str.capitalize(), 'on ' + (get_config(
-                    'formatting', connected_str, 'ansi', 'rich', 'style',
+            (
+                colored(connected_str, 'on ' + (get_config(
+                    'formatting', connected_status_str, 'ansi', 'rich', 'style',
                     warn=False,
-                ) or '') + '  ') if ANSI else (colored(connected_str.capitalize(), 'on white') + ' ')
-            )
+                ) or '') + '  ') if ANSI else (colored(connected_str, 'on white') + ' ')
+            ) + ' ' + connected_icon
         )
 
         left = (
@@ -1164,7 +1256,6 @@ def input_with_sigint(_input, session, shell: Optional[Shell] = None):
             ### NOTE: would it be better to do nothing instead?
             if len(parsed.strip()) == 0:
                 if _clear_screen:
-                    from meerschaum.utils.formatting._shell import clear_screen
                     clear_screen()
         except KeyboardInterrupt:
             print("^C")

meerschaum/_internal/shell/__init__.py

@@ -8,5 +8,8 @@ Import the Shell class definition
 
 from meerschaum._internal.shell.Shell import Shell
 from meerschaum._internal.shell.Shell import (
-    default_action_completer, _completer_wrapper, _insert_shell_actions,
+    default_action_completer,
+    _completer_wrapper,
+    _insert_shell_actions,
+    _remove_shell_actions,
 )

meerschaum/_internal/static.py

@@ -240,6 +240,7 @@ STATIC_CONFIG: Dict[str, Any] = {
         'noask': 'MRSM_NOASK',
         'noninteractive': 'MRSM_NONINTERACTIVE',
         'id': 'MRSM_SERVER_ID',
+        'test_flavors': 'MRSM_TEST_FLAVORS',
         'daemon_id': 'MRSM_DAEMON_ID',
         'systemd_log_path': 'MRSM_SYSTEMD_LOG_PATH',
         'systemd_stdin_path': 'MRSM_SYSTEMD_STDIN_PATH',
@@ -322,6 +323,9 @@ STATIC_CONFIG: Dict[str, Any] = {
     },
     'jobs': {
         'check_restart_seconds': 1.0,
+        'stop_token': '<------- MRSM_STOP_TOKEN ------->',
+        'clear_token': '<------- MRSM_CLEAR_TOKEN ------->',
+        'flush_token': '<------- MRSM_FLUSH_TOKEN ------->\n',
     },
     'tokens': {
         'minimum_length': 24,
@@ -352,5 +356,4 @@ STATIC_CONFIG: Dict[str, Any] = {
             'users:delete': "Delete the associated user account (or other users for admins).",
         },
     },
-    
 }

meerschaum/_internal/term/TermPageHandler.py

@@ -10,11 +10,10 @@ from __future__ import annotations
 from typing import Any
 
 import meerschaum as mrsm
-from meerschaum.utils.warnings import warn
 tornado_web = mrsm.attempt_import('tornado.web', lazy=False)
 terminado = mrsm.attempt_import('terminado', lazy=False)
 
-tmux_suffix = mrsm.get_config('system', 'webterm', 'tmux', 'session_suffix')
+tmux_suffix = mrsm.get_config('api', 'webterm', 'tmux', 'session_suffix')
 
 
 class TermPageHandler(tornado_web.RequestHandler):

meerschaum/_internal/term/__init__.py

@@ -8,6 +8,9 @@ Build the web console virtual terminal using Tornado and xterm.js.
 
 from __future__ import annotations
 
+import os
+import json
+import pathlib
 from typing import Optional, Tuple
 
 import meerschaum as mrsm
@@ -16,7 +19,6 @@ from meerschaum._internal.term.TermPageHandler import TermPageHandler, CustomTer
 from meerschaum.config._paths import API_TEMPLATES_PATH, API_STATIC_PATH
 from meerschaum.utils.venv import venv_executable
 from meerschaum.utils.misc import is_tmux_available
-from meerschaum.utils.daemon._names import get_new_daemon_name
 
 tornado, tornado_ioloop, terminado = attempt_import(
     'tornado', 'tornado.ioloop', 'terminado', lazy=False,
@@ -25,6 +27,8 @@ tornado, tornado_ioloop, terminado = attempt_import(
 
 def get_webterm_app_and_manager(
     instance_keys: Optional[str] = None,
+    port: Optional[int] = None,
+    env_path: Optional[pathlib.Path] = None,
 ) -> Tuple[
     tornado.web.Application,
     terminado.UniqueTermManager,
@@ -36,19 +40,49 @@ def get_webterm_app_and_manager(
     -------
     A tuple of the Tornado web application and term manager.
     """
+    from meerschaum.config.environment import get_env_vars
+    from meerschaum._internal.static import STATIC_CONFIG
+    if env_path is None:
+        from meerschaum.config.paths import WEBTERM_INTERNAL_RESOURCES_PATH
+        env_path = WEBTERM_INTERNAL_RESOURCES_PATH / (str(port) + '.json')
+
+    env_vars = get_env_vars()
+    env_dict = {
+        env_var: os.environ[env_var]
+        for env_var in env_vars
+        if env_var not in (
+            STATIC_CONFIG['environment']['systemd_log_path'],
+            STATIC_CONFIG['environment']['systemd_result_path'],
+            STATIC_CONFIG['environment']['systemd_delete_job'],
+            STATIC_CONFIG['environment']['systemd_stdin_path'],
+            STATIC_CONFIG['environment']['daemon_id'],
+        )
+    }
+    with open(env_path, 'w+', encoding='utf-8') as f:
+        json.dump(env_dict, f)
+
     shell_kwargs_str = f"mrsm_instance='{instance_keys}'" if instance_keys else ""
     commands = [
         venv_executable(None),
         '-c',
-        "import os; _ = os.environ.pop('COLUMNS', None); _ = os.environ.pop('LINES', None); "
-        "from meerschaum._internal.entry import get_shell; "
+        "import os\n"
+        "import pathlib\n"
+        "import json\n"
+        f"env_path = pathlib.Path('{env_path.as_posix()}')\n"
+        "with open(env_path, 'r', encoding='utf-8') as f:\n"
+        "    env_dict = json.load(f)\n"
+        "os.environ.update(env_dict)\n"
+        "_ = os.environ.pop('COLUMNS', None)\n"
+        "_ = os.environ.pop('LINES', None)\n"
+        "from meerschaum._internal.entry import get_shell\n"
         f"get_shell({shell_kwargs_str}).cmdloop()"
     ]
-    webterm_cf = mrsm.get_config('system', 'webterm')
+    webterm_cf = mrsm.get_config('api', 'webterm')
     if webterm_cf.get('tmux', {}).get('enabled', False) and is_tmux_available():
-        commands = ['tmux', 'new-session', '-A', '-s', 'MRSM_SESSION'] + commands
+        commands = ['tmux', 'new-session', '-A', '-s', f'MRSM_SESSION--{port}'] + commands
 
-    term_manager = terminado.NamedTermManager(shell_command=commands)
+    term_manager = terminado.NamedTermManager(shell_command=commands, extra_env=env_dict)
+    term_manager._port = port
     handlers = [
         (
             r"/websocket/(.+)/?",

meerschaum/_internal/term/tools.py

@@ -6,13 +6,14 @@
 Utility functions regarding the webterm.
 """
 
+from typing import List, Optional
+
 import meerschaum as mrsm
-from meerschaum.utils.typing import List
 
 
 def is_webterm_running(
-    host: str,
-    port: int,
+    host: Optional[str] = None,
+    port: Optional[int] = None,
     protocol: str = 'http',
     session_id: str = 'mrsm',
 ) -> int:
@@ -20,10 +21,12 @@ def is_webterm_running(
     Determine whether the webterm service is running on a given host and port.
     """
     requests = mrsm.attempt_import('requests', lazy=False)
+    host = host or mrsm.get_config('api', 'webterm', 'host')
+    port = port or mrsm.get_config('api', 'webterm', 'port')
     url = f'{protocol}://{host}:{port}/webterm/{session_id}'
     try:
         r = requests.get(url, timeout=3)
-    except Exception as e:
+    except Exception:
         return False
     if not r:
         return False
@@ -39,22 +42,44 @@ def kill_tmux_session(session: str) -> bool:
     return run_process(command, capture_output=True) == 0
 
 
-def get_mrsm_tmux_sessions() -> List[str]:
+def get_mrsm_tmux_sessions(port: Optional[int] = None) -> List[str]:
     """
     Return a list of tmux sessions created by Meerschaum.
     """
     from meerschaum.utils.process import run_process
-    tmux_suffix = mrsm.get_config('system', 'webterm', 'tmux', 'session_suffix')
+    tmux_suffix = mrsm.get_config('api', 'webterm', 'tmux', 'session_suffix')
     command = ['tmux', 'ls']
     proc = run_process(command, capture_output=True, as_proc=True)
     if proc.returncode != 0:
         return []
+
+    port = port or mrsm.get_config('api', 'webterm', 'port')
+
     sessions = [
         line.split(':', maxsplit=1)[0]
         for line in proc.stdout.read().decode('utf-8').split('\n')
     ]
+    mrsm_sessions_ports = []
+    for session in sessions:
+        if '--' not in session:
+            continue
+
+        parts = session.split('--', maxsplit=1)
+        if len(parts) != 2:
+            continue
+
+        if not parts[0].endswith(tmux_suffix):
+            continue
+
+        try:
+            session_port = int(parts[1])
+        except Exception:
+            continue
+
+        mrsm_sessions_ports.append((session, session_port))
+
     return [
         session
-        for session in sessions
-        if session.endswith(tmux_suffix)
+        for session, session_port in mrsm_sessions_ports
+        if session_port == port
     ]

meerschaum/actions/__init__.py

@@ -9,7 +9,6 @@ Default actions available to the mrsm CLI.
 from __future__ import annotations
 from meerschaum.utils.typing import Callable, Any, Optional, Union, List, Dict, SuccessTuple
 from meerschaum.utils.packages import get_modules_from_package
-_custom_actions = []
 
 __all__ = (
     'get_action',
@@ -307,6 +306,8 @@ __all__ = ['actions', 'get_subactions', 'get_action', 'get_main_action_name', 'g
 ### functions that do not begin with '_' from all submodules.
 from inspect import getmembers, isfunction
 actions = {}
+_custom_actions_plugins: Dict[str, str] = {}
+_plugins_actions: Dict[str, List[str]] = {}
 
 for module in modules:
     ### A couple important things happening here:
@@ -329,8 +330,8 @@ for module in modules:
                         if isfunction(ob[1])
                             ### check that the function belongs to the module
                             and ob[0] == module.__name__.replace('_', '').split('.')[-1]
-                            ### skip functions that start with '_'
-                            and ob[0][0] != '_'
+                            ### skip functions that start with '__'
+                            and ob[0][0] != '__'
             ]
         )
     )
@@ -362,4 +363,5 @@ __pdoc__ = {
 }
 for a in actions:
     __pdoc__[a] = False
-meerschaum.plugins.load_plugins()
+
+meerschaum.plugins.load_plugins(skip_if_loaded=True)