meerschaum 3.0.0rc4__py3-none-any.whl → 3.0.0rc7__py3-none-any.whl

meerschaum/actions/delete.py

@@ -18,7 +18,7 @@ def delete(
     Delete an element.
 
     Command:
-        `delete {config, pipes, plugins, users, connectors, jobs}`
+        `delete {config, pipes, plugins, users, connectors, jobs, venvs}`
 
     """
     from meerschaum.actions import choose_subaction
@@ -159,7 +159,9 @@ def _delete_config(
         answer = yes_no(
             "Are you sure you want to delete the following configuration files?" +
             f"{sep + sep.join([str(p) for p in paths])}\n",
-            default='n', noask=noask, yes=yes
+            default='n',
+            noask=noask,
+            yes=yes,
         )
 
     if answer or force:

meerschaum/actions/edit.py

@@ -67,7 +67,12 @@ def _complete_edit(
     return default_action_completer(action=(['edit'] + action), **kw)
 
 
-def _edit_config(action: Optional[List[str]] = None, **kw: Any) -> SuccessTuple:
+def _edit_config(
+    action: Optional[List[str]] = None,
+    noask: bool = False,
+    debug: bool = False,
+    **kwargs: Any
+) -> SuccessTuple:
     """
     Edit Meerschaum configuration files.
 
@@ -87,11 +92,26 @@ def _edit_config(action: Optional[List[str]] = None, **kw: Any) -> SuccessTuple:
         ```
     """
     from meerschaum.config._edit import edit_config
-    if action is None:
-        action = []
-    if len(action) == 0:
-        action.append('meerschaum')
-    return edit_config(keys=action, **kw)
+    from meerschaum.config._read_config import get_possible_keys
+    from meerschaum.actions import actions
+    from meerschaum.utils.prompt import choose
+
+    if not action:
+        action = [
+            choose(
+                "Choose a configuration file to edit:",
+                get_possible_keys(),
+                default='meerschaum',
+                noask=noask,
+            )
+        ]
+
+    edit_success, edit_msg = edit_config(keys=action, debug=debug, **kwargs)
+    if not edit_success:
+        return edit_success, edit_msg
+
+    return actions['reload'](debug=debug)
+
 
 def _complete_edit_config(action: Optional[List[str]] = None, **kw: Any) -> List[str]:
     from meerschaum.config._read_config import get_possible_keys
@@ -343,7 +363,6 @@ def _edit_plugins(
     from meerschaum.utils.warnings import warn
     from meerschaum.utils.prompt import prompt, yes_no
     from meerschaum.utils.misc import edit_file
-    from meerschaum.utils.packages import reload_meerschaum
     from meerschaum.actions import actions
 
     if not action:
@@ -376,7 +395,7 @@ def _edit_plugins(
             continue
 
         edit_file(plugin_file_path)
-        reload_meerschaum(debug=debug)
+        actions['reload'](debug=debug)
 
     return True, "Success"
 

meerschaum/actions/login.py

@@ -10,13 +10,13 @@ from __future__ import annotations
 from meerschaum.utils.typing import SuccessTuple, Optional, List, Any
 
 def login(
-        action: Optional[List[str]] = None,
-        connector_keys: Optional[List[str]] = None,
-        yes: bool = False,
-        noask: bool = False,
-        debug: bool = False,
-        **kw: Any
-    ) -> SuccessTuple:
+    action: Optional[List[str]] = None,
+    connector_keys: Optional[List[str]] = None,
+    yes: bool = False,
+    noask: bool = False,
+    debug: bool = False,
+    **kw: Any
+) -> SuccessTuple:
     """
     Log into a Meerschaum API instance.
     """
@@ -56,7 +56,7 @@ def login(
     for k in _keys:
         try:
             _connectors.append(parse_instance_keys(k))
-        except Exception as e:
+        except Exception:
             warn(f"Unable to build connector '{k}'. Is it registered?", stack=False)
 
     meerschaum_config = get_config('meerschaum')

meerschaum/actions/register.py

@@ -512,14 +512,20 @@ def _register_tokens(
             "To which user should this token be registered? Enter the number.",
             usernames,
         )
-        user_id = instance_connector.get_user_id(
-            User(username, instance=mrsm_instance),
-            debug=debug,
-        )
-        if user_id is None:
-            return False, f"Cannot load ID for user '{username}'."
+    else:
+        username = getattr(instance_connector, 'username')
+
+    if not username:
+        return False, "Cannot register a token without a user."
+
+    user_id = instance_connector.get_user_id(
+        User(username, instance=mrsm_instance),
+        debug=debug,
+    )
+    if user_id is None:
+        return False, f"Cannot load ID for user '{username}'."
 
-        user = User(username, user_id=user_id, instance=mrsm_instance)
+    user = User(username, user_id=user_id, instance=mrsm_instance)
 
     token = Token(
         label=name,

meerschaum/actions/reload.py

@@ -9,13 +9,30 @@ Reload the running Meerschaum instance.
 from __future__ import annotations
 from meerschaum.utils.typing import Any, SuccessTuple, List, Optional
 
+
 def reload(
-        action: Optional[List[str]] = None,
-        debug: bool = False,
-        **kw: Any
-    ) -> SuccessTuple:
+    action: Optional[List[str]] = None,
+    debug: bool = False,
+    _stop_daemons: bool = True,
+    **kw: Any
+) -> SuccessTuple:
     """
     Reload the running Meerschaum instance.
     """
     from meerschaum.utils.packages import reload_meerschaum
-    return reload_meerschaum(debug=debug)
+    from meerschaum.actions import actions
+
+    if _stop_daemons:
+        from meerschaum._internal.cli.workers import get_existing_cli_worker_indices
+        indices = get_existing_cli_worker_indices()
+        cli_action = 'restart' if indices else 'stop'
+
+        stop_daemon_success, stop_daemon_msg = actions[cli_action](['daemons'], debug=debug, **kw)
+        if not stop_daemon_success:
+            return stop_daemon_success, stop_daemon_msg
+
+    reload_success, reload_msg = reload_meerschaum(debug=debug)
+    if not reload_success:
+        return reload_success, reload_msg
+
+    return True, "Success"

meerschaum/actions/restart.py

@@ -7,6 +7,7 @@ Restart stopped jobs which have not been manually stopped.
 
 from meerschaum.utils.typing import SuccessTuple, Optional, List, Any
 
+
 def restart(
     action: Optional[List[str]] = None,
     executor_keys: Optional[str] = None,
@@ -18,6 +19,7 @@ def restart(
     from meerschaum.actions import choose_subaction
     attach_options = {
         'jobs': _restart_jobs,
+        'daemons': _restart_daemons,
     }
     return choose_subaction(action, attach_options, executor_keys=executor_keys, **kwargs)
 
@@ -111,3 +113,15 @@ def _restart_jobs(
             time.sleep(min_seconds)
 
     return check_success, check_msg
+
+
+def _restart_daemons(
+    action: Optional[List[str]] = None,
+    **kwargs
+) -> SuccessTuple:
+    """
+    Restart the CLI daemons.
+    """
+    from meerschaum.actions import actions
+    kwargs['force'] = True
+    return actions['start'](['daemons'], **kwargs)

meerschaum/actions/show.py

@@ -46,6 +46,7 @@ def show(
         'schedules'  : _show_schedules,
         'venvs'      : _show_venvs,
         'tokens'     : _show_tokens,
+        'daemons'    : _show_daemons,
     }
     return choose_subaction(action, show_options, **kw)
 
@@ -205,7 +206,7 @@ def _show_version(nopretty: bool = False, **kw : Any) -> SuccessTuple:
         msg = "Meerschaum v" + version
         _print = info
     _print(msg)
-    return (True, "Success")
+    return True, "Success"
 
 
 def _show_connectors(
@@ -563,6 +564,7 @@ def _complete_show_packages(
 
     return possibilities
 
+
 def _show_jobs(
     action: Optional[List[str]] = None,
     executor_keys: Optional[str] = None,
@@ -763,7 +765,7 @@ def _show_environment(
     """
     import os
     from meerschaum.utils.formatting import pprint
-    from meerschaum.config._environment import get_env_vars
+    from meerschaum.config.environment import get_env_vars
     pprint(
         {
             env_var: os.environ[env_var]
@@ -930,7 +932,7 @@ def _show_tokens(
     from meerschaum.utils.packages import import_rich
     from meerschaum.utils.formatting import UNICODE, get_console
     rich = import_rich()
-    rich_table, rich_json = mrsm.attempt_import('rich.table', 'rich.json')
+    rich_table, rich_json, rich_box = mrsm.attempt_import('rich.table', 'rich.json', 'rich.box')
 
     conn = parse_instance_keys(mrsm_instance)
 
@@ -994,7 +996,11 @@ def _show_tokens(
         else "[-]"
     )
 
-    table = rich_table.Table(title=f"Registered Tokens on instance '{conn}'")
+    table = rich_table.Table(
+        title=f"Registered Tokens on instance '{conn}'",
+        box=rich_box.ROUNDED,
+        title_style='bold',
+    )
     table.add_column("ID")
     table.add_column("Label")
     table.add_column("User")
@@ -1018,6 +1024,65 @@ def _show_tokens(
     return True, "Success"
 
 
+def _show_daemons() -> SuccessTuple:
+    """
+    Print information about the running CLI daemons.
+    """
+    from meerschaum._internal.cli.workers import get_existing_cli_workers
+    workers = get_existing_cli_workers()
+    if not workers:
+        return True, "No CLI daemons are running."
+
+    rich_table, rich_box, rich_text = mrsm.attempt_import('rich.table', 'rich.box', 'rich.text')
+    table = rich_table.Table(
+        rich_table.Column("Index", justify='center'),
+        rich_table.Column("Status"),
+        rich_table.Column("Locked", justify='center'),
+        rich_table.Column("Ready", justify='center'),
+        rich_table.Column("Actions"),
+        title='CLI Daemons',
+        box=rich_box.ROUNDED,
+        title_style='bold',
+    )
+
+    running_icon = mrsm.get_config('formatting', 'emoji', 'running')
+    paused_icon = mrsm.get_config('formatting', 'emoji', 'paused')
+    stopped_icon = mrsm.get_config('formatting', 'emoji', 'stopped')
+    locked_icon = mrsm.get_config('formatting', 'emoji', 'locked')
+    unlocked_icon = mrsm.get_config('formatting', 'emoji', 'unlocked')
+    status_styles = {
+        'running': 'green',
+        'stopped': 'red',
+        'paused': 'yellow',
+    }
+
+    for worker in workers:
+        status = worker.job.status
+        status_text = rich_text.Text(
+            status,
+            style=status_styles[status],
+        )
+        if not worker.is_ready():
+            ready_icon = stopped_icon
+        else:
+            ready_icon = (
+                running_icon
+                if status == 'running'
+                else paused_icon
+            )
+
+        table.add_row(
+            str(worker.ix),
+            status_text,
+            (locked_icon if worker.lock_path.exists() else unlocked_icon),
+            ready_icon,
+            str(worker.job.daemon.rotating_log.get_latest_subfile_index()),
+        )
+
+    mrsm.pprint(table)
+    return True, "Success"
+
+
 ### NOTE: This must be the final statement of the module.
 ###       Any subactions added below these lines will not
 ###       be added to the `help` docstring.

meerschaum/actions/start.py

@@ -7,6 +7,8 @@ Start subsystems (API server, logging daemon, etc.).
 """
 
 from __future__ import annotations
+
+import meerschaum as mrsm
 from meerschaum.utils.typing import SuccessTuple, Optional, List, Any, Union, Dict
 
 
@@ -25,6 +27,7 @@ def start(
         'webterm': _start_webterm,
         'connectors': _start_connectors,
         'pipeline': _start_pipeline,
+        'daemons': _start_daemons,
     }
     return choose_subaction(action, options, **kw)
 
@@ -308,6 +311,7 @@ def _start_gui(
     action: Optional[List[str]] = None,
     mrsm_instance: Optional[str] = None,
     port: Optional[int] = None,
+    webterm_port: Optional[int] = None,
     debug: bool = False,
     **kw
 ) -> SuccessTuple:
@@ -321,7 +325,6 @@ def _start_gui(
 
     from meerschaum.utils.venv import venv_exec
     from meerschaum.utils.packages import attempt_import
-    from meerschaum.utils.warnings import warn
     from meerschaum.utils.debug import dprint
     from meerschaum.utils.networking import find_open_ports
     from meerschaum.connectors.parse import parse_instance_keys
@@ -329,9 +332,10 @@ def _start_gui(
     webview, requests = attempt_import('webview', 'requests')
 
     success, msg = True, "Success"
-    host = '127.0.0.1'
-    if port is None:
-        port = 8765
+    host = mrsm.get_config('webterm', 'host')
+    port = port or webterm_port
+    if not port:
+        port = mrsm.get_config('webterm', 'port')
 
     if not is_webterm_running(host, port, session_id='mrsm'):
         port = next(find_open_ports(port, 9000))
@@ -369,7 +373,7 @@ def _start_gui(
                 break
         except Exception as e:
             if debug:
-                dprint(e)
+                dprint(str(e))
             continue
     if starting_up is False:
         return False, f"The webterm failed to start within {timeout} seconds."
@@ -417,7 +421,6 @@ def _start_webterm(
         - `-i`, '--instance'
             The default instance to use for the Webterm shell.
     """
-    import uuid
     from meerschaum._internal.term import get_webterm_app_and_manager, tornado_ioloop
     from meerschaum._internal.term.tools import (
         is_webterm_running,
@@ -426,15 +429,17 @@ def _start_webterm(
     )
     from meerschaum.utils.networking import find_open_ports
     from meerschaum.utils.warnings import info
+    from meerschaum.config.paths import WEBTERM_INTERNAL_RESOURCES_PATH
 
     if host is None:
-        host = '127.0.0.1'
+        host = mrsm.get_config('api', 'webterm', 'host')
     if port is None:
-        port = 8765
+        port = mrsm.get_config('api', 'webterm', 'port')
     if sysargs is None:
         sysargs = ['start', 'webterm']
     session_id = 'mrsm'
-    tornado_app, term_manager = get_webterm_app_and_manager(instance_keys=mrsm_instance)
+
+    env_path = WEBTERM_INTERNAL_RESOURCES_PATH / (str(port) + '.json')
 
     if is_webterm_running(host, port, session_id=session_id):
         if force:
@@ -445,11 +450,18 @@ def _start_webterm(
                 + "    Include `-f` to start another server on a new port\n"
                 + "    or specify a different port with `-p`."
             )
+
+    tornado_app, term_manager = get_webterm_app_and_manager(
+        instance_keys=mrsm_instance,
+        port=port,
+        env_path=env_path,
+    )
     if not nopretty:
         info(
             f"Starting the webterm at http://{host}:{port}/webterm/{session_id} ..."
             "\n    Press CTRL+C to quit."
         )
+
     tornado_app.listen(port, host)
     loop = tornado_ioloop.IOLoop.instance()
     try:
@@ -462,10 +474,15 @@ def _start_webterm(
         term_manager.shutdown()
         loop.close()
 
-    sessions = get_mrsm_tmux_sessions()
+    sessions = get_mrsm_tmux_sessions(port=port)
     for session in sessions:
         kill_tmux_session(session)
 
+    try:
+        env_path.unlink()
+    except Exception:
+        pass
+
     return True, "Success"
 
 
@@ -483,7 +500,6 @@ def _start_connectors(
     from meerschaum.connectors.parse import parse_instance_keys
     from meerschaum.utils.pool import get_pool
     from meerschaum.utils.warnings import warn
-    from meerschaum.utils.formatting import pprint
     from meerschaum.utils.misc import items_str
     if action is None:
         action = []
@@ -575,7 +591,6 @@ def _start_pipeline(
     """
     import json
     import time
-    import sys
     from meerschaum._internal.entry import entry
     from meerschaum.utils.warnings import info, warn
     from meerschaum.utils.misc import is_int
@@ -630,7 +645,7 @@ def _start_pipeline(
     def do_entry() -> None:
         nonlocal success, msg, proc
         if timeout_seconds is None:
-            success, msg = entry(sub_args_line, _patch_args=patch_args)
+            success, msg = entry(sub_args_line, _patch_args=patch_args, _use_cli_daemon=False)
             return
 
         sub_args_line_escaped = sub_args_line.replace("'", "<QUOTE>")
@@ -640,7 +655,7 @@ def _start_pipeline(
             "from meerschaum._internal.entry import entry\n\n"
             f"sub_args_line = '{sub_args_line_escaped}'.replace(\"<QUOTE>\", \"'\")\n"
             f"patch_args = json.loads('{patch_args_escaped_str}'.replace('<QUOTE>', \"'\"))\n"
-            "success, msg = entry(sub_args_line, _patch_args=patch_args)\n"
+            "success, msg = entry(sub_args_line, _patch_args=patch_args, _use_cli_daemon=False)\n"
             f"print('{fence_begin}' + json.dumps((success, msg)) + '{fence_end}')"
         )
         proc = venv_exec(src, venv=None, as_proc=True)
@@ -688,6 +703,112 @@ def _start_pipeline(
     return success, msg
 
 
+def _start_daemons(
+    timeout_seconds: Union[int, float, None] = None,
+    yes: bool = False,
+    force: bool = False,
+    noask: bool = False,
+    debug: bool = False,
+    **kwargs
+) -> SuccessTuple:
+    """
+    Start the Meerschaum CLI daemon processes.
+    """
+    from meerschaum.utils.warnings import warn, dprint
+    from meerschaum._internal.cli.daemons import (
+        get_cli_daemon,
+        get_cli_lock_path,
+    )
+    from meerschaum._internal.cli.workers import (
+        get_existing_cli_workers,
+        get_existing_cli_worker_indices,
+    )
+    from meerschaum.utils.prompt import yes_no
+    from meerschaum.actions import actions
+
+    workers = get_existing_cli_workers()
+    if not workers:
+        if debug:
+            dprint("No daemons are running, spawning a new process...")
+        workers = [get_cli_daemon()]
+
+    accepted_restart = False
+    any_daemons_are_running = any((worker.job.status == 'running') for worker in workers)
+    lock_paths = [get_cli_lock_path(ix) for ix in get_existing_cli_worker_indices()]
+    any_locks_exist = any(lock_path.exists() for lock_path in lock_paths)
+
+    if any_locks_exist:
+        warn(
+            "Locks are currently held by the CLI daemons.\n"
+            "Run again with `--force` to remove the locks.",
+            stack=False,
+        )
+
+        if not force:
+            return False, "Actions are currently running."
+
+        for lock_path in lock_paths:
+            try:
+                if lock_path.exists():
+                    lock_path.unlink()
+            except Exception as e:
+                warn(f"Failed to release lock:\n{e}")
+
+    if any_daemons_are_running:
+        accepted_restart = force or yes_no(
+            "Restart running CLI daemons?",
+            yes=yes,
+            noask=noask,
+            default='n',
+        )
+
+        if not accepted_restart:
+            return True, "Daemons are already running; nothing to do."
+
+        stop_success, stop_msg = actions['stop'](
+            ['daemons'],
+            timeout_seconds=timeout_seconds,
+            debug=debug,
+        )
+        if not stop_success:
+            return stop_success, stop_msg
+
+    worker = get_cli_daemon()
+
+    if debug:
+        dprint("Cleaning up CLI daemon...")
+    cleanup_success, cleanup_msg = worker.cleanup()
+    if not cleanup_success:
+        return cleanup_success, cleanup_msg
+
+    if debug:
+        dprint("Starting CLI daemon...")
+
+    start_success, start_msg = worker.job.start()
+    if not start_success:
+        return start_success, start_msg
+
+    return True, "Success"
+
+
+def _start_worker(action: Optional[List[str]] = None, **kwargs: Any) -> SuccessTuple:
+    """
+    Start a CLI worker process. This is intended for internal use only.
+    """
+    from meerschaum._internal.cli.workers import ActionWorker
+    from meerschaum.utils.misc import is_int
+
+    if not action:
+        return False, "No worker index is provided."
+
+    if not is_int(action[0]):
+        return False, "Invalid worker index."
+
+    ix = int(action[0])
+    worker = ActionWorker(ix)
+    return worker.run()
+
+
 ### NOTE: This must be the final statement of the module.
 ###       Any subactions added below these lines will not
 ###       be added to the `help` docstring.

meerschaum/actions/stop.py

@@ -7,7 +7,7 @@ Stop running jobs that were started with `-d` or `start job`.
 """
 
 from __future__ import annotations
-from meerschaum.utils.typing import Optional, List, SuccessTuple, Any
+from meerschaum.utils.typing import Optional, List, SuccessTuple, Any, Union
 
 
 def stop(action: Optional[List[str]] = None, **kw) -> SuccessTuple:
@@ -17,6 +17,7 @@ def stop(action: Optional[List[str]] = None, **kw) -> SuccessTuple:
     from meerschaum.actions import choose_subaction
     options = {
         'jobs': _stop_jobs,
+        'daemons': _stop_daemons,
     }
     return choose_subaction(action, options, **kw)
 
@@ -53,7 +54,7 @@ def _complete_stop(
         return options[sub](action=action, **kw)
 
     from meerschaum._internal.shell import default_action_completer
-    return default_action_completer(action=(['start'] + action), **kw)
+    return default_action_completer(action=(['stop'] + action), **kw)
 
 
 def _stop_jobs(
@@ -113,7 +114,7 @@ def _stop_jobs(
 
     if not jobs_to_stop:
         if jobs:
-            return True, "The selected jobs are currently running."
+            return True, "The selected jobs are currently stopped."
         return False, "No running, paused or restarting jobs to stop."
 
     if not action:
@@ -163,6 +164,38 @@ def _stop_jobs(
     return success, msg
 
 
+def _stop_daemons(
+    timeout_seconds: Union[int, float, None] = None,
+    debug: bool = False,
+    **kwargs
+) -> SuccessTuple:
+    """
+    Stop the Meerschaum CLI daemon.
+    """
+    import shutil
+    from meerschaum._internal.cli.workers import get_existing_cli_workers
+    from meerschaum.config.paths import CLI_RESOURCES_PATH
+    workers = get_existing_cli_workers()
+
+    for worker in workers:
+        stop_success, stop_msg = worker.job.stop(timeout_seconds=timeout_seconds, debug=debug)
+        if not stop_success:
+            return stop_success, stop_msg
+
+        cleanup_success, cleanup_msg = worker.cleanup(debug=debug)
+        if not cleanup_success:
+            return cleanup_success, cleanup_msg
+
+    try:
+        if CLI_RESOURCES_PATH.exists():
+            shutil.rmtree(CLI_RESOURCES_PATH)
+        CLI_RESOURCES_PATH.mkdir(parents=True, exist_ok=True)
+    except Exception as e:
+        return False, f"Failed to clean up CLI resources directory.\n{e}"
+
+    return True, "Success"
+
+
 ### NOTE: This must be the final statement of the module.
 ###       Any subactions added below these lines will not
 ###       be added to the `help` docstring.

meerschaum/actions/sync.py

@@ -289,6 +289,7 @@ def _sync_pipes(
     from meerschaum.utils.formatting import print_pipes_results
     from meerschaum._internal.static import STATIC_CONFIG
     from meerschaum.utils.misc import interval_str
+    from meerschaum.utils.daemon import running_in_daemon
 
     noninteractive_val = os.environ.get(STATIC_CONFIG['environment']['noninteractive'], None)
     noninteractive = str(noninteractive_val).lower() in ('1', 'true', 'yes')
@@ -301,7 +302,11 @@ def _sync_pipes(
     cooldown = 2 * (min_seconds + 1)
     success_pipes, failure_pipes = [], []
     while run:
-        _progress = progress() if shell and not noninteractive else None
+        _progress = (
+            progress()
+            if (shell and not noninteractive and not running_in_daemon())
+            else None
+        )
         cm = _progress if _progress is not None else contextlib.nullcontext()
 
         lap_begin = time.perf_counter()