meerschaum 3.0.0rc6__tar.gz → 3.0.0rc7__tar.gz

{meerschaum-3.0.0rc6/meerschaum.egg-info → meerschaum-3.0.0rc7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: meerschaum
-Version: 3.0.0rc6
+Version: 3.0.0rc7
 Summary: Sync Time-Series Pipes with Meerschaum
 Author-email: Bennett Meares <bennett.meares@gmail.com>
 Maintainer-email: Bennett Meares <bennett.meares@gmail.com>

{meerschaum-3.0.0rc6 → meerschaum-3.0.0rc7}/meerschaum/_internal/arguments/_parser.py

@@ -140,7 +140,11 @@ def parse_help(sysargs: Union[List[str], Dict[str, Any]]) -> None:
     ### Check for subactions.
     if len(args['action']) > 1:
         try:
-            subaction = get_subactions(args['action'][0])[args['action'][1]]
+            subactions = get_subactions(args['action'][0])
+            subaction_name = args['action'][1]
+            if subaction_name not in subactions:
+                subaction_name = subaction_name + 's'
+            subaction = subactions[subaction_name]
         except Exception:
             subaction = None
         if subaction is not None:
@@ -389,6 +393,10 @@ groups['api'].add_argument(
     '--host', type=str,
     help="The host address to bind to for the API server. Defaults to '0.0.0.0'."
 )
+groups['api'].add_argument(
+    '--webterm-port', type=int,
+    help="The port on which to run the webterm server.",
+)
 groups['api'].add_argument(
     '-w', '--workers', type=int,
     help = "How many workers to run a concurrent action (e.g. running the API or syncing pipes)"

{meerschaum-3.0.0rc6 → meerschaum-3.0.0rc7}/meerschaum/_internal/cli/entry.py

@@ -12,7 +12,7 @@ import shlex
 import shutil
 import traceback
 import signal
-from typing import Optional, Dict, List, Any, Union
+from typing import Optional, Dict, List, Any
 
 import meerschaum as mrsm
 
@@ -30,20 +30,47 @@ def entry_with_daemon(
     -------
     A `SuccessTuple` indicating success.
     """
+    from meerschaum.actions import get_action
+    from meerschaum.plugins import load_plugins, _actions_daemon_enabled
     from meerschaum._internal.entry import entry_without_daemon
     from meerschaum._internal.cli.workers import ActionWorker
     from meerschaum._internal.cli.daemons import (
         get_available_cli_daemon_ix,
         get_cli_session_id,
     )
+    from meerschaum.config import get_possible_keys
+    from meerschaum._internal.arguments import split_pipeline_sysargs, split_chained_sysargs
     daemon_is_ready = True
 
+    load_plugins(skip_if_loaded=True)
+
     found_acceptable_prefix = False
     found_unacceptable_prefix = False
-    allowed_prefixes = mrsm.get_config('system', 'cli', 'allowed_prefixes')
-    disallowed_prefixes = mrsm.get_config('system', 'cli', 'disallowed_prefixes')
+    found_disabled_action = False
+    allowed_prefixes = (
+        mrsm.get_config('system', 'cli', 'allowed_prefixes')
+    )
+    disallowed_prefixes = (
+        mrsm.get_config('system', 'cli', 'disallowed_prefixes')
+    )
     refresh_seconds = mrsm.get_config('system', 'cli', 'refresh_seconds')
     sysargs_str = sysargs if isinstance(sysargs, str) else shlex.join(sysargs or [])
+    debug = ' --debug' in sysargs_str
+    _sysargs = shlex.split(sysargs_str)
+    _sysargs, _pipeline_args = split_pipeline_sysargs(_sysargs)
+    _chained_sysargs = split_chained_sysargs(_sysargs)
+    _action_functions = {
+        _action_func.__name__: _action_func
+        for _step_sysargs in _chained_sysargs
+        if (_action_func := get_action(_step_sysargs))
+    }
+    for action_name, enabled in _actions_daemon_enabled.items():
+        if action_name not in _action_functions:
+            continue
+        if enabled:
+            continue
+        found_disabled_action = True
+        break
 
     for prefix in allowed_prefixes:
         if sysargs_str.startswith(prefix) or prefix == '*':
@@ -55,7 +82,7 @@ def entry_with_daemon(
             found_unacceptable_prefix = True
             break
 
-    if not found_acceptable_prefix or found_unacceptable_prefix:
+    if not found_acceptable_prefix or found_unacceptable_prefix or found_disabled_action:
         daemon_is_ready = False
 
     try:
@@ -87,6 +114,8 @@ def entry_with_daemon(
             time.sleep(refresh_seconds)
 
     if not daemon_is_ready or worker is None:
+        if debug:
+            print("Revert to entry without daemon.")
         return entry_without_daemon(sysargs, _patch_args=_patch_args)
 
     session_id = _session_id or get_cli_session_id()
@@ -100,6 +129,9 @@ def entry_with_daemon(
         },
         **dict(os.environ),
     }
+    for key in get_possible_keys():
+        _ = mrsm.get_config(key)
+    config = mrsm.get_config()
     
     worker.write_input_data({
         'session_id': session_id,
@@ -107,6 +139,7 @@ def entry_with_daemon(
         'sysargs': sysargs,
         'patch_args': _patch_args,
         'env': env,
+        'config': config,
     })
 
     accepted = False
@@ -121,6 +154,8 @@ def entry_with_daemon(
 
         time.sleep(refresh_seconds)
 
+    worker.start_cli_logs_refresh_thread()
+
     try:
         log = worker.job.daemon.rotating_log
         worker.job.monitor_logs(
@@ -178,6 +213,7 @@ def entry_with_daemon(
         if not exit_success:
             print(exit_data['traceback'])
 
+    worker.stop_cli_logs_refresh_thread()
     worker.write_input_data({'increment': True})
     success = (worker_data or {}).get('success', False)
     message = (worker_data or {}).get('message', "Failed to retrieve message from CLI worker.")

{meerschaum-3.0.0rc6 → meerschaum-3.0.0rc7}/meerschaum/_internal/cli/workers.py

@@ -7,15 +7,15 @@ Define utilities for managing the workers jobs.
 
 import os
 import pathlib
-import time
 import json
 import asyncio
+import time
 from typing import List, Dict, Any, Union, TextIO
 
 import meerschaum as mrsm
-from meerschaum.utils.daemon import StdinFile, RotatingFile
 from meerschaum.utils.warnings import warn
 from meerschaum.jobs import Job
+from meerschaum.utils.threading import Thread
 from meerschaum._internal.static import STATIC_CONFIG
 
 STOP_TOKEN: str = STATIC_CONFIG['jobs']['stop_token']
@@ -57,6 +57,7 @@ class ActionWorker:
             if refresh_seconds is not None
             else mrsm.get_config('system', 'cli', 'refresh_seconds')
         )
+        self.refresh_logs_stop_event = asyncio.Event()
 
     @property
     def input_file_path(self) -> pathlib.Path:
@@ -173,7 +174,7 @@ class ActionWorker:
         except Exception as e:
             warn(f"Failed to release lock for {self}:\n{e}")
 
-    def send_signal(self, signalnum: int):
+    def send_signal(self, signalnum):
         """
         Send a signal to the running job.
         """
@@ -292,9 +293,8 @@ class ActionWorker:
         Run the worker's process loop.
         """
         from meerschaum._internal.entry import entry
-        from meerschaum.utils.misc import set_env
         from meerschaum.config import replace_config
-        from meerschaum.config._environment import apply_environment_uris, apply_environment_patches
+        from meerschaum.config.environment import replace_env
 
         self.create_fifos()
 
@@ -318,16 +318,15 @@ class ActionWorker:
             action_id = input_data.get('action_id', None)
             patch_args = input_data.get('patch_args', None)
             env = input_data.get('env', {})
+            config = input_data.get('config', {})
             self.write_output_data({
                 'state': 'accepted',
                 'session_id': session_id,
                 'action_id': action_id,
             })
 
-            with set_env(env):
-                with replace_config({}):
-                    apply_environment_patches(env=env)
-                    apply_environment_uris(env=env)
+            with replace_config(config):
+                with replace_env(env):
                     action_success, action_msg = entry(
                         sysargs,
                         _use_cli_daemon=False,
@@ -373,6 +372,35 @@ class ActionWorker:
 
         return True, "Success"
 
+    def touch_cli_logs_loop(self):
+        """
+        Touch the CLI daemon's logs to refresh the logs monitoring.
+        """
+        while not self.refresh_logs_stop_event.is_set():
+            self.job.daemon.rotating_log.touch()
+            time.sleep(self.refresh_seconds)
+
+    def start_cli_logs_refresh_thread(self):
+        """
+        Spin up a daemon thread to refresh the CLI's logs.
+        """
+        self._logs_refresh_thread = Thread(
+            target=self.touch_cli_logs_loop,
+            daemon=True,
+        )
+        self._logs_refresh_thread.start()
+
+    def stop_cli_logs_refresh_thread(self):
+        """
+        Stop the logs refresh thread.
+        """
+        self.refresh_logs_stop_event.set()
+        thread = self.__dict__.pop('_logs_refresh_thread', None)
+        if thread is None:
+            return
+
+        thread.join()
+
     def __repr__(self) -> str:
         return self.__str__()
 

{meerschaum-3.0.0rc6 → meerschaum-3.0.0rc7}/meerschaum/_internal/entry.py

@@ -65,7 +65,7 @@ def entry(
     sysargs_list = shlex.split(sysargs) if isinstance(sysargs, str) else sysargs
     if (
         not _use_cli_daemon
-        or not sysargs
+        or (not sysargs or (sysargs[0] and sysargs[0].startswith('-')))
         or '--no-daemon' in sysargs_list
         or '--daemon' in sysargs_list
         or '-d' in sysargs_list
@@ -357,7 +357,7 @@ def _do_action_wrapper(
         try:
             result = action_function(**filter_keywords(action_function, **kw))
         except Exception as e:
-            if kw.get('debug', False):
+            if kw.get('debug', True):
                 import traceback
                 traceback.print_exception(type(e), e, e.__traceback__)
             result = False, (

{meerschaum-3.0.0rc6 → meerschaum-3.0.0rc7}/meerschaum/_internal/shell/Shell.py

@@ -82,6 +82,7 @@ ESCAPED_AND_KEY: str = STATIC_CONFIG['system']['arguments']['escaped_and_key']
 PIPELINE_KEY: str = STATIC_CONFIG['system']['arguments']['pipeline_key']
 ESCAPED_PIPELINE_KEY: str = STATIC_CONFIG['system']['arguments']['escaped_pipeline_key']
 
+
 def _insert_shell_actions(
     _shell: Optional['Shell'] = None,
     actions: Optional[Dict[str, Callable[[Any], SuccessTuple]]] = None,
@@ -100,6 +101,10 @@ def _insert_shell_actions(
     _shell_class = _shell if _shell is not None else shell_pkg.Shell
 
     for a, f in actions.items():
+        existing_method = getattr(_shell_class, 'do_' + a, None)
+        if existing_method == f:
+            continue
+
         add_method_to_class(
             func = f,
             class_def = _shell_class,
@@ -114,6 +119,35 @@ def _insert_shell_actions(
         setattr(_shell_class, 'complete_' + a, completer)
 
 
+def _remove_shell_actions(
+    _shell: Optional['Shell'] = None,
+    actions: Optional[Dict[str, Callable[[Any], SuccessTuple]]] = None,
+) -> None:
+    """
+    Remove the actions added to the shell.
+    """
+    import meerschaum._internal.shell as shell_pkg
+    if actions is None:
+        from meerschaum.actions import actions as _actions
+        actions = _actions
+
+    _shell_class = _shell if _shell is not None else shell_pkg.Shell
+
+    for a, f in actions.items():
+        try:
+            delattr(_shell_class, 'do_' + a)
+        except AttributeError:
+            pass
+
+        if a in reserved_completers:
+            continue
+
+        try:
+            delattr(_shell_class, 'complete_' + a)
+        except AttributeError:
+            pass
+
+
 def _completer_wrapper(
     target: Callable[[Any], List[str]]
 ) -> Callable[['mrsm._internal.shell.Shell', str, str, int, int], Any]:
@@ -357,7 +391,7 @@ class Shell(cmd.Cmd):
             shell_attrs['instance'] = instance
             shell_attrs['instance_keys'] = remove_ansi(str(instance))
         if shell_attrs.get('repo_keys', None) is None:
-            shell_attrs['repo_keys'] = get_config('meerschaum', 'default_repository', patch=patch)
+            shell_attrs['repo_keys'] = get_config('meerschaum', 'repository', patch=patch)
         if shell_attrs.get('executor_keys', None) is None:
             shell_attrs['executor_keys'] = get_executor_keys_from_context()
 
@@ -622,7 +656,7 @@ class Shell(cmd.Cmd):
             if key == 'mrsm_instance':
                 default_value = get_config('meerschaum', 'instance')
             elif key == 'repository':
-                default_value = get_config('meerschaum', 'default_repository')
+                default_value = get_config('meerschaum', 'repository')
             elif key == 'executor_keys':
                 default_value = get_executor_keys_from_context()
             else:
@@ -871,7 +905,7 @@ class Shell(cmd.Cmd):
         """
         Temporarily set a default Meerschaum repository for the duration of the shell.
         The default repository (mrsm.io) is loaded from the Meerschaum configuraton file
-          (at keys 'meerschaum:default_repository').
+          (at keys 'meerschaum.repository').
         
         You can change the default repository with `edit config`.
         
@@ -902,7 +936,7 @@ class Shell(cmd.Cmd):
         except (IndexError, AttributeError):
             repo_keys = ''
         if repo_keys == '':
-            repo_keys = get_config('meerschaum', 'default_repository', patch=True)
+            repo_keys = get_config('meerschaum', 'repository', patch=True)
 
         conn = parse_repo_keys(repo_keys, debug=debug)
         if conn is None or not conn:
@@ -985,7 +1019,7 @@ class Shell(cmd.Cmd):
         show pipes -h
         ```
         """
-        from meerschaum.actions import actions
+        from meerschaum.actions import get_action
         from meerschaum._internal.arguments._parser import parse_help
         from meerschaum._internal.arguments._parse_arguments import parse_line
         import textwrap
@@ -994,12 +1028,15 @@ class Shell(cmd.Cmd):
             del args['action']
             shell_attrs['_actions']['show'](['actions'], **args)
             return ""
+
         if args['action'][0] not in shell_attrs['_actions']:
+            action_func = get_action(args['action'])
             try:
-                print(textwrap.dedent(getattr(self, f"do_{args['action'][0]}").__doc__))
+                print(textwrap.dedent(action_func.__doc__))
             except Exception:
-                print(f"No help on '{args['action'][0]}'.")
+                print(f"No help on '{shlex.join(args['action'])}'.")
             return ""
+
         parse_help(args)
         return ""
 
@@ -1101,8 +1138,16 @@ def input_with_sigint(_input, session, shell: Optional[Shell] = None):
             return None
         if not shell_attrs['_update_bottom_toolbar'] and platform.system() == 'Windows':
             return shell_attrs['_old_bottom_toolbar']
-        size = os.get_terminal_size()
-        num_cols, num_lines = size.columns, size.lines
+        try:
+            size = os.get_terminal_size()
+            num_cols, num_lines = size.columns, size.lines
+        except Exception:
+            from meerschaum.utils.misc import is_int
+            num_cols, num_lines = os.environ.get('COLUMNS', 80), os.environ.get('LINES', 120)
+            if is_int(str(num_cols)):
+                num_cols = int(num_cols)
+            if is_int(str(num_lines)):
+                num_lines = int(num_lines)
         truncation_suffix = (
             '…'
             if UNICODE

{meerschaum-3.0.0rc6 → meerschaum-3.0.0rc7}/meerschaum/_internal/shell/__init__.py

@@ -8,5 +8,8 @@ Import the Shell class definition
 
 from meerschaum._internal.shell.Shell import Shell
 from meerschaum._internal.shell.Shell import (
-    default_action_completer, _completer_wrapper, _insert_shell_actions,
+    default_action_completer,
+    _completer_wrapper,
+    _insert_shell_actions,
+    _remove_shell_actions,
 )

{meerschaum-3.0.0rc6 → meerschaum-3.0.0rc7}/meerschaum/_internal/term/TermPageHandler.py

@@ -10,11 +10,10 @@ from __future__ import annotations
 from typing import Any
 
 import meerschaum as mrsm
-from meerschaum.utils.warnings import warn
 tornado_web = mrsm.attempt_import('tornado.web', lazy=False)
 terminado = mrsm.attempt_import('terminado', lazy=False)
 
-tmux_suffix = mrsm.get_config('system', 'webterm', 'tmux', 'session_suffix')
+tmux_suffix = mrsm.get_config('api', 'webterm', 'tmux', 'session_suffix')
 
 
 class TermPageHandler(tornado_web.RequestHandler):

{meerschaum-3.0.0rc6 → meerschaum-3.0.0rc7}/meerschaum/_internal/term/__init__.py

@@ -8,6 +8,9 @@ Build the web console virtual terminal using Tornado and xterm.js.
 
 from __future__ import annotations
 
+import os
+import json
+import pathlib
 from typing import Optional, Tuple
 
 import meerschaum as mrsm
@@ -16,7 +19,6 @@ from meerschaum._internal.term.TermPageHandler import TermPageHandler, CustomTer
 from meerschaum.config._paths import API_TEMPLATES_PATH, API_STATIC_PATH
 from meerschaum.utils.venv import venv_executable
 from meerschaum.utils.misc import is_tmux_available
-from meerschaum.utils.daemon._names import get_new_daemon_name
 
 tornado, tornado_ioloop, terminado = attempt_import(
     'tornado', 'tornado.ioloop', 'terminado', lazy=False,
@@ -25,6 +27,8 @@ tornado, tornado_ioloop, terminado = attempt_import(
 
 def get_webterm_app_and_manager(
     instance_keys: Optional[str] = None,
+    port: Optional[int] = None,
+    env_path: Optional[pathlib.Path] = None,
 ) -> Tuple[
     tornado.web.Application,
     terminado.UniqueTermManager,
@@ -36,19 +40,49 @@ def get_webterm_app_and_manager(
     -------
     A tuple of the Tornado web application and term manager.
     """
+    from meerschaum.config.environment import get_env_vars
+    from meerschaum._internal.static import STATIC_CONFIG
+    if env_path is None:
+        from meerschaum.config.paths import WEBTERM_INTERNAL_RESOURCES_PATH
+        env_path = WEBTERM_INTERNAL_RESOURCES_PATH / (str(port) + '.json')
+
+    env_vars = get_env_vars()
+    env_dict = {
+        env_var: os.environ[env_var]
+        for env_var in env_vars
+        if env_var not in (
+            STATIC_CONFIG['environment']['systemd_log_path'],
+            STATIC_CONFIG['environment']['systemd_result_path'],
+            STATIC_CONFIG['environment']['systemd_delete_job'],
+            STATIC_CONFIG['environment']['systemd_stdin_path'],
+            STATIC_CONFIG['environment']['daemon_id'],
+        )
+    }
+    with open(env_path, 'w+', encoding='utf-8') as f:
+        json.dump(env_dict, f)
+
     shell_kwargs_str = f"mrsm_instance='{instance_keys}'" if instance_keys else ""
     commands = [
         venv_executable(None),
         '-c',
-        "import os; _ = os.environ.pop('COLUMNS', None); _ = os.environ.pop('LINES', None); "
-        "from meerschaum._internal.entry import get_shell; "
+        "import os\n"
+        "import pathlib\n"
+        "import json\n"
+        f"env_path = pathlib.Path('{env_path.as_posix()}')\n"
+        "with open(env_path, 'r', encoding='utf-8') as f:\n"
+        "    env_dict = json.load(f)\n"
+        "os.environ.update(env_dict)\n"
+        "_ = os.environ.pop('COLUMNS', None)\n"
+        "_ = os.environ.pop('LINES', None)\n"
+        "from meerschaum._internal.entry import get_shell\n"
         f"get_shell({shell_kwargs_str}).cmdloop()"
     ]
-    webterm_cf = mrsm.get_config('system', 'webterm')
+    webterm_cf = mrsm.get_config('api', 'webterm')
     if webterm_cf.get('tmux', {}).get('enabled', False) and is_tmux_available():
-        commands = ['tmux', 'new-session', '-A', '-s', 'MRSM_SESSION'] + commands
+        commands = ['tmux', 'new-session', '-A', '-s', f'MRSM_SESSION--{port}'] + commands
 
-    term_manager = terminado.NamedTermManager(shell_command=commands)
+    term_manager = terminado.NamedTermManager(shell_command=commands, extra_env=env_dict)
+    term_manager._port = port
     handlers = [
         (
             r"/websocket/(.+)/?",

{meerschaum-3.0.0rc6 → meerschaum-3.0.0rc7}/meerschaum/_internal/term/tools.py

@@ -6,13 +6,14 @@
 Utility functions regarding the webterm.
 """
 
+from typing import List, Optional
+
 import meerschaum as mrsm
-from meerschaum.utils.typing import List
 
 
 def is_webterm_running(
-    host: str,
-    port: int,
+    host: Optional[str] = None,
+    port: Optional[int] = None,
     protocol: str = 'http',
     session_id: str = 'mrsm',
 ) -> int:
@@ -20,10 +21,12 @@ def is_webterm_running(
     Determine whether the webterm service is running on a given host and port.
     """
     requests = mrsm.attempt_import('requests', lazy=False)
+    host = host or mrsm.get_config('api', 'webterm', 'host')
+    port = port or mrsm.get_config('api', 'webterm', 'port')
     url = f'{protocol}://{host}:{port}/webterm/{session_id}'
     try:
         r = requests.get(url, timeout=3)
-    except Exception as e:
+    except Exception:
         return False
     if not r:
         return False
@@ -39,22 +42,44 @@ def kill_tmux_session(session: str) -> bool:
     return run_process(command, capture_output=True) == 0
 
 
-def get_mrsm_tmux_sessions() -> List[str]:
+def get_mrsm_tmux_sessions(port: Optional[int] = None) -> List[str]:
     """
     Return a list of tmux sessions created by Meerschaum.
     """
     from meerschaum.utils.process import run_process
-    tmux_suffix = mrsm.get_config('system', 'webterm', 'tmux', 'session_suffix')
+    tmux_suffix = mrsm.get_config('api', 'webterm', 'tmux', 'session_suffix')
     command = ['tmux', 'ls']
     proc = run_process(command, capture_output=True, as_proc=True)
     if proc.returncode != 0:
         return []
+
+    port = port or mrsm.get_config('api', 'webterm', 'port')
+
     sessions = [
         line.split(':', maxsplit=1)[0]
         for line in proc.stdout.read().decode('utf-8').split('\n')
     ]
+    mrsm_sessions_ports = []
+    for session in sessions:
+        if '--' not in session:
+            continue
+
+        parts = session.split('--', maxsplit=1)
+        if len(parts) != 2:
+            continue
+
+        if not parts[0].endswith(tmux_suffix):
+            continue
+
+        try:
+            session_port = int(parts[1])
+        except Exception:
+            continue
+
+        mrsm_sessions_ports.append((session, session_port))
+
     return [
         session
-        for session in sessions
-        if session.endswith(tmux_suffix)
+        for session, session_port in mrsm_sessions_ports
+        if session_port == port
     ]

{meerschaum-3.0.0rc6 → meerschaum-3.0.0rc7}/meerschaum/actions/__init__.py

@@ -9,7 +9,6 @@ Default actions available to the mrsm CLI.
 from __future__ import annotations
 from meerschaum.utils.typing import Callable, Any, Optional, Union, List, Dict, SuccessTuple
 from meerschaum.utils.packages import get_modules_from_package
-_custom_actions = []
 
 __all__ = (
     'get_action',
@@ -307,6 +306,8 @@ __all__ = ['actions', 'get_subactions', 'get_action', 'get_main_action_name', 'g
 ### functions that do not begin with '_' from all submodules.
 from inspect import getmembers, isfunction
 actions = {}
+_custom_actions_plugins: Dict[str, str] = {}
+_plugins_actions: Dict[str, List[str]] = {}
 
 for module in modules:
     ### A couple important things happening here:
@@ -362,4 +363,5 @@ __pdoc__ = {
 }
 for a in actions:
     __pdoc__[a] = False
-meerschaum.plugins.load_plugins()
+
+meerschaum.plugins.load_plugins(skip_if_loaded=True)