meerschaum 2.9.4__py3-none-any.whl → 3.0.0__py3-none-any.whl

meerschaum/__init__.py

@@ -3,7 +3,7 @@
 # vim:fenc=utf-8
 
 """
-Copyright 2023 Bennett Meares
+Copyright 2025 Bennett Meares
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -19,13 +19,14 @@ limitations under the License.
 """
 
 import atexit
+
 from meerschaum.utils.typing import SuccessTuple
 from meerschaum.utils.packages import attempt_import
 from meerschaum.core.Pipe import Pipe
 from meerschaum.plugins import Plugin
 from meerschaum.utils.venv import Venv
 from meerschaum.jobs import Job, make_executor
-from meerschaum.connectors import get_connector, Connector, make_connector
+from meerschaum.connectors import get_connector, Connector, InstanceConnector, make_connector
 from meerschaum.utils import get_pipes
 from meerschaum.utils.formatting import pprint
 from meerschaum._internal.docs import index as __doc__
@@ -42,6 +43,7 @@ __all__ = (
     "get_config",
     "Pipe",
     "Plugin",
+    "SuccessTuple",
     "Venv",
     "Plugin",
     "Job",
@@ -55,6 +57,7 @@ __all__ = (
     "utils",
     "SuccessTuple",
     "Connector",
+    "InstanceConnector",
     "make_connector",
     "entry",
 )

meerschaum/_internal/__init__.py

@@ -6,3 +6,4 @@
 Modules, classes, and variables declared in _internal are not intended for general public use.
 """
 
+from meerschaum._internal.static import STATIC_CONFIG

meerschaum/_internal/arguments/_parse_arguments.py

@@ -23,7 +23,7 @@ def split_pipeline_sysargs(sysargs: List[str]) -> Tuple[List[str], List[str]]:
     """
     Split `sysargs` into the main pipeline and the flags following the pipeline separator (`:`).
     """
-    from meerschaum.config.static import STATIC_CONFIG
+    from meerschaum._internal.static import STATIC_CONFIG
     pipeline_key = STATIC_CONFIG['system']['arguments']['pipeline_key']
     if pipeline_key not in sysargs:
         return sysargs, []
@@ -40,7 +40,7 @@ def split_chained_sysargs(sysargs: List[str]) -> List[List[str]]:
     Split a `sysargs` list containing "and" keys (`+`)
     into a list of individual `sysargs`.
     """
-    from meerschaum.config.static import STATIC_CONFIG
+    from meerschaum._internal.static import STATIC_CONFIG
     and_key = STATIC_CONFIG['system']['arguments']['and_key']
 
     if not sysargs or and_key not in sysargs:
@@ -90,7 +90,7 @@ def parse_arguments(sysargs: List[str]) -> Dict[str, Any]:
 
     """
     import shlex
-    from meerschaum.config.static import STATIC_CONFIG
+    from meerschaum._internal.static import STATIC_CONFIG
     from meerschaum._internal.arguments._parser import parser
 
     global _loaded_plugins_args
@@ -360,7 +360,7 @@ def remove_leading_action(
     """
     from meerschaum.actions import get_action, get_main_action_name
     from meerschaum.utils.warnings import warn
-    from meerschaum.config.static import STATIC_CONFIG
+    from meerschaum._internal.static import STATIC_CONFIG
     action_function = get_action(action, _actions=_actions)
     if action_function is None:
         return action

meerschaum/_internal/arguments/_parser.py

@@ -40,7 +40,8 @@ class ArgumentParser(argparse.ArgumentParser):
 
 def parse_datetime(dt_str: str) -> Union[datetime, int, str]:
     """Parse a string into a datetime."""
-    from meerschaum.utils.misc import is_int, round_time
+    from meerschaum.utils.dtypes import round_time
+    from meerschaum.utils.misc import is_int
     if is_int(dt_str):
         return int(dt_str)
 
@@ -139,7 +140,11 @@ def parse_help(sysargs: Union[List[str], Dict[str, Any]]) -> None:
     ### Check for subactions.
     if len(args['action']) > 1:
         try:
-            subaction = get_subactions(args['action'][0])[args['action'][1]]
+            subactions = get_subactions(args['action'][0])
+            subaction_name = args['action'][1]
+            if subaction_name not in subactions:
+                subaction_name = subaction_name + 's'
+            subaction = subactions[subaction_name]
         except Exception:
             subaction = None
         if subaction is not None:
@@ -195,6 +200,7 @@ groups['pipes'] = parser.add_argument_group(title='Pipes options')
 groups['sync'] = parser.add_argument_group(title='Sync options')
 groups['api'] = parser.add_argument_group(title='API options')
 groups['plugins'] = parser.add_argument_group(title='Plugins options')
+groups['tokens'] = parser.add_argument_group(title='Tokens options')
 groups['packages'] = parser.add_argument_group(title='Packages options')
 groups['debug'] = parser.add_argument_group(title='Debugging options')
 groups['misc'] = parser.add_argument_group(title='Miscellaneous options')
@@ -375,7 +381,7 @@ groups['sync'].add_argument(
 groups['sync'].add_argument(
     '--cache', action='store_true',
     help=(
-        "When syncing or viewing a pipe's data, sync to a local database for later analysis."
+        "Sync pipes' metadata to disk."
     )
 )
 
@@ -387,13 +393,21 @@ groups['api'].add_argument(
     '--host', type=str,
     help="The host address to bind to for the API server. Defaults to '0.0.0.0'."
 )
+groups['api'].add_argument(
+    '--webterm-port', type=int,
+    help="The port on which to run the webterm server.",
+)
 groups['api'].add_argument(
     '-w', '--workers', type=int,
     help = "How many workers to run a concurrent action (e.g. running the API or syncing pipes)"
 )
 groups['api'].add_argument(
     '--no-dash', '--nodash', action='store_true',
-    help = 'When starting the API, do not start the Web interface.',
+    help = 'When starting the API, do not start the Web Console dashboard.',
+)
+groups['api'].add_argument(
+    '--no-webterm', '--nowebterm', action='store_true',
+    help="When starting the API, do not start the Webterm.",
 )
 groups['api'].add_argument(
     '--private', '--private-mode', action='store_true',
@@ -419,12 +433,23 @@ groups['api'].add_argument(
     '--certfile', type=str,
     help = "Start the API server with this certfile (requires --keyfile).",
 )
+
 ### Plugins options
 groups['plugins'].add_argument(
     '-r', '--repository', '--repo', type=str,
     help="Meerschaum plugins repository to connect to. Specify an API label (default: 'mrsm')"
 )
 
+### Tokens options
+groups['tokens'].add_argument(
+    '--ttl-days', type=int,
+    help="For how many days should a token be valid.",
+)
+groups['tokens'].add_argument(
+    '--scopes', '--scope', nargs='+',
+    help="Which scope permissions to grant to a token.",
+)
+
 ### Packages options
 groups['packages'].add_argument(
     '--venv', type=str,
@@ -491,3 +516,7 @@ groups['misc'].add_argument(
         + "Default behavior is to only allow recognized actions."
     )
 )
+groups['misc'].add_argument(
+    '--no-daemon', action='store_true',
+    help="Do not use the Meerschaum CLI daemon to execute the action.",
+)

meerschaum/_internal/cli/__init__.py

@@ -0,0 +1,6 @@
+#! /usr/bin/env python3
+# vim:fenc=utf-8
+
+"""
+Define internal utilities for interacting with the CLI daemons.
+"""

meerschaum/_internal/cli/daemons.py

@@ -0,0 +1,103 @@
+#! /usr/bin/env python3
+# vim:fenc=utf-8
+
+"""
+Define the CLI daemons utilities.
+"""
+
+import os
+import threading
+import pathlib
+from typing import Optional, List
+
+import meerschaum as mrsm
+
+
+def get_cli_daemon(ix: Optional[int] = None):
+    """
+    Get the CLI daemon.
+    """
+    from meerschaum._internal.cli.workers import ActionWorker
+    ix = ix if ix is not None else get_available_cli_daemon_ix()
+    return ActionWorker(ix)
+
+
+def get_cli_lock_path(ix: int) -> pathlib.Path:
+    """
+    Return the path to a CLI daemon's lock file.
+    """
+    from meerschaum.config.paths import CLI_RESOURCES_PATH
+    return CLI_RESOURCES_PATH / f"ix-{ix}.lock"
+
+
+def get_cli_session_id() -> str:
+    """
+    Return the session ID to use for the current process and thread.
+    """
+    return f"{os.getpid()}.{threading.current_thread().ident}"
+
+
+def get_available_cli_daemon_ix() -> int:
+    """
+    Return the index for an available CLI daemon.
+    """
+    max_ix = mrsm.get_config('system', 'cli', 'max_daemons') - 1
+    ix = 0
+    while True:
+        lock_path = get_cli_lock_path(ix)
+        if not lock_path.exists():
+            return ix
+        
+        ix += 1
+        if ix > max_ix:
+            raise EnvironmentError("Too many CLI daemons are running.")
+
+
+def get_existing_cli_daemon_indices() -> List[int]:
+    """
+    Return a list of the existing CLI daemons.
+    """
+    from meerschaum.utils.daemon import get_daemon_ids
+    daemon_ids = [daemon_id for daemon_id in get_daemon_ids() if daemon_id.startswith('.cli.')]
+    indices = []
+
+    for daemon_id in daemon_ids:
+        try:
+            ix = int(daemon_id[len('.cli.'):])
+            indices.append(ix)
+        except Exception:
+            pass
+
+    return indices
+
+
+def get_existing_cli_daemons() -> 'List[ActionWorker]':
+    """
+    Return a list of the existing CLI daemons.
+    """
+    from meerschaum._internal.cli.workers import ActionWorker
+    indices = get_existing_cli_daemon_indices()
+    return [
+        ActionWorker(ix)
+        for ix in indices
+    ]
+
+
+def _get_cli_session_dir_path(session_id: str) -> pathlib.Path:
+    """
+    Return the path to the file handles for the CLI session.
+    """
+    from meerschaum.config.paths import CLI_RESOURCES_PATH
+    return CLI_RESOURCES_PATH / session_id
+
+
+def _get_cli_stream_path(
+    session_id: str,
+    action_id: str,
+    stream: str = 'stdout',
+) -> pathlib.Path:
+    """
+    Return the file path for the stdout / stderr file stream.
+    """
+    session_dir_path = _get_cli_session_dir_path(session_id)
+    return session_dir_path / f'{action_id}.{stream}'

meerschaum/_internal/cli/entry.py

@@ -0,0 +1,220 @@
+#! /usr/bin/env python3
+# vim:fenc=utf-8
+
+"""
+Define the CLI daemon entrypoint.
+"""
+
+import os
+import time
+import uuid
+import shlex
+import shutil
+import traceback
+import signal
+from typing import Optional, Dict, List, Any
+
+import meerschaum as mrsm
+
+
+def entry_with_daemon(
+    sysargs: Optional[List[str]] = None,
+    _patch_args: Optional[Dict[str, Any]] = None,
+    _use_cli_daemon: bool = True,
+    _session_id: Optional[str] = None,
+) -> mrsm.SuccessTuple:
+    """
+    Parse arguments and launch a Meerschaum action.
+
+    Returns
+    -------
+    A `SuccessTuple` indicating success.
+    """
+    from meerschaum.actions import get_action
+    from meerschaum.plugins import load_plugins, _actions_daemon_enabled
+    from meerschaum._internal.entry import entry_without_daemon
+    from meerschaum._internal.cli.workers import ActionWorker
+    from meerschaum._internal.cli.daemons import (
+        get_available_cli_daemon_ix,
+        get_cli_session_id,
+    )
+    from meerschaum.config import get_possible_keys
+    from meerschaum._internal.arguments import split_pipeline_sysargs, split_chained_sysargs
+    daemon_is_ready = True
+
+    load_plugins(skip_if_loaded=True)
+
+    found_acceptable_prefix = False
+    found_unacceptable_prefix = False
+    found_disabled_action = False
+    allowed_prefixes = (
+        mrsm.get_config('system', 'cli', 'allowed_prefixes')
+    )
+    disallowed_prefixes = (
+        mrsm.get_config('system', 'cli', 'disallowed_prefixes')
+    )
+    refresh_seconds = mrsm.get_config('system', 'cli', 'refresh_seconds')
+    sysargs_str = sysargs if isinstance(sysargs, str) else shlex.join(sysargs or [])
+    debug = ' --debug' in sysargs_str
+    _sysargs = shlex.split(sysargs_str)
+    _sysargs, _pipeline_args = split_pipeline_sysargs(_sysargs)
+    _chained_sysargs = split_chained_sysargs(_sysargs)
+    _action_functions = {
+        _action_func.__name__: _action_func
+        for _step_sysargs in _chained_sysargs
+        if (_action_func := get_action(_step_sysargs))
+    }
+    for action_name, enabled in _actions_daemon_enabled.items():
+        if action_name not in _action_functions:
+            continue
+        if enabled:
+            continue
+        found_disabled_action = True
+        break
+
+    for prefix in allowed_prefixes:
+        if sysargs_str.startswith(prefix) or prefix == '*':
+            found_acceptable_prefix = True
+            break
+
+    for prefix in disallowed_prefixes:
+        if sysargs_str.startswith(prefix) or prefix == '*':
+            found_unacceptable_prefix = True
+            break
+
+    if not found_acceptable_prefix or found_unacceptable_prefix or found_disabled_action:
+        daemon_is_ready = False
+
+    try:
+        daemon_ix = get_available_cli_daemon_ix() if daemon_is_ready else -1
+    except EnvironmentError as e:
+        from meerschaum.utils.warnings import warn
+        warn(e, stack=False)
+        daemon_ix = -1
+        daemon_is_ready = False
+
+    worker = ActionWorker(daemon_ix, refresh_seconds=refresh_seconds) if daemon_ix != -1 else None
+    if worker and worker.lock_path.exists():
+        daemon_is_ready = False
+
+    start_success, start_msg = (
+        worker.job.start()
+        if worker and daemon_is_ready
+        else (False, "Lock exists.")
+    )
+
+    if not start_success:
+        daemon_is_ready = False
+
+    if start_success and worker:
+        start = time.perf_counter()
+        while (time.perf_counter() - start) < 3:
+            if worker.is_ready():
+                break
+            time.sleep(refresh_seconds)
+
+    if not daemon_is_ready or worker is None:
+        if debug:
+            print("Revert to entry without daemon.")
+        return entry_without_daemon(sysargs, _patch_args=_patch_args)
+
+    session_id = _session_id or get_cli_session_id()
+    action_id = uuid.uuid4().hex
+
+    terminal_size = shutil.get_terminal_size()
+    env = {
+        **{
+            'LINES': str(terminal_size.lines),
+            'COLUMNS': str(terminal_size.columns),
+        },
+        **dict(os.environ),
+    }
+    for key in get_possible_keys():
+        _ = mrsm.get_config(key)
+    config = mrsm.get_config()
+    
+    worker.write_input_data({
+        'session_id': session_id,
+        'action_id': action_id,
+        'sysargs': sysargs,
+        'patch_args': _patch_args,
+        'env': env,
+        'config': config,
+    })
+
+    accepted = False
+    exit_data = None
+    worker_data = None
+
+    while not accepted:
+        state = worker.read_output_data().get('state', None)
+        if state == 'accepted':
+            accepted = True
+            break
+
+        time.sleep(refresh_seconds)
+
+    worker.start_cli_logs_refresh_thread()
+
+    try:
+        log = worker.job.daemon.rotating_log
+        worker.job.monitor_logs(
+            stop_on_exit=True,
+            callback_function=worker.monitor_callback,
+            _log=log,
+            _wait_if_stopped=False,
+        )
+        worker_data = worker.read_output_data()
+    except KeyboardInterrupt:
+        exit_data = {
+            'session_id': session_id,
+            'action_id': action_id,
+            'signal': signal.SIGINT,
+            'traceback': traceback.format_exc(),
+            'success': True,
+            'message': 'Exiting due to keyboard interrupt.',
+        }
+    except Exception as e:
+        exit_data = {
+            'session_id': session_id,
+            'action_id': action_id,
+            'signal': signal.SIGINT,
+            'traceback': traceback.format_exc(),
+            'success': False,
+            'message': f'Encountered exception: {e}',
+        }
+    except SystemExit:
+        exit_data = {
+            'session_id': session_id,
+            'action_id': action_id,
+            'signal': signal.SIGTERM,
+            'traceback': traceback.format_exc(),
+            'success': True,
+            'message': 'Exiting on SIGTERM.',
+        }
+    except BrokenPipeError:
+        return False, "Connection to daemon is broken."
+
+    if exit_data:
+        exit_success = bool(exit_data['success'])
+        exit_signal = exit_data['signal']
+        worker.send_signal(exit_signal)
+        try:
+            worker.job.monitor_logs(
+                stop_on_exit=True,
+                callback_function=worker.monitor_callback,
+                _wait_if_stopped=False,
+                _log=log,
+            )
+        except (KeyboardInterrupt, SystemExit):
+            worker.send_signal(signal.SIGTERM)
+        worker_data = worker.read_output_data()
+
+        if not exit_success:
+            print(exit_data['traceback'])
+
+    worker.stop_cli_logs_refresh_thread()
+    worker.write_input_data({'increment': True})
+    success = (worker_data or {}).get('success', False)
+    message = (worker_data or {}).get('message', "Failed to retrieve message from CLI worker.")
+    return success, message