meerschaum 2.9.4__py3-none-any.whl → 3.0.0__py3-none-any.whl

meerschaum/actions/start.py

@@ -7,6 +7,8 @@ Start subsystems (API server, logging daemon, etc.).
 """
 
 from __future__ import annotations
+
+import meerschaum as mrsm
 from meerschaum.utils.typing import SuccessTuple, Optional, List, Any, Union, Dict
 
 
@@ -25,6 +27,7 @@ def start(
         'webterm': _start_webterm,
         'connectors': _start_connectors,
         'pipeline': _start_pipeline,
+        'daemons': _start_daemons,
     }
     return choose_subaction(action, options, **kw)
 
@@ -91,6 +94,7 @@ def _start_jobs(
     sysargs: Optional[List[str]] = None,
     executor_keys: Optional[str] = None,
     rm: bool = False,
+    line: Optional[str] = None,
     debug: bool = False,
     **kw
 ) -> SuccessTuple:
@@ -122,6 +126,7 @@ def _start_jobs(
           - `start job --name happy_seal`
                 Start the job 'happy_seal' but via the `--name` flag.
     """
+    import shlex
     from meerschaum.utils.warnings import warn, info
     from meerschaum.utils.daemon._names import get_new_daemon_name
     from meerschaum.jobs import (
@@ -212,7 +217,29 @@ def _start_jobs(
 
     def _run_new_job(name: Optional[str] = None):
         name = name or get_new_daemon_name()
-        job = Job(name, sysargs, executor_keys=executor_keys, delete_after_completion=rm)
+        if len(sysargs or []) >= 2 and sysargs[0] == 'start' and sysargs[1].startswith('job'):
+            return (False, "Create a new job with `-d` / `--daemon`."), name
+
+        job_sysargs = (
+            shlex.join([a for a in sysargs if a not in ('-d', '--daemon')])
+            if not line
+            else line
+        )
+        job = Job(name, job_sysargs, executor_keys=executor_keys, delete_after_completion=rm)
+
+        if not job.exists():
+            try:
+                confirm = yes_no(
+                    f"Create new job '{name}'?\n    {job_sysargs}\n",
+                    default='y',
+                    yes=kw.get('yes', False),
+                    noask=kw.get('noask', False),
+                )
+            except Exception:
+                confirm = True
+            if not confirm:
+                return (False, "Nothing changed."), name
+
         return job.start(debug=debug), name
 
     def _run_existing_job(name: str):
@@ -285,6 +312,7 @@ def _start_jobs(
             if new_job
             else _run_existing_job(_name)
         )
+
         if not kw.get('nopretty', False):
             print_tuple(success_tuple)
 
@@ -300,7 +328,10 @@ def _start_jobs(
         + ("Failed to start job" + ("s" if len(_failures) != 1 else '')
             + f" {items_str(_failures)}." if _failures else '')
     )
-    _install_healthcheck_job()
+    if not msg:
+        msg = "Nothing changed."
+    else:
+        _install_healthcheck_job()
     return len(_failures) == 0, msg
 
 
@@ -308,6 +339,7 @@ def _start_gui(
     action: Optional[List[str]] = None,
     mrsm_instance: Optional[str] = None,
     port: Optional[int] = None,
+    webterm_port: Optional[int] = None,
     debug: bool = False,
     **kw
 ) -> SuccessTuple:
@@ -321,7 +353,6 @@ def _start_gui(
 
     from meerschaum.utils.venv import venv_exec
     from meerschaum.utils.packages import attempt_import
-    from meerschaum.utils.warnings import warn
     from meerschaum.utils.debug import dprint
     from meerschaum.utils.networking import find_open_ports
     from meerschaum.connectors.parse import parse_instance_keys
@@ -329,9 +360,10 @@ def _start_gui(
     webview, requests = attempt_import('webview', 'requests')
 
     success, msg = True, "Success"
-    host = '127.0.0.1'
-    if port is None:
-        port = 8765
+    host = mrsm.get_config('webterm', 'host')
+    port = port or webterm_port
+    if not port:
+        port = mrsm.get_config('webterm', 'port')
 
     if not is_webterm_running(host, port, session_id='mrsm'):
         port = next(find_open_ports(port, 9000))
@@ -369,7 +401,7 @@ def _start_gui(
                 break
         except Exception as e:
             if debug:
-                dprint(e)
+                dprint(str(e))
             continue
     if starting_up is False:
         return False, f"The webterm failed to start within {timeout} seconds."
@@ -417,7 +449,6 @@ def _start_webterm(
         - `-i`, '--instance'
             The default instance to use for the Webterm shell.
     """
-    import uuid
     from meerschaum._internal.term import get_webterm_app_and_manager, tornado_ioloop
     from meerschaum._internal.term.tools import (
         is_webterm_running,
@@ -426,15 +457,17 @@ def _start_webterm(
     )
     from meerschaum.utils.networking import find_open_ports
     from meerschaum.utils.warnings import info
+    from meerschaum.config.paths import WEBTERM_INTERNAL_RESOURCES_PATH
 
     if host is None:
-        host = '127.0.0.1'
+        host = mrsm.get_config('api', 'webterm', 'host')
     if port is None:
-        port = 8765
+        port = mrsm.get_config('api', 'webterm', 'port')
     if sysargs is None:
         sysargs = ['start', 'webterm']
     session_id = 'mrsm'
-    tornado_app, term_manager = get_webterm_app_and_manager(instance_keys=mrsm_instance)
+
+    env_path = WEBTERM_INTERNAL_RESOURCES_PATH / (str(port) + '.json')
 
     if is_webterm_running(host, port, session_id=session_id):
         if force:
@@ -445,11 +478,18 @@ def _start_webterm(
                 + "    Include `-f` to start another server on a new port\n"
                 + "    or specify a different port with `-p`."
             )
+
+    tornado_app, term_manager = get_webterm_app_and_manager(
+        instance_keys=mrsm_instance,
+        port=port,
+        env_path=env_path,
+    )
     if not nopretty:
         info(
             f"Starting the webterm at http://{host}:{port}/webterm/{session_id} ..."
             "\n    Press CTRL+C to quit."
         )
+
     tornado_app.listen(port, host)
     loop = tornado_ioloop.IOLoop.instance()
     try:
@@ -462,10 +502,15 @@ def _start_webterm(
         term_manager.shutdown()
         loop.close()
 
-    sessions = get_mrsm_tmux_sessions()
+    sessions = get_mrsm_tmux_sessions(port=port)
     for session in sessions:
         kill_tmux_session(session)
 
+    try:
+        env_path.unlink()
+    except Exception:
+        pass
+
     return True, "Success"
 
 
@@ -483,7 +528,6 @@ def _start_connectors(
     from meerschaum.connectors.parse import parse_instance_keys
     from meerschaum.utils.pool import get_pool
     from meerschaum.utils.warnings import warn
-    from meerschaum.utils.formatting import pprint
     from meerschaum.utils.misc import items_str
     if action is None:
         action = []
@@ -575,7 +619,6 @@ def _start_pipeline(
     """
     import json
     import time
-    import sys
     from meerschaum._internal.entry import entry
     from meerschaum.utils.warnings import info, warn
     from meerschaum.utils.misc import is_int
@@ -630,7 +673,7 @@ def _start_pipeline(
     def do_entry() -> None:
         nonlocal success, msg, proc
         if timeout_seconds is None:
-            success, msg = entry(sub_args_line, _patch_args=patch_args)
+            success, msg = entry(sub_args_line, _patch_args=patch_args, _use_cli_daemon=False)
             return
 
         sub_args_line_escaped = sub_args_line.replace("'", "<QUOTE>")
@@ -640,7 +683,7 @@ def _start_pipeline(
             "from meerschaum._internal.entry import entry\n\n"
             f"sub_args_line = '{sub_args_line_escaped}'.replace(\"<QUOTE>\", \"'\")\n"
             f"patch_args = json.loads('{patch_args_escaped_str}'.replace('<QUOTE>', \"'\"))\n"
-            "success, msg = entry(sub_args_line, _patch_args=patch_args)\n"
+            "success, msg = entry(sub_args_line, _patch_args=patch_args, _use_cli_daemon=False)\n"
             f"print('{fence_begin}' + json.dumps((success, msg)) + '{fence_end}')"
         )
         proc = venv_exec(src, venv=None, as_proc=True)
@@ -676,7 +719,7 @@ def _start_pipeline(
 
     try:
         run_loop()
-    except KeyboardInterrupt:
+    except (KeyboardInterrupt, SystemExit):
         warn("Cancelled pipeline.", stack=False)
         if proc is not None:
             _stop_process(proc)
@@ -688,6 +731,112 @@ def _start_pipeline(
     return success, msg
 
 
+def _start_daemons(
+    timeout_seconds: Union[int, float, None] = None,
+    yes: bool = False,
+    force: bool = False,
+    noask: bool = False,
+    debug: bool = False,
+    **kwargs
+) -> SuccessTuple:
+    """
+    Start the Meerschaum CLI daemon processes.
+    """
+    from meerschaum.utils.warnings import warn, dprint
+    from meerschaum._internal.cli.daemons import (
+        get_cli_daemon,
+        get_cli_lock_path,
+    )
+    from meerschaum._internal.cli.workers import (
+        get_existing_cli_workers,
+        get_existing_cli_worker_indices,
+    )
+    from meerschaum.utils.prompt import yes_no
+    from meerschaum.actions import actions
+
+    workers = get_existing_cli_workers()
+    if not workers:
+        if debug:
+            dprint("No daemons are running, spawning a new process...")
+        workers = [get_cli_daemon()]
+
+    accepted_restart = False
+    any_daemons_are_running = any((worker.job.status == 'running') for worker in workers)
+    lock_paths = [get_cli_lock_path(ix) for ix in get_existing_cli_worker_indices()]
+    any_locks_exist = any(lock_path.exists() for lock_path in lock_paths)
+
+    if any_locks_exist:
+        warn(
+            "Locks are currently held by the CLI daemons.\n"
+            "Run again with `--force` to remove the locks.",
+            stack=False,
+        )
+
+        if not force:
+            return False, "Actions are currently running."
+
+        for lock_path in lock_paths:
+            try:
+                if lock_path.exists():
+                    lock_path.unlink()
+            except Exception as e:
+                warn(f"Failed to release lock:\n{e}")
+
+    if any_daemons_are_running:
+        accepted_restart = force or yes_no(
+            "Restart running CLI daemons?",
+            yes=yes,
+            noask=noask,
+            default='n',
+        )
+
+        if not accepted_restart:
+            return True, "Daemons are already running; nothing to do."
+
+        stop_success, stop_msg = actions['stop'](
+            ['daemons'],
+            timeout_seconds=timeout_seconds,
+            debug=debug,
+        )
+        if not stop_success:
+            return stop_success, stop_msg
+
+    worker = get_cli_daemon()
+
+    if debug:
+        dprint("Cleaning up CLI daemon...")
+    cleanup_success, cleanup_msg = worker.cleanup()
+    if not cleanup_success:
+        return cleanup_success, cleanup_msg
+
+    if debug:
+        dprint("Starting CLI daemon...")
+
+    start_success, start_msg = worker.job.start()
+    if not start_success:
+        return start_success, start_msg
+
+    return True, "Success"
+
+
+def _start_worker(action: Optional[List[str]] = None, **kwargs: Any) -> SuccessTuple:
+    """
+    Start a CLI worker process. This is intended for internal use only.
+    """
+    from meerschaum._internal.cli.workers import ActionWorker
+    from meerschaum.utils.misc import is_int
+
+    if not action:
+        return False, "No worker index is provided."
+
+    if not is_int(action[0]):
+        return False, "Invalid worker index."
+
+    ix = int(action[0])
+    worker = ActionWorker(ix)
+    return worker.run()
+
+
 ### NOTE: This must be the final statement of the module.
 ###       Any subactions added below these lines will not
 ###       be added to the `help` docstring.

meerschaum/actions/stop.py

@@ -7,7 +7,8 @@ Stop running jobs that were started with `-d` or `start job`.
 """
 
 from __future__ import annotations
-from meerschaum.utils.typing import Optional, List, Dict, SuccessTuple, Any
+from meerschaum.utils.typing import Optional, List, SuccessTuple, Any, Union
+
 
 def stop(action: Optional[List[str]] = None, **kw) -> SuccessTuple:
     """
@@ -16,6 +17,7 @@ def stop(action: Optional[List[str]] = None, **kw) -> SuccessTuple:
     from meerschaum.actions import choose_subaction
     options = {
         'jobs': _stop_jobs,
+        'daemons': _stop_daemons,
     }
     return choose_subaction(action, options, **kw)
 
@@ -52,7 +54,7 @@ def _complete_stop(
         return options[sub](action=action, **kw)
 
     from meerschaum._internal.shell import default_action_completer
-    return default_action_completer(action=(['start'] + action), **kw)
+    return default_action_completer(action=(['stop'] + action), **kw)
 
 
 def _stop_jobs(
@@ -111,6 +113,8 @@ def _stop_jobs(
         )
 
     if not jobs_to_stop:
+        if jobs:
+            return True, "The selected jobs are currently stopped."
         return False, "No running, paused or restarting jobs to stop."
 
     if not action:
@@ -160,6 +164,38 @@ def _stop_jobs(
     return success, msg
 
 
+def _stop_daemons(
+    timeout_seconds: Union[int, float, None] = None,
+    debug: bool = False,
+    **kwargs
+) -> SuccessTuple:
+    """
+    Stop the Meerschaum CLI daemon.
+    """
+    import shutil
+    from meerschaum._internal.cli.workers import get_existing_cli_workers
+    from meerschaum.config.paths import CLI_RESOURCES_PATH
+    workers = get_existing_cli_workers()
+
+    for worker in workers:
+        stop_success, stop_msg = worker.job.stop(timeout_seconds=timeout_seconds, debug=debug)
+        if not stop_success:
+            return stop_success, stop_msg
+
+        cleanup_success, cleanup_msg = worker.cleanup(debug=debug)
+        if not cleanup_success:
+            return cleanup_success, cleanup_msg
+
+    try:
+        if CLI_RESOURCES_PATH.exists():
+            shutil.rmtree(CLI_RESOURCES_PATH)
+        CLI_RESOURCES_PATH.mkdir(parents=True, exist_ok=True)
+    except Exception as e:
+        return False, f"Failed to clean up CLI resources directory.\n{e}"
+
+    return True, "Success"
+
+
 ### NOTE: This must be the final statement of the module.
 ###       Any subactions added below these lines will not
 ###       be added to the `help` docstring.

meerschaum/actions/sync.py

@@ -287,8 +287,9 @@ def _sync_pipes(
     from meerschaum.utils.formatting._shell import progress
     from meerschaum.utils.formatting._shell import clear_screen
     from meerschaum.utils.formatting import print_pipes_results
-    from meerschaum.config.static import STATIC_CONFIG
+    from meerschaum._internal.static import STATIC_CONFIG
     from meerschaum.utils.misc import interval_str
+    from meerschaum.utils.daemon import running_in_daemon
 
     noninteractive_val = os.environ.get(STATIC_CONFIG['environment']['noninteractive'], None)
     noninteractive = str(noninteractive_val).lower() in ('1', 'true', 'yes')
@@ -301,7 +302,11 @@ def _sync_pipes(
     cooldown = 2 * (min_seconds + 1)
     success_pipes, failure_pipes = [], []
     while run:
-        _progress = progress() if shell and not noninteractive else None
+        _progress = (
+            progress()
+            if (shell and not noninteractive and not running_in_daemon())
+            else None
+        )
         cm = _progress if _progress is not None else contextlib.nullcontext()
 
         lap_begin = time.perf_counter()

meerschaum/actions/tag.py

@@ -8,12 +8,12 @@ Functions for editing elements belong here.
 
 from __future__ import annotations
 import meerschaum as mrsm
-from meerschaum.utils.typing import List, Any, SuccessTuple, Optional, Dict
+from meerschaum.utils.typing import List, Any, SuccessTuple, Optional
 
 def tag(
-        action: Optional[List[str]] = None,
-        **kwargs: Any
-    ) -> SuccessTuple:
+    action: Optional[List[str]] = None,
+    **kwargs: Any
+) -> SuccessTuple:
     """
     Edit an existing element.
     """
@@ -25,10 +25,10 @@ def tag(
 
 
 def _tag_pipes(
-        action: Optional[List[str]] = None,
-        debug: bool = False,
-        **kwargs: Any
-    ) -> SuccessTuple:
+    action: Optional[List[str]] = None,
+    debug: bool = False,
+    **kwargs: Any
+) -> SuccessTuple:
     """
     Add or remove tags to registered pipes.
     Prefix a tag with a leading underscore to remove it.
@@ -68,6 +68,7 @@ def _tag_pipes(
                 pipe_was_edited = True
 
         if pipe_was_edited:
+            pipe.tags = pipe_tags
             edited_pipes.append(pipe)
 
     if not edited_pipes:

meerschaum/actions/verify.py

@@ -57,8 +57,11 @@ def _verify_packages(
     Verify the versions of packages.
     """
     from meerschaum.utils.packages import (
-        attempt_import, all_packages, is_installed, venv_contains_package,
-        _monkey_patch_get_distribution, manually_import_module,
+        attempt_import,
+        all_packages,
+        is_installed,
+        venv_contains_package,
+        manually_import_module,
     )
 
     venv_packages, base_packages, miss_packages = [], [], []
@@ -78,12 +81,6 @@ def _verify_packages(
         )
         _where_list.append(import_name)
 
-    if 'flask_compress' in venv_packages or 'dash' in venv_packages:
-        flask_compress = attempt_import('flask_compress', lazy=False, debug=debug)
-        _monkey_patch_get_distribution('flask-compress', flask_compress.__version__)
-        if 'flask_compress' in venv_packages:
-            venv_packages.remove('flask_compress')
-
     for import_name in base_packages:
         manually_import_module(import_name, debug=debug, venv=None)
     for import_name in venv_packages:

meerschaum/api/__init__.py

@@ -14,14 +14,13 @@ from fnmatch import fnmatch
 import meerschaum as mrsm
 from meerschaum.utils.typing import Dict, Any, Optional, PipesDict
 from meerschaum.config import get_config
-from meerschaum.config.static import STATIC_CONFIG, SERVER_ID
+from meerschaum._internal.static import STATIC_CONFIG, SERVER_ID
 from meerschaum.utils.packages import attempt_import
 from meerschaum.utils import get_pipes as _get_pipes
 from meerschaum.config._paths import API_UVICORN_CONFIG_PATH, API_UVICORN_RESOURCES_PATH
 from meerschaum.plugins import _api_plugins
 from meerschaum.utils.warnings import warn, dprint
 from meerschaum.utils.threading import RLock
-from meerschaum.utils.misc import is_pipe_registered
 from meerschaum.connectors.parse import parse_instance_keys
 
 from meerschaum import __version__ as version
@@ -67,16 +66,13 @@ from meerschaum.api._exceptions import APIPermissionError
 uvicorn_config_path = API_UVICORN_RESOURCES_PATH / SERVER_ID / 'config.json'
 
 uvicorn_config = None
-sys_config = get_config('system', 'api')
-permissions_config = get_config('system', 'api', 'permissions')
+sys_config = get_config('api')
+permissions_config = get_config('api', 'permissions')
 
 def get_uvicorn_config() -> Dict[str, Any]:
     """Read the Uvicorn configuration JSON and return a dictionary."""
     global uvicorn_config
     import json
-    runtime = os.environ.get(STATIC_CONFIG['environment']['runtime'], None)
-    if runtime == 'api':
-        return get_config('system', 'api', 'uvicorn')
     _uvicorn_config = uvicorn_config
     with _locks['uvicorn_config']:
         if uvicorn_config is None:
@@ -85,6 +81,8 @@ def get_uvicorn_config() -> Dict[str, Any]:
                     uvicorn_config = json.load(f)
                 _uvicorn_config = uvicorn_config
             except Exception:
+                import traceback
+                traceback.print_exc()
                 _uvicorn_config = sys_config.get('uvicorn', None)
 
             if _uvicorn_config is None:
@@ -95,11 +93,17 @@ def get_uvicorn_config() -> Dict[str, Any]:
 
 debug = get_uvicorn_config().get('debug', False)
 no_dash = get_uvicorn_config().get('no_dash', False)
+no_webterm = get_uvicorn_config().get('no_webterm', False)
 no_auth = get_uvicorn_config().get('no_auth', False)
 private = get_uvicorn_config().get('private', False)
 production = get_uvicorn_config().get('production', False)
 _include_dash = (not no_dash)
+_include_webterm = (not no_webterm) and _include_dash
 docs_enabled = not production or sys_config.get('endpoints', {}).get('docs_in_production', True)
+webterm_port = (
+    get_uvicorn_config().get('webterm_port', None)
+    or mrsm.get_config('api', 'webterm', 'port')
+)
 
 default_instance_keys = None
 _instance_connectors = defaultdict(lambda: None)
@@ -127,7 +131,7 @@ def get_api_connector(instance_keys: Optional[str] = None):
     )
     found_match: bool = False
     for allowed_keys_pattern in allowed_instance_keys:
-        if fnmatch(instance_keys, allowed_keys_pattern):
+        if fnmatch(str(instance_keys), allowed_keys_pattern):
             found_match = True
             break
     if not found_match:
@@ -139,7 +143,9 @@ def get_api_connector(instance_keys: Optional[str] = None):
         if _instance_connectors[instance_keys] is None:
             try:
                 is_valid_connector = True
-                _instance_connectors[instance_keys] = parse_instance_keys(instance_keys, debug=debug)
+                instance_connector = parse_instance_keys(instance_keys, debug=debug)
+                instance_connector._cache_connector = get_cache_connector()
+                _instance_connectors[instance_keys] = instance_connector
             except Exception:
                 is_valid_connector = False
 
@@ -166,7 +172,7 @@ def get_cache_connector(connector_keys: Optional[str] = None):
         return None
 
     connector_keys = connector_keys or get_config(
-        'system', 'api', 'cache', 'connector',
+        'api', 'cache', 'connector',
         warn=False,
     )
     if connector_keys is None:
@@ -194,7 +200,11 @@ def pipes(instance_keys: Optional[str] = None, refresh: bool = False) -> PipesDi
     with _locks['pipes-' + instance_keys]:
         pipes = _instance_pipes[instance_keys]
         if pipes is None or refresh:
-            pipes = _get_pipes(mrsm_instance=instance_keys)
+            pipes = _get_pipes(
+                mrsm_instance=instance_keys,
+                cache=True,
+                cache_connector_keys=get_cache_connector(),
+            )
             _instance_pipes[instance_keys] = pipes
     return pipes
 
@@ -210,9 +220,29 @@ def get_pipe(
     if location_key in ('[None]', 'None', 'null'):
         location_key = None
     instance_keys = str(get_api_connector(instance_keys))
-    pipe = mrsm.Pipe(connector_keys, metric_key, location_key, mrsm_instance=instance_keys)
-    if is_pipe_registered(pipe, pipes(instance_keys)):
-        return pipes(instance_keys, refresh=refresh)[connector_keys][metric_key][location_key]
+    if connector_keys == 'mrsm':
+        raise fastapi.HTTPException(
+            status_code=403,
+            detail="Unable to serve any pipes with connector keys `mrsm` over the API.",
+        )
+
+    pipes_dict = pipes(instance_keys)
+    if (
+        not refresh
+        and connector_keys in pipes_dict
+        and metric_key in pipes_dict[connector_keys]
+        and location_key in pipes_dict[connector_keys][metric_key]
+    ):
+        return pipes_dict[connector_keys][metric_key][location_key]
+
+    pipe = mrsm.Pipe(
+        connector_keys,
+        metric_key,
+        location_key,
+        mrsm_instance=instance_keys,
+        cache=True,
+        cache_connector_keys=get_cache_connector(),
+    )
     return pipe
 
 
@@ -291,7 +321,7 @@ def __getattr__(name: str):
     raise AttributeError(f"Could not import '{name}'.")
 
 ### Import everything else within the API.
-from meerschaum.api._oauth2 import manager
+from meerschaum.api._oauth2 import manager, ScopedAuth
 import meerschaum.api.routes as routes
 import meerschaum.api._events
 import meerschaum.api._websockets