meerschaum 2.2.1__py3-none-any.whl → 2.2.2__py3-none-any.whl

meerschaum/_internal/shell/Shell.py

@@ -203,6 +203,29 @@ def _check_complete_keys(line: str) -> Optional[List[str]]:
     return None
 
 
+def get_shell_intro(with_color: bool = True) -> str:
+    """
+    Return the introduction message string.
+    """
+    from meerschaum.utils.formatting import CHARSET, ANSI, colored
+    intro = get_config('shell', CHARSET, 'intro', patch=patch)
+    intro += '\n' + ''.join(
+        [' '
+            for i in range(
+                string_width(intro) - len('v' + version)
+            )
+        ]
+    ) + 'v' + version
+
+    if not with_color or not ANSI:
+        return intro
+
+    return colored(
+        intro,
+        **get_config('shell', 'ansi', 'intro', 'rich')
+    )
+
+
 class Shell(cmd.Cmd):
     def __init__(
             self,
@@ -277,25 +300,20 @@ class Shell(cmd.Cmd):
         except Exception as e:
             pass
 
-
     def load_config(self, instance: Optional[str] = None):
         """
         Set attributes from the shell configuration.
         """
         from meerschaum.utils.misc import remove_ansi
-        from meerschaum.utils.formatting import CHARSET, ANSI, UNICODE, colored
+        from meerschaum.utils.formatting import CHARSET, ANSI, colored
         
         if shell_attrs.get('intro', None) != '':
-            self.intro = get_config('shell', CHARSET, 'intro', patch=patch)
-            self.intro += '\n' + ''.join(
-                [' '
-                    for i in range(
-                        string_width(self.intro) - len('v' + version)
-                    )
-                ]
-            ) + 'v' + version
-        else:
-            self.intro = ""
+            self.intro = (
+                get_shell_intro(with_color=False)
+                if shell_attrs.get('intro', None) != ''
+                else ""
+            )
+
         shell_attrs['intro'] = self.intro
         shell_attrs['_prompt'] = get_config('shell', CHARSET, 'prompt', patch=patch)
         self.prompt = shell_attrs['_prompt']
@@ -822,7 +840,7 @@ def input_with_sigint(_input, session, shell: Optional[Shell] = None):
     """
     Replace built-in `input()` with prompt_toolkit.prompt.
     """
-    from meerschaum.utils.formatting import CHARSET, ANSI, UNICODE, colored
+    from meerschaum.utils.formatting import CHARSET, ANSI, colored
     from meerschaum.connectors import is_connected, connectors
     from meerschaum.utils.misc import remove_ansi
     from meerschaum.config import get_config
@@ -849,11 +867,17 @@ def input_with_sigint(_input, session, shell: Optional[Shell] = None):
                 shell_attrs['instance_keys'], 'on ' + get_config(
                     'shell', 'ansi', 'instance', 'rich', 'style'
                 )
-            ) if ANSI else colored(shell_attrs['instance_keys'], 'on white')
+            )
+            if ANSI
+            else colored(shell_attrs['instance_keys'], 'on white')
         )
         repo_colored = (
-            colored(shell_attrs['repo_keys'], 'on ' + get_config('shell', 'ansi', 'repo', 'rich', 'style'))
-            if ANSI else colored(shell_attrs['repo_keys'], 'on white')
+            colored(
+                shell_attrs['repo_keys'],
+                'on ' + get_config('shell', 'ansi', 'repo', 'rich', 'style')
+            )
+            if ANSI
+            else colored(shell_attrs['repo_keys'], 'on white')
         )
         try:
             typ, label = shell_attrs['instance_keys'].split(':')

meerschaum/_internal/term/__init__.py

@@ -14,9 +14,10 @@ from typing import List, Tuple
 from meerschaum.utils.packages import attempt_import
 from meerschaum._internal.term.TermPageHandler import TermPageHandler
 from meerschaum.config._paths import API_TEMPLATES_PATH, API_STATIC_PATH
+from meerschaum.utils.venv import venv_executable
 
 tornado, tornado_ioloop, terminado = attempt_import(
-    'tornado', 'tornado.ioloop', 'terminado', lazy=False, venv=None,
+    'tornado', 'tornado.ioloop', 'terminado', lazy=False,
 )
 
 def get_webterm_app_and_manager() -> Tuple[
@@ -31,7 +32,7 @@ def get_webterm_app_and_manager() -> Tuple[
     A tuple of the Tornado web application and term manager.
     """
     commands = [
-        sys.executable,
+        venv_executable('mrsm'),
         '-c',
         "import os; _ = os.environ.pop('COLUMNS', None); _ = os.environ.pop('LINES', None); "
         "from meerschaum._internal.entry import get_shell; "

meerschaum/_internal/term/tools.py

@@ -17,7 +17,7 @@ def is_webterm_running(host: str, port: int, protocol: str = 'http') -> int:
     requests = attempt_import('requests')
     url = f'{protocol}://{host}:{port}'
     try:
-        r = requests.get(url)
+        r = requests.get(url, timeout=3)
     except Exception as e:
         return False
     if not r:

meerschaum/actions/api.py

@@ -156,6 +156,7 @@ def _api_start(
     from meerschaum.config.static import STATIC_CONFIG, SERVER_ID
     from meerschaum.connectors.parse import parse_instance_keys
     from meerschaum.utils.pool import get_pool
+    from meerschaum.utils.venv import get_module_venv
     import shutil
     from copy import deepcopy
 
@@ -169,7 +170,10 @@ def _api_start(
     ### `check_update` must be False, because otherwise Uvicorn's hidden imports will break things.
     dotenv = attempt_import('dotenv', lazy=False)
     uvicorn, gunicorn = attempt_import(
-        'uvicorn', 'gunicorn', venv=None, lazy=False, check_update=False,
+        'uvicorn', 'gunicorn',
+        lazy = False,
+        check_update = False,
+        venv = 'mrsm',
     )
 
     uvicorn_config_path = API_UVICORN_RESOURCES_PATH / SERVER_ID / 'config.json'
@@ -305,19 +309,51 @@ def _api_start(
     ### remove custom keys before calling uvicorn
 
     def _run_uvicorn():
-        try:
-            uvicorn.run(
-                **filter_keywords(
-                    uvicorn.run,
-                    **{
-                        k: v
-                        for k, v in uvicorn_config.items()
-                        if k not in custom_keys
-                    }
-                )
-            )
-        except KeyboardInterrupt:
-            pass
+        uvicorn_flags = [
+            '--host', host,
+            '--port', str(port),
+            (
+                '--proxy-headers'
+                if uvicorn_config.get('proxy_headers')
+                else '--no-proxy-headers'
+            ),
+            (
+                '--use-colors'
+                if uvicorn_config.get('use_colors')
+                else '--no-use-colors'
+            ),
+            '--env-file', uvicorn_config['env_file'],
+        ]
+        if uvicorn_reload := uvicorn_config.get('reload'):
+            uvicorn_flags.append('--reload')
+        if (
+            uvicorn_reload
+            and (reload_dirs := uvicorn_config.get('reload_dirs'))
+        ):
+            if not isinstance(reload_dirs, list):
+                reload_dirs = [reload_dirs]
+            for reload_dir in reload_dirs:
+                uvicorn_flags += ['--reload-dir', reload_dir]
+        if (
+            uvicorn_reload
+            and (reload_excludes := uvicorn_config.get('reload_excludes'))
+        ):
+            if not isinstance(reload_excludes, list):
+                reload_excludes = [reload_excludes]
+            for reload_exclude in reload_excludes:
+                uvicorn_flags += ['--reload-exclude', reload_exclude]
+        if (uvicorn_workers := uvicorn_config.get('workers')) is not None:
+            uvicorn_flags += ['--workers', str(uvicorn_workers)]
+
+        uvicorn_args = uvicorn_flags + ['meerschaum.api:app']
+        run_python_package(
+            'uvicorn',
+            uvicorn_args,
+            venv = get_module_venv(uvicorn),
+            as_proc = False,
+            foreground = True,
+            debug = debug,
+        )
 
     def _run_gunicorn():
         gunicorn_args = [
@@ -338,23 +374,21 @@ def _api_start(
             ]
         if debug:
             gunicorn_args += ['--log-level=debug', '--enable-stdio-inheritance', '--reload']
-        try:
-            run_python_package(
-                'gunicorn',
-                gunicorn_args,
-                env = {
-                    k: (
-                        json.dumps(v)
-                        if isinstance(v, (dict, list))
-                        else v
-                    )
-                    for k, v in env_dict.items()
-                },
-                venv = None,
-                debug = debug,
-            )
-        except KeyboardInterrupt:
-            pass
+
+        run_python_package(
+            'gunicorn',
+            gunicorn_args,
+            env = {
+                k: (
+                    json.dumps(v)
+                    if isinstance(v, (dict, list))
+                    else v
+                )
+                for k, v in env_dict.items()
+            },
+            venv = get_module_venv(gunicorn),
+            debug = debug,
+        )
 
 
     _run_uvicorn() if not production else _run_gunicorn()

meerschaum/actions/python.py

@@ -10,37 +10,39 @@ from meerschaum.utils.typing import SuccessTuple, Any, List, Optional
 
 def python(
         action: Optional[List[str]] = None,
+        venv: Optional[str] = 'mrsm',
+        sub_args: Optional[List[str]] = None,
         debug: bool = False,
         **kw: Any
     ) -> SuccessTuple:
     """
-    Launch a Python interpreter with Meerschaum imported. Commands are optional.
-    Note that quotes must be escaped and commands must be separated by semicolons
+    Launch a virtual environment's Python interpreter with Meerschaum imported.
+    You may pass flags to the Python binary by surrounding each flag with `[]`.
     
     Usage:
         `python {commands}`
     
-    Example:
-        `python print(\\'Hello, World!\\'); pipes = mrsm.get_pipes()`
-    
-        ```
-        Hello, World!
-
-        >>> import meerschaum as mrsm
-        >>> print('Hello, World!')
-        >>> pipes = mrsm.get_pipes()
-        ```
+    Examples:
+        mrsm python
+        mrsm python --venv noaa
+        mrsm python [-i] [-c 'print("hi")']
     """
-    import sys, subprocess
+    import sys, subprocess, os
     from meerschaum.utils.debug import dprint
     from meerschaum.utils.warnings import warn, error
-    from meerschaum.utils.process import run_process
+    from meerschaum.utils.venv import venv_executable
+    from meerschaum.utils.misc import generate_password
     from meerschaum.config import __version__ as _version
+    from meerschaum.config.paths import VIRTENV_RESOURCES_PATH, PYTHON_RESOURCES_PATH
+    from meerschaum.utils.packages import run_python_package, attempt_import
 
     if action is None:
         action = []
 
-    joined_actions = ['import meerschaum as mrsm']
+    if venv == 'None':
+        venv = None
+
+    joined_actions = ["import meerschaum as mrsm"]
     line = ""
     for i, a in enumerate(action):
         if a == '':
@@ -51,34 +53,64 @@ def python(
             line = ""
 
     ### ensure meerschaum is imported
-    #  joined_actions = ['import meerschaum as mrsm;'] + joined_actions
     if debug:
-        dprint(joined_actions)
+        dprint(str(joined_actions))
 
-    print_command = 'import sys; print("""'
-    ps1 = ">>> "
+    ### TODO: format the pre-executed code using the pygments lexer.
+    print_command = (
+        'from meerschaum.utils.packages import attempt_import; '
+        + 'ptft = attempt_import("prompt_toolkit.formatted_text", lazy=False); '
+        + 'pts = attempt_import("prompt_toolkit.shortcuts"); '
+        + 'ansi = ptft.ANSI("""'
+    )
+    ps1 = "\\033[1m>>> \\033[0m"
     for i, a in enumerate(joined_actions):
         line = ps1 + f"{a}".replace(';', '\n')
         if '\n' not in line and i != len(joined_actions) - 1:
             line += "\n"
         print_command += line
-    print_command += '""")'
+    print_command += (
+        '"""); '
+        + 'pts.print_formatted_text(ansi); '
+    )
 
-    command = ""
+    command = print_command
     for a in joined_actions:
         command += a
         if not a.endswith(';'):
             command += ';'
         command += ' '
 
-    command += print_command
-
     if debug:
         dprint(f"command:\n{command}")
+
+    init_script_path = PYTHON_RESOURCES_PATH / (generate_password(8) + '.py')
+    with open(init_script_path, 'w', encoding='utf-8') as f:
+        f.write(command)
+
+    env_dict = os.environ.copy()
+    venv_path = (VIRTENV_RESOURCES_PATH / venv) if venv is not None else None
+    if venv_path is not None:
+        env_dict.update({'VIRTUAL_ENV': venv_path.as_posix()})
+
     try:
-        return_code = run_process([sys.executable, '-i', '-c', command], foreground=True)
+        ptpython = attempt_import('ptpython', venv=venv, allow_outside_venv=False)
+        return_code = run_python_package(
+            'ptpython',
+            sub_args or ['--dark-bg', '-i', init_script_path.as_posix()],
+            venv = venv,
+            foreground = True,
+            env = env_dict,
+        )
     except KeyboardInterrupt:
         return_code = 1
+
+    try:
+        if init_script_path.exists():
+            init_script_path.unlink()
+    except Exception as e:
+        warn(f"Failed to clean up tempory file '{init_script_path}'.")
+
     return return_code == 0, (
         "Success" if return_code == 0
         else f"Python interpreter returned {return_code}."

meerschaum/actions/start.py

@@ -333,8 +333,7 @@ def _start_gui(
     from meerschaum.connectors.parse import parse_instance_keys
     from meerschaum._internal.term.tools import is_webterm_running
     import platform
-    webview = attempt_import('webview')
-    requests = attempt_import('requests')
+    webview, requests = attempt_import('webview', 'requests')
     import json
     import time
 
@@ -365,7 +364,7 @@ def _start_gui(
     base_url = 'http://' + api_kw['host'] + ':' + str(api_kw['port'])
 
     process = venv_exec(
-        start_tornado_code, as_proc=True, venv=None, debug=debug, capture_output=(not debug)
+        start_tornado_code, as_proc=True, debug=debug, capture_output=(not debug)
     )
     timeout = 10
     start = time.perf_counter()
@@ -446,7 +445,6 @@ def _start_webterm(
                 + "    Include `-f` to start another server on a new port\n"
                 + "    or specify a different port with `-p`."
             )
-
     if not nopretty:
         info(f"Starting the webterm at http://{host}:{port} ...\n    Press CTRL+C to quit.")
     tornado_app.listen(port, host)

meerschaum/actions/uninstall.py

@@ -7,7 +7,7 @@ Uninstall plugins and pip packages.
 """
 
 from __future__ import annotations
-from meerschaum.utils.typing import List, Any, SuccessTuple, Optional
+from meerschaum.utils.typing import List, Any, SuccessTuple, Optional, Union
 
 def uninstall(
         action: Optional[List[str]] = None,
@@ -145,13 +145,10 @@ def _complete_uninstall_plugins(action: Optional[List[str]] = None, **kw) -> Lis
             possibilities.append(name)
     return possibilities
 
-class NoVenv:
-    pass
-
 def _uninstall_packages(
         action: Optional[List[str]] = None,
         sub_args: Optional[List[str]] = None,
-        venv: Union[str, None, NoVenv] = NoVenv,
+        venv: Optional[str] = 'mrsm',
         yes: bool = False,
         force: bool = False,
         noask: bool = False,
@@ -169,9 +166,7 @@ def _uninstall_packages(
 
     from meerschaum.utils.warnings import info
     from meerschaum.utils.packages import pip_uninstall
-
-    if venv is NoVenv:
-        venv = 'mrsm'
+    from meerschaum.utils.misc import items_str
 
     if not (yes or force) and noask:
         return False, "Skipping uninstallation. Add `-y` or `-f` to agree to the uninstall prompt."
@@ -183,7 +178,8 @@ def _uninstall_packages(
         debug = debug,
     ):
         return True, (
-            f"Successfully removed packages from virtual environment 'mrsm':\n" + ', '.join(action)
+            f"Successfully removed packages from virtual environment '{venv}':\n"
+            + items_str(action)
         )
 
     return False, f"Failed to uninstall packages:\n" + ', '.join(action)

meerschaum/actions/upgrade.py

@@ -130,7 +130,7 @@ def _upgrade_packages(
         upgrade packages docs
         ```
     """
-    from meerschaum.utils.packages import packages, pip_install
+    from meerschaum.utils.packages import packages, pip_install, get_prerelease_dependencies
     from meerschaum.utils.warnings import info, warn
     from meerschaum.utils.prompt import yes_no
     from meerschaum.utils.formatting import make_header, pprint
@@ -140,7 +140,7 @@ def _upgrade_packages(
     if venv is NoVenv:
         venv = 'mrsm'
     if len(action) == 0:
-        group = 'full'
+        group = 'api'
     else:
         group = action[0]
 
@@ -148,7 +148,7 @@ def _upgrade_packages(
         invalid_msg = f"Invalid dependency group '{group}'."
         avail_msg = make_header("Available groups:")
         for k in packages:
-            avail_msg += "\n  - {k}"
+            avail_msg += f"\n  - {k}"
 
         warn(invalid_msg + "\n\n" + avail_msg, stack=False)
         return False, invalid_msg
@@ -160,10 +160,18 @@ def _upgrade_packages(
         f"(dependency group '{group}')?"
     )
     to_install = [install_name for import_name, install_name in packages[group].items()]
+    prereleases_to_install = get_prerelease_dependencies(to_install)
+    to_install = [
+        install_name
+        for install_name in to_install
+        if install_name not in prereleases_to_install
+    ]
 
     success, msg = False, f"Nothing installed."
     if force or yes_no(question, noask=noask, yes=yes):
         success = pip_install(*to_install, debug=debug)
+        if success and prereleases_to_install:
+            success = pip_install(*prereleases_to_install, debug=debug)
         msg = (
             f"Successfully installed {len(packages[group])} packages." if success
             else f"Failed to install packages in dependency group '{group}'."

meerschaum/api/__init__.py

@@ -31,6 +31,7 @@ CHECK_UPDATE = os.environ.get(STATIC_CONFIG['environment']['runtime'], None) !=
 
 endpoints = STATIC_CONFIG['api']['endpoints']
 
+uv = attempt_import('uv', lazy=False, check_update=CHECK_UPDATE)
 (
     fastapi,
     aiofiles,

meerschaum/api/dash/callbacks/__init__.py

@@ -11,3 +11,7 @@ import meerschaum.api.dash.callbacks.login
 import meerschaum.api.dash.callbacks.plugins
 import meerschaum.api.dash.callbacks.jobs
 import meerschaum.api.dash.callbacks.register
+from meerschaum.api.dash.callbacks.custom import init_dash_plugins, add_plugin_pages
+
+init_dash_plugins()
+add_plugin_pages()

meerschaum/api/dash/callbacks/custom.py

@@ -0,0 +1,39 @@
+#! /usr/bin/env python3
+# -*- coding: utf-8 -*-
+# vim:fenc=utf-8
+
+"""
+Import custom callbacks created by plugins.
+"""
+
+import traceback
+from meerschaum.api.dash import dash_app
+from meerschaum.plugins import _dash_plugins, _plugin_endpoints_to_pages
+from meerschaum.utils.warnings import warn
+from meerschaum.api.dash.callbacks.dashboard import _paths, _required_login
+
+
+def init_dash_plugins():
+    """
+    Fire the initial callbacks for Dash plugins.
+    """
+    for _module_name, _functions in _dash_plugins.items():
+        for _function in _functions:
+            try:
+                _function(dash_app)
+            except Exception as e:
+                warn(
+                    f"Failed to load function '{_function.__name__}' "
+                    + f"from plugin '{_module_name}':\n"
+                    + traceback.format_exc()
+                )
+
+
+def add_plugin_pages():
+    """
+    Allow users to add pages via the `@web_page` decorator.
+    """
+    for _endpoint, _page_dict in _plugin_endpoints_to_pages.items():
+        _paths[_endpoint] = _page_dict['function']()
+        if _page_dict['login_required']:
+            _required_login.add(_endpoint)

meerschaum/api/dash/callbacks/dashboard.py

@@ -94,6 +94,8 @@ omit_actions = {
     'repo',
     'instance',
 }
+
+### Map endpoints to page layouts.
 _paths = {
     'login'   : pages.login.layout,
     ''        : pages.dashboard.layout,
@@ -101,7 +103,8 @@ _paths = {
     'register': pages.register.layout,
 }
 _required_login = {''}
- 
+
+
 @dash_app.callback(
     Output('page-layout-div', 'children'),
     Output('session-store', 'data'),
@@ -147,16 +150,31 @@ def update_page_layout_div(
     else:
         session_store_to_return = dash.no_update
 
-    _path = (
+    base_path = (
         pathname.rstrip('/') + '/'
     ).replace(
         (dash_endpoint + '/'),
         ''
     ).rstrip('/').split('/')[0]
+
+    complete_path = (
+        pathname.rstrip('/') + '/'
+    ).replace(
+        dash_endpoint + '/',
+        ''
+    ).rstrip('/')
+
+    if complete_path in _paths:
+        path_str = complete_path
+    elif base_path in _paths:
+        path_str = base_path
+    else:
+        path_str = ''
+
     path = (
-        _path
-        if no_auth or _path not in _required_login else (
-            _path
+        path_str
+        if no_auth or path_str not in _required_login else (
+            path_str
             if session_id in active_sessions
             else 'login'
         )
@@ -868,10 +886,25 @@ dash_app.clientside_callback(
             location = "None";
         }
 
+        var subaction = "pipes";
+        if (action == "python"){
+            subaction = (
+                '"' + "pipe = mrsm.Pipe('"
+                + pipe_meta.connector
+                + "', '"
+                + pipe_meta.metric
+                + "'"
+            );
+            if (location != "None"){
+                subaction += ", '" + location + "'";
+            }
+            subaction += ", instance='" + pipe_meta.instance + "')" + '"';
+        }
+
         iframe.contentWindow.postMessage(
             {
                 action: action,
-                subaction: "pipes",
+                subaction: subaction,
                 connector_keys: [pipe_meta.connector],
                 metric_keys: [pipe_meta.metric],
                 location_keys: [location],

meerschaum/api/dash/callbacks/login.py

@@ -45,6 +45,7 @@ def show_registration_disabled_collapse(n_clicks, is_open):
     State('username-input', 'value'),
     State('password-input', 'value'),
     State('location', 'href'),
+    State('location', 'pathname'),
 )
 def login_button_click(
         username_submit,
@@ -53,6 +54,7 @@ def login_button_click(
         username,
         password,
         location_href,
+        location_pathname,
     ):
     """
     When the user submits the login form, check the login.
@@ -69,4 +71,4 @@ def login_button_click(
     except HTTPException:
         form_class += ' is-invalid'
         session_data = None
-    return session_data, form_class, (dash.no_update if not session_data else endpoints['dash'])
+    return session_data, form_class, dash.no_update