meerschaum 2.4.0.dev1__py3-none-any.whl → 2.4.0rc2__py3-none-any.whl

meerschaum/api/dash/components.py

@@ -7,27 +7,24 @@ Custom components are defined here.
 """
 
 from __future__ import annotations
-from meerschaum.utils.venv import Venv
+
 from meerschaum.utils.packages import attempt_import, import_dcc, import_html
 from meerschaum.utils.typing import SuccessTuple, List
 from meerschaum.config.static import STATIC_CONFIG
 from meerschaum.utils.misc import remove_ansi
 from meerschaum._internal.shell.Shell import get_shell_intro
 from meerschaum.api import endpoints, CHECK_UPDATE
-from meerschaum.connectors import instance_types
+from meerschaum.connectors import instance_types, _load_builtin_custom_connectors
 from meerschaum.utils.misc import get_connector_labels
 from meerschaum.config import __doc__ as doc
 dbc = attempt_import('dash_bootstrap_components', lazy=False, check_update=CHECK_UPDATE)
 html, dcc = import_html(check_update=CHECK_UPDATE), import_dcc(check_update=CHECK_UPDATE)
 dex = attempt_import('dash_extensions', lazy=False, check_update=CHECK_UPDATE)
 dash_ace = attempt_import('dash_ace', lazy=False, check_update=CHECK_UPDATE)
-
-component_ids = {
-
-}
+_load_builtin_custom_connectors()
 
 go_button = dbc.Button('Execute', id='go-button', color='primary', style={'width': '100%'})
-test_button = dbc.Button('Test', id='test-button', color='danger', style={'display' : 'none'})
+test_button = dbc.Button('Test', id='test-button', color='danger', style={'display': 'none'})
 get_items_menu = dbc.DropdownMenu(
     label='More', id='get-items-menu', children=[
         dbc.DropdownMenuItem("Plugins", id='get-plugins-button'),
@@ -42,21 +39,21 @@ get_items_menu = dbc.DropdownMenu(
 )
 show_pipes_button = dbc.Button(
     'Pipes',
-    id = 'get-pipes-button',
-    color = 'info',
-    style = {'width': '100%'},
+    id='get-pipes-button',
+    color='info',
+    style={'width': '100%'},
 )
 show_jobs_button = dbc.Button(
     'Jobs',
-    id = 'get-jobs-button',
-    color = 'success',
-    style = {'width': '100%'},
+    id='get-jobs-button',
+    color='success',
+    style={'width': '100%'},
 )
 cancel_button = dbc.Button(
     'Terminal',
-    id = 'cancel-button',
-    color = 'dark',
-    style = {'width': '100%', 'background-color': 'black', 'display': 'none'},
+    id='cancel-button',
+    color='dark',
+    style={'width': '100%', 'background-color': 'black', 'display': 'none'},
 )
 bottom_buttons_content = dbc.Card(
     dbc.CardBody(
@@ -70,28 +67,26 @@ bottom_buttons_content = dbc.Card(
     )
 )
 console_div = html.Div(
-    id = 'console-div',
-    children = [html.Pre(get_shell_intro(), id='console-pre')],
+    id='console-div',
+    children=[html.Pre(get_shell_intro(), id='console-pre')],
 )
 
-location = dcc.Location(id='location', refresh=False)
+location = dcc.Location(id='mrsm-location', refresh=False)
 
 search_parameters_editor = dash_ace.DashAceEditor(
-    id = 'search-parameters-editor',
-    theme = 'monokai',
-    mode = 'json',
-    tabSize = 2,
-    placeholder = (
+    id='search-parameters-editor',
+    theme='monokai',
+    mode='json',
+    tabSize=2,
+    placeholder=(
         'Additional search parameters. ' +
         'Simple dictionary format or JSON accepted.'
     ),
-    style = {'height' : 100},
+    style={'height': 100},
 )
 
 sidebar = dbc.Offcanvas(
-    children=[
-    
-    ],
+    children=[],
     title='Pages',
 )
 
@@ -99,13 +94,13 @@ download_dataframe = dcc.Download(id='download-dataframe-csv')
 download_logs = dcc.Download(id='download-logs')
 
 instance_select = dbc.Select(
-    id = 'instance-select',
-    size = 'sm',
-    options = [
+    id='instance-select',
+    size='sm',
+    options=[
         {'label': i, 'value': i}
         for i in get_connector_labels(*instance_types)
     ],
-    class_name = 'dbc_dark custom-select custom-select-sm',
+    class_name='dbc_dark custom-select custom-select-sm',
 )
 
 
@@ -116,17 +111,17 @@ navbar = dbc.Navbar(
                 dbc.Row(
                     [
                         dbc.Col(
-                                html.Img(
-                                    src = endpoints['dash'] + "/assets/logo_48x48.png",
-                                    title = doc,
-                                ),
+                            html.Img(
+                                src=endpoints['dash'] + "/assets/logo_48x48.png",
+                                title=doc,
+                            ),
                         ),
                     ],
-                    align = 'center',
-                    className = 'g-0 navbar-logo-row',
+                    align='center',
+                    className='g-0 navbar-logo-row',
                 ),
-                href = '/docs',
-                style = {"textDecoration": "none"},
+                href='/docs',
+                style={"textDecoration": "none"},
             ),
             dbc.NavbarToggler(id="navbar-toggler", n_clicks=0),
             dbc.Collapse(
@@ -136,30 +131,30 @@ navbar = dbc.Navbar(
                         dbc.Col(
                             dbc.Button(
                                 "Sign out",
-                                color = 'link',
-                                style = {'margin-left': '30px'},
-                                id = 'sign-out-button',
+                                color='link',
+                                style={'margin-left': '30px'},
+                                id='sign-out-button',
                             ),
                         ),
                     ],
-                    className = "g-0 ms-auto flex-nowrap mt-3 mt-md-0",
+                    className="g-0 ms-auto flex-nowrap mt-3 mt-md-0",
                 ),
-                id = 'navbar-collapse',
-                is_open = False,
-                navbar = True,
+                id='navbar-collapse',
+                is_open=False,
+                navbar=True,
             ),
         ],
-        style = {'max-width': '96%'},
+        style={'max-width': '96%'},
     ),
-    color = 'dark', dark=True,
-    style = {'width': '100% !important'},
+    color='dark', dark=True,
+    style={'width': '100% !important'},
 )
 
 refresh_jobs_interval = dcc.Interval(
-    id = 'refresh-jobs-interval',
-    interval = 1 * 1000,
-    n_intervals = 0,
-    disabled = False,
+    id='refresh-jobs-interval',
+    interval=(1 * 1000),
+    n_intervals=0,
+    disabled=False,
 )
 
 def alert_from_success_tuple(success: SuccessTuple) -> dbc.Alert:
@@ -169,11 +164,11 @@ def alert_from_success_tuple(success: SuccessTuple) -> dbc.Alert:
     return dbc.Alert('', is_open=False) if not isinstance(success, tuple) else (
         dbc.Alert(
             remove_ansi(success[1]),
-            id = 'success-alert',
-            dismissable = True,
-            fade = True,
-            is_open = not (success[1] in STATIC_CONFIG['system']['success']['ignore']),
-            color = 'success' if success[0] else 'danger',
+            id='success-alert',
+            dismissable=True,
+            fade=True,
+            is_open=(success[1] not in STATIC_CONFIG['system']['success']['ignore']),
+            color='success' if success[0] else 'danger',
         )
     )
 

meerschaum/api/dash/jobs.py

@@ -7,24 +7,20 @@ Functions for interacting with jobs via the web interface.
 """
 
 from __future__ import annotations
-from dash.dependencies import Input, Output, State
-from meerschaum.utils.typing import List, Optional, Dict, Any, Tuple, Union, WebState
+from meerschaum.utils.typing import List, Dict, WebState
 from meerschaum.utils.packages import attempt_import, import_html, import_dcc
-from meerschaum.api.dash.components import alert_from_success_tuple, build_cards_grid
-from meerschaum.api.dash.users import is_session_authenticated
+from meerschaum.api.dash.sessions import is_session_authenticated
 from meerschaum.api import CHECK_UPDATE
-dbc = attempt_import('dash_bootstrap_components', lazy=False, check_update=CHECK_UPDATE)
-html, dcc = import_html(), import_dcc()
 from meerschaum.jobs import (
     get_jobs,
-    get_running_jobs,
-    get_paused_jobs,
-    get_stopped_jobs,
     get_executor_keys_from_context,
     Job,
 )
 from meerschaum.config import get_config
 from meerschaum.utils.misc import sorted_dict
+from dash.dependencies import Input, Output, State
+dbc = attempt_import('dash_bootstrap_components', lazy=False, check_update=CHECK_UPDATE)
+html, dcc = import_html(), import_dcc()
 
 STATUS_EMOJI: Dict[str, str] = {
     'running': get_config('formatting', 'emoji', 'running'),

meerschaum/api/dash/pages/pipes.py

@@ -5,12 +5,15 @@
 Display pipes via a shareable URL.
 """
 
-from meerschaum.api import get_api_connector, CHECK_UPDATE
+from meerschaum.api import CHECK_UPDATE
 from meerschaum.utils.packages import import_html, import_dcc
+from meerschaum.api.dash.components import download_dataframe
+
 html, dcc = import_html(check_update=CHECK_UPDATE), import_dcc(check_update=CHECK_UPDATE)
 import dash_bootstrap_components as dbc
 
 layout = dbc.Container([
     dcc.Location('pipes-location'),
+    download_dataframe,
     html.Div(id='pipe-output-div'),
 ])

meerschaum/api/dash/pipes.py

@@ -12,18 +12,16 @@ import shlex
 from textwrap import dedent
 from dash.dependencies import Input, Output, State
 from meerschaum.utils.typing import List, Optional, Dict, Any, Tuple, Union
-from meerschaum.utils.misc import string_to_dict, json_serialize_datetime
+from meerschaum.utils.misc import string_to_dict
 from meerschaum.utils.packages import attempt_import, import_dcc, import_html, import_pandas
 from meerschaum.utils.sql import get_pd_type
 from meerschaum.utils.yaml import yaml
 from meerschaum.connectors.sql._fetch import get_pipe_query
-from meerschaum.api import endpoints, CHECK_UPDATE
-from meerschaum.api.dash import (
-    dash_app, debug, _get_pipes
-)
+from meerschaum.api import CHECK_UPDATE
+from meerschaum.api.dash import debug, _get_pipes
 from meerschaum.api.dash.connectors import get_web_connector
 from meerschaum.api.dash.components import alert_from_success_tuple, build_cards_grid
-from meerschaum.api.dash.users import is_session_authenticated
+from meerschaum.api.dash.sessions import is_session_authenticated
 from meerschaum.config import get_config
 import meerschaum as mrsm
 dbc = attempt_import('dash_bootstrap_components', lazy=False, check_update=CHECK_UPDATE)
@@ -235,12 +233,10 @@ def build_pipe_card(
             ),
             dbc.Col(
                 dbc.Button(
-                    [
-                        html.I(
-                            className="bi bi-share",
-                        ),
-                    ],
-                    #  href=pipe_url,
+                    "🔗",
+                    href=pipe_url,
+                    external_link=True,
+                    target="_blank",
                     style={'float': 'right'},
                     outline=True,
                     color='link',
@@ -553,11 +549,11 @@ def accordion_items_from_pipe(
         backtrack_df = pipe.get_backtrack_data(debug=debug, limit=1)
         try:
             json_text = backtrack_df.fillna(pd.NA).to_json(
-                orient = 'records',
-                date_format = 'iso',
-                force_ascii = False,
-                indent = 4,
-                date_unit = 'ns',
+                orient='records',
+                date_format='iso',
+                force_ascii=False,
+                indent=4,
+                date_unit='ns',
             ) if backtrack_df is not None else '[]'
         except Exception as e:
             warn(e)

meerschaum/api/dash/plugins.py

@@ -13,7 +13,8 @@ from meerschaum.api import get_api_connector, endpoints, CHECK_UPDATE
 html, dcc = import_html(check_update=CHECK_UPDATE), import_dcc(check_update=CHECK_UPDATE)
 import dash_bootstrap_components as dbc
 from meerschaum.core import Plugin
-from meerschaum.api.dash import dash_app, debug, active_sessions
+from meerschaum.api.dash import dash_app, debug
+from meerschaum.api.dash.sessions import get_username_from_session
 
 
 def get_plugins_cards(
@@ -88,9 +89,10 @@ def is_plugin_owner(plugin_name: str, session_data: Dict['str', Any]) -> bool:
     Check whether the currently logged in user is the owner of a plugin.
     """
     plugin = Plugin(plugin_name)
-    _username = active_sessions.get(
-        session_data.get('session-id', None), {}
-    ).get('username', None)
+    session_id = session_data.get('session-id', None)
+    if session_id is None:
+        return False
+    _username = get_username_from_session(session_id)
     _plugin_username = get_api_connector().get_plugin_username(plugin, debug=debug)
     return (
         _username is not None

meerschaum/api/dash/sessions.py

@@ -0,0 +1,176 @@
+#! /usr/bin/env python3
+# vim:fenc=utf-8
+
+"""
+Define caching functions for session management.
+"""
+
+import json
+
+from meerschaum.utils.typing import Union, Dict, Any, Optional
+from meerschaum.api import debug, no_auth
+from meerschaum.api import get_cache_connector, get_api_connector
+from meerschaum.core import User
+from meerschaum.config import get_config
+from meerschaum.utils.warnings import dprint
+
+SESSION_KEY_TEMPLATE: str = 'mrsm_session_id:{session_id}'
+EXPIRES_SECONDS: int = get_config('system', 'api', 'cache', 'session_expires_minutes') * 60
+_active_sessions: Dict[str, Dict[str, Any]] = {}
+
+
+def get_session_key(session_id: str) -> str:
+    """
+    Return the session key for the cache connector.
+    """
+    return SESSION_KEY_TEMPLATE.format(session_id=session_id)
+
+
+def set_session(session_id: str, session_data: Dict[str, Any]):
+    """
+    Set a `session_id` to a dictionary.
+    """
+    conn = get_cache_connector()
+    if conn is None:
+        _active_sessions[session_id] = session_data
+        if debug:
+            dprint(f"Setting in-memory data for {session_id}:\n{session_data}")
+        return
+
+    session_key = get_session_key(session_id)
+    session_data_str = json.dumps(session_data, separators=(',', ':'))
+    if debug:
+        dprint(f"Setting production data for {session_id=}:\n{session_data_str}")
+
+    conn.set(session_key, session_data_str, ex=EXPIRES_SECONDS)
+
+
+def update_session(session_id: Optional[str], session_data: Dict[str, Any]):
+    """
+    Update the session's data dictionary.
+    """
+    existing_session_data = get_session_data(session_id) or {}
+    existing_session_data.update(session_data)
+    set_session(str(session_id), existing_session_data)
+
+
+def get_session_data(session_id: Optional[str]) -> Union[Dict[str, Any], None]:
+    """
+    Return the session data dictionary.
+    """
+    if debug:
+        dprint(f"Getting session data for {session_id=}")
+    conn = get_cache_connector()
+    if conn is None:
+        return _active_sessions.get(str(session_id), None)
+
+    session_key = get_session_key(str(session_id))
+    session_data_str = conn.get(session_key)
+    if not session_data_str:
+        return None
+
+    return json.loads(session_data_str)
+
+
+def get_username_from_session(session_id: Optional[str]) -> Union[str, None]:
+    """
+    If a `session_id` has been set, return the username.
+    Otherwise return `None`.
+    """
+    if debug:
+        dprint(f"Getting username for {session_id=}")
+    session_data = get_session_data(session_id)
+    if session_data is None:
+        return None
+
+    return session_data.get('username', None)
+
+
+def is_session_active(session_id: Union[str, None]) -> bool:
+    """
+    Return whether a given `session_id` has been set.
+    """
+    return get_username_from_session(str(session_id)) is not None
+
+
+def delete_session(session_id: str):
+    """
+    Delete a session if it's been set.
+    """
+    if debug:
+        dprint(f"Deleting {session_id=}")
+    conn = get_cache_connector()
+    if conn is None:
+        _ = _active_sessions.pop(session_id, None)
+        return
+
+    session_key = get_session_key(session_id)
+    conn.client.delete(session_key)
+
+
+def is_session_authenticated(session_id: Optional[str]) -> bool:
+    """
+    Check is a session ID is active.
+    If running in secure mode, check whether a session ID corresponds to an admin.
+
+    Parameters
+    ----------
+    session_id: str
+        The session UUID.
+
+    Returns
+    -------
+    A bool whether the session is authenticated to perform actions.
+    """
+    if debug:
+        dprint(f"Checking authentication for {session_id=}")
+
+    if no_auth:
+        return True
+
+    if session_id is None:
+        return False
+
+    session_data = get_session_data(session_id)
+    if session_data is None:
+        return False
+    username = session_data.get('username', None)
+    if username is None:
+        return False
+
+    cached_auth = session_data.get('authenticated', None)
+    if cached_auth is not None:
+        return cached_auth
+
+    permissions = get_config('system', 'api', 'permissions')
+    allow_non_admin = permissions.get('actions', {}).get('non_admin', False)
+
+    is_auth = True if allow_non_admin else session_is_admin(session_id)
+    update_session(session_id, {'authenticated': is_auth})
+    return is_auth
+
+
+def session_is_admin(session_id: str) -> bool:
+    """
+    Check whether a session ID corresponds to an admin user.
+    """
+    if debug:
+        dprint(f"Check admin for {session_id=}")
+    session_data = get_session_data(session_id)
+    if session_data is None:
+        return False
+
+    username = session_data.get('username', None)
+    if username is None:
+        return False
+
+    cached_admin = session_data.get('admin', None)
+    if cached_admin is not None:
+        return cached_admin
+
+    conn = get_api_connector()
+    user = User(username, instance=conn)
+    user_type = conn.get_user_type(user, debug=debug)
+    is_admin = user_type == 'admin'
+    update_session(session_id, {'admin': is_admin})
+    return is_admin

meerschaum/api/dash/users.py

@@ -8,13 +8,10 @@ Functions for interacting with users.
 
 from __future__ import annotations
 
-from meerschaum.api import debug, CHECK_UPDATE, get_api_connector, no_auth
-from meerschaum.api.dash import active_sessions, authenticated_sessions, unauthenticated_sessions
+from meerschaum.api import debug, CHECK_UPDATE
 from meerschaum.api.dash.connectors import get_web_connector
-from meerschaum.utils.typing import WebState, SuccessTuple, List, Tuple, Optional
+from meerschaum.utils.typing import WebState, SuccessTuple, List, Tuple
 from meerschaum.utils.packages import attempt_import, import_html, import_dcc
-from meerschaum.config import get_config
-from meerschaum.core import User
 dcc, html = import_dcc(check_update=CHECK_UPDATE), import_html(check_update=CHECK_UPDATE)
 dbc = attempt_import('dash_bootstrap_components', lazy=False, check_update=CHECK_UPDATE)
 
@@ -35,51 +32,3 @@ def get_users_cards(state: WebState) -> Tuple[List[dbc.Card], List[SuccessTuple]
         )
 
     return cards, alerts
-
-
-def is_session_authenticated(session_id: str) -> bool:
-    """
-    Check is a session ID is active.
-    If running in secure mode, check whether a session ID corresponds to an admin.
-
-    Parameters
-    ----------
-    session_id: str
-        The session UUID.
-
-    Returns
-    -------
-    A bool whether the session is authenticated to perform actions.
-    """
-    if no_auth:
-        return True
-    if session_id not in active_sessions:
-        return False
-    if session_id in unauthenticated_sessions:
-        return False
-    if session_id in authenticated_sessions:
-        return True
-
-    permissions = get_config('system', 'api', 'permissions')
-    allow_non_admin = permissions.get('actions', {}).get('non_admin', False)
-
-    is_auth = True if allow_non_admin else session_is_admin(session_id)
-    username = active_sessions.get(session_id, {}).get('username', None)
-
-    if is_auth:
-        authenticated_sessions[session_id] = username
-    else:
-        unauthenticated_sessions[session_id] = username
-
-    return is_auth
-
-
-def session_is_admin(session_id: str) -> bool:
-    """
-    Check whether a session ID corresponds to an admin user.
-    """
-    conn = get_api_connector()
-    username = active_sessions.get(session_id, {}).get('username', None)
-    user = User(username, instance=conn)
-    user_type = conn.get_user_type(user, debug=debug)
-    return user_type == 'admin'

meerschaum/api/dash/webterm.py

@@ -7,16 +7,12 @@ Functions for interacting with the Webterm via the dashboard.
 """
 
 import time
-from urllib.parse import urlparse
-from meerschaum.config import get_config
-from meerschaum.api import debug, CHECK_UPDATE, get_api_connector, no_auth
-from meerschaum.api.dash import active_sessions
-from meerschaum.api.dash.users import is_session_authenticated
+from meerschaum.api import CHECK_UPDATE, get_api_connector
+from meerschaum.api.dash.sessions import is_session_authenticated, get_username_from_session
 from meerschaum.api.dash.components import alert_from_success_tuple, console_div
-from meerschaum.utils.typing import WebState, SuccessTuple, List, Tuple, Optional, Any
-from meerschaum.utils.packages import attempt_import, import_html, import_dcc, run_python_package
+from meerschaum.utils.typing import WebState, Tuple, Any
+from meerschaum.utils.packages import attempt_import, import_html, import_dcc
 from meerschaum._internal.term.tools import is_webterm_running
-from meerschaum.config.static import STATIC_CONFIG
 from meerschaum.utils.threading import Thread, RLock
 dcc, html = import_dcc(check_update=CHECK_UPDATE), import_html(check_update=CHECK_UPDATE)
 dbc = attempt_import('dash_bootstrap_components', lazy=False, check_update=CHECK_UPDATE)
@@ -30,7 +26,7 @@ def get_webterm(state: WebState) -> Tuple[Any, Any]:
     Start the webterm and return its iframe.
     """
     session_id = state['session-store.data'].get('session-id', None)
-    username = active_sessions.get(session_id, {}).get('username', None)
+    username = get_username_from_session(session_id)
     if not is_session_authenticated(session_id):
         msg = f"User '{username}' is not authorized to access the webterm."
         return (
@@ -57,24 +53,23 @@ def get_webterm(state: WebState) -> Tuple[Any, Any]:
     return console_div, [alert_from_success_tuple((False, "Could not start the webterm server."))]
 
 
-
 webterm_procs = {}
 def start_webterm() -> None:
     """
     Start the webterm thread.
     """
-    from meerschaum._internal.entry import entry
     from meerschaum.utils.packages import run_python_package
 
     def run():
+        conn = get_api_connector()
         _ = run_python_package(
             'meerschaum',
-            ['start', 'webterm'],
-            capture_output = True,
-            as_proc = True,
-            store_proc_dict = webterm_procs,
-            store_proc_key = 'process',
-            venv = None,
+            ['start', 'webterm', '-i', str(conn)],
+            capture_output=True,
+            as_proc=True,
+            store_proc_dict=webterm_procs,
+            store_proc_key='process',
+            venv=None,
         )
 
     with _locks['webterm_thread']:

meerschaum/api/resources/static/js/terminado.js

@@ -15,7 +15,7 @@ function make_terminal(element, size, ws_url) {
   term.attachCustomKeyEventHandler(copyPasteKeyEventHandler);
   term.open(element);
 
-  sessionStore = sessionStorage.getItem("session-store");
+  sessionStore = localStorage.getItem("session-store");
   ws.onopen = function (event) {
     ws.send(sessionStore);
     ws.send(