meerschaum 2.3.5.dev0__py3-none-any.whl → 2.4.0__py3-none-any.whl

meerschaum/api/__init__.py

@@ -6,7 +6,7 @@
 Meerschaum API backend. Start an API instance with `start api`.
 """
 from __future__ import annotations
-import pathlib, os
+import os
 from meerschaum.utils.typing import Dict, Any, Optional
 
 from meerschaum import __version__ as version
@@ -21,7 +21,7 @@ from meerschaum.utils.packages import attempt_import
 from meerschaum.utils import get_pipes as _get_pipes
 from meerschaum.config._paths import API_UVICORN_CONFIG_PATH, API_UVICORN_RESOURCES_PATH
 from meerschaum.plugins import _api_plugins
-from meerschaum.utils.warnings import warn
+from meerschaum.utils.warnings import warn, dprint
 from meerschaum.utils.threading import RLock
 
 _locks = {'pipes': RLock(), 'connector': RLock(), 'uvicorn_config': RLock()}
@@ -44,8 +44,8 @@ uv = attempt_import('uv', lazy=False, check_update=CHECK_UPDATE)
     'starlette.responses',
     'multipart',
     'packaging.version',
-    lazy = False,
-    check_update = CHECK_UPDATE,
+    lazy=False,
+    check_update=CHECK_UPDATE,
 )
 (
     typing_extensions,
@@ -53,9 +53,9 @@ uv = attempt_import('uv', lazy=False, check_update=CHECK_UPDATE)
 ) = attempt_import(
     'typing_extensions',
     'uvicorn.workers',
-    lazy = False,
-    check_update = CHECK_UPDATE,
-    venv = None,
+    lazy=False,
+    check_update=CHECK_UPDATE,
+    venv=None,
 )
 from meerschaum.api._chain import check_allow_chaining, DISALLOW_CHAINING_MESSAGE
 uvicorn_config_path = API_UVICORN_RESOURCES_PATH / SERVER_ID / 'config.json'
@@ -75,7 +75,7 @@ def get_uvicorn_config() -> Dict[str, Any]:
     with _locks['uvicorn_config']:
         if uvicorn_config is None:
             try:
-                with open(uvicorn_config_path, 'r') as f:
+                with open(uvicorn_config_path, 'r', encoding='utf-8') as f:
                     uvicorn_config = json.load(f)
                 _uvicorn_config = uvicorn_config
             except Exception as e:
@@ -93,12 +93,12 @@ debug = get_uvicorn_config().get('debug', False)
 no_dash = get_uvicorn_config().get('no_dash', False)
 no_auth = get_uvicorn_config().get('no_auth', False)
 private = get_uvicorn_config().get('private', False)
+production = get_uvicorn_config().get('production', False)
 _include_dash = (not no_dash)
 
 connector = None
 def get_api_connector(instance_keys: Optional[str] = None):
     """Create the instance connector."""
-    from meerschaum.utils.debug import dprint
     global connector
     with _locks['connector']:
         if connector is None:
@@ -111,6 +111,40 @@ def get_api_connector(instance_keys: Optional[str] = None):
         dprint(f"API instance connector: {connector}")
     return connector
 
+cache_connector = None
+def get_cache_connector(connector_keys: Optional[str] = None):
+    """Return the `valkey` connector if running in production."""
+    global cache_connector
+    if cache_connector is not None:
+        return cache_connector
+
+    if not production:
+        return None
+
+    enable_valkey_cache = get_config('system', 'experimental', 'valkey_session_cache')
+    if not enable_valkey_cache:
+        return None
+
+    connector_keys = connector_keys or get_config(
+        'system', 'api', 'cache', 'connector',
+        warn=False,
+    )
+    if connector_keys is None:
+        return None
+
+    if not connector_keys.startswith('valkey'):
+        warn(f"Invalid cache connector '{connector_keys}'.")
+        return None
+
+    if cache_connector is None:
+        from meerschaum.connectors.parse import parse_instance_keys
+        cache_connector = parse_instance_keys(connector_keys)
+
+    if debug:
+        dprint(f"Cache connector: {cache_connector}")
+
+    return cache_connector
+
 
 _pipes = None
 def pipes(refresh=False):
@@ -138,17 +172,17 @@ def get_pipe(connector_keys, metric_key, location_key, refresh=False):
 
 app = fastapi.FastAPI(
     title = 'Meerschaum API',
-    description = __doc__,
-    version = __version__,
-    contact = {
+    description=__doc__,
+    version=__version__,
+    contact={
         'name': 'Bennett Meares',
         'url': 'https://meerschaum.io',
     },
-    license_info = {
+    license_info={
         'name': 'Apache 2.0',
         'url': 'https://www.apache.org/licenses/LICENSE-2.0.html',
     },
-    open_api_tags = [
+    open_api_tags=[
         {
             'name': 'Pipes',
             'description': 'Access pipes by indexing their keys.',

meerschaum/api/_events.py

@@ -6,26 +6,29 @@
 Declare FastAPI events in this module (startup, shutdown, etc.).
 """
 
-import sys, os, time
+import sys
+import os
+import time
 from meerschaum.api import (
     app,
     get_api_connector,
+    get_cache_connector,
     get_uvicorn_config,
     debug,
     no_dash,
-    uvicorn_config_path,
 )
 from meerschaum.utils.debug import dprint
 from meerschaum.connectors.poll import retry_connect
 from meerschaum.utils.warnings import warn
-from meerschaum._internal.term.tools import is_webterm_running
 from meerschaum.jobs import (
+    get_jobs,
     start_check_jobs_thread,
     stop_check_jobs_thread,
-    get_executor_keys_from_context,
 )
+from meerschaum.config.static import STATIC_CONFIG
+
+TEMP_PREFIX: str = STATIC_CONFIG['api']['jobs']['temp_prefix']
 
-_check_jobs_thread = None
 
 @app.on_event("startup")
 async def startup():
@@ -39,10 +42,17 @@ async def startup():
 
         connected = retry_connect(
             get_api_connector(),
-            workers = get_uvicorn_config().get('workers', None),
-            debug = debug
+            workers=get_uvicorn_config().get('workers', None),
+            debug=debug
         )
-    except Exception as e:
+        cache_connector = get_cache_connector()
+        if cache_connector is not None:
+            connected = retry_connect(
+                cache_connector,
+                workers=get_uvicorn_config().get('workers', None),
+                debug=debug,
+            )
+    except Exception:
         import traceback
         traceback.print_exc()
         connected = False
@@ -51,8 +61,7 @@ async def startup():
         await shutdown()
         os._exit(1)
 
-    if get_executor_keys_from_context() == 'local':
-        start_check_jobs_thread()
+    start_check_jobs_thread()
 
 
 @app.on_event("shutdown")
@@ -65,17 +74,17 @@ async def shutdown():
     if get_api_connector().type == 'sql':
         get_api_connector().engine.dispose()
 
-    if get_executor_keys_from_context() == 'local':
-        stop_check_jobs_thread()
+    stop_check_jobs_thread()
 
-    from meerschaum.api.routes._actions import _temp_jobs
-    for name, job in _temp_jobs.items():
+    temp_jobs = {
+        name: job
+        for name, job in get_jobs(include_hidden=True).items()
+        if name.startswith(TEMP_PREFIX)
+    }
+    for job in temp_jobs.values():
         job.delete()
 
     ### Terminate any running jobs left over.
     if 'meerschaum.api.dash' in sys.modules:
-        from meerschaum.api.dash.actions import running_jobs, stop_action
         from meerschaum.api.dash.webterm import stop_webterm
         stop_webterm()
-        for session_id in running_jobs:
-            stop_action({'session-store.data': {'session-id': session_id}})

meerschaum/api/_oauth2.py

@@ -39,10 +39,10 @@ def generate_secret_key() -> str:
     from meerschaum.config._paths import API_SECRET_KEY_PATH
     if not API_SECRET_KEY_PATH.exists():
         secret_key = os.urandom(24).hex()
-        with open(API_SECRET_KEY_PATH, 'w+') as f:
+        with open(API_SECRET_KEY_PATH, 'w+', encoding='utf-8') as f:
             f.write(secret_key)
     else:
-        with open(API_SECRET_KEY_PATH, 'r') as f:
+        with open(API_SECRET_KEY_PATH, 'r', encoding='utf-8') as f:
             secret_key = f.read()
 
     return secret_key.encode('utf-8')

meerschaum/api/_websockets.py

@@ -6,13 +6,13 @@
 Implement WebSockets for the Meerschaum API via FastAPI.
 """
 
-import time, uuid
 from datetime import datetime, timezone
+
 from meerschaum.api import (
-    app, get_api_connector, get_uvicorn_config, debug, fastapi, endpoints
+    app,
+    fastapi,
+    endpoints,
 )
-from meerschaum.api.dash.users import is_session_authenticated
-from meerschaum.utils.typing import Optional
 
 _websocket_endpoint = endpoints['websocket']
 
@@ -28,6 +28,7 @@ async def websocket_endpoint(
     """
     Communicate with the Web Interface over a websocket.
     """
+    from meerschaum.api.dash.sessions import is_session_authenticated
     await websocket.accept()
     try:
         initial_data = await websocket.receive_json()

meerschaum/api/dash/__init__.py

@@ -16,7 +16,6 @@ flask_compress = attempt_import('flask_compress', lazy=False)
 _monkey_patch_get_distribution('flask-compress', flask_compress.__version__)
 dash = attempt_import('dash', lazy=False)
 
-
 from meerschaum.utils.typing import List, Optional
 from meerschaum.config.static import _static_config
 from meerschaum.api import (
@@ -38,14 +37,6 @@ with warnings.catch_warnings():
 html, dcc = import_html(), import_dcc()
 from meerschaum.api.dash.components import location
 
-active_sessions = {}
-authenticated_sessions = {}
-unauthenticated_sessions = {}
-running_jobs = {}
-running_monitors = {}
-stopped_jobs = {}
-stopped_monitors = {}
-
 stylesheets = [
     '/static/css/bootstrap.min.css',
     '/static/css/dbc_dark.css',
@@ -54,12 +45,12 @@ stylesheets = [
 scripts = ['/static/js/node_modules/xterm/lib/xterm.js']
 dash_app = enrich.DashProxy(
     __name__,
-    title = 'Meerschaum Web',
-    requests_pathname_prefix = endpoints['dash'] + '/',
-    external_stylesheets = stylesheets,
-    update_title = None,
-    suppress_callback_exceptions = True,
-    transforms = [
+    title='Meerschaum Web',
+    requests_pathname_prefix=endpoints['dash'] + '/',
+    external_stylesheets=stylesheets,
+    update_title=None,
+    suppress_callback_exceptions=True,
+    transforms=[
         enrich.TriggerTransform(),
         enrich.MultiplexerTransform(),
     ],
@@ -67,7 +58,7 @@ dash_app = enrich.DashProxy(
 
 dash_app.layout = html.Div([
     location,
-    dcc.Store(id='session-store', storage_type='session', data={}),
+    dcc.Store(id='session-store', storage_type='local', data={}),
     html.Div([], id='page-layout-div'),
 ])
 

meerschaum/api/dash/callbacks/__init__.py

@@ -11,6 +11,7 @@ import meerschaum.api.dash.callbacks.login
 import meerschaum.api.dash.callbacks.plugins
 import meerschaum.api.dash.callbacks.jobs
 import meerschaum.api.dash.callbacks.register
+import meerschaum.api.dash.callbacks.pipes
 from meerschaum.api.dash.callbacks.custom import init_dash_plugins, add_plugin_pages
 
 init_dash_plugins()

meerschaum/api/dash/callbacks/dashboard.py

@@ -7,18 +7,21 @@ Callbacks for the main dashboard.
 """
 
 from __future__ import annotations
-import sys, textwrap, json, datetime, uuid
+
+import textwrap
+import json
+import uuid
 from dash.dependencies import Input, Output, State, ALL, MATCH
 from dash.exceptions import PreventUpdate
-from meerschaum.config import get_config
-from meerschaum.config.static import _static_config
 from meerschaum.utils.typing import List, Optional, Any, Tuple
 from meerschaum.api import get_api_connector, endpoints, no_auth, CHECK_UPDATE
-from meerschaum.api.dash import (
-    dash_app, debug, pipes, _get_pipes,
-    active_sessions, authenticated_sessions, unauthenticated_sessions,
+from meerschaum.api.dash import dash_app, debug
+from meerschaum.api.dash.sessions import (
+    is_session_active,
+    delete_session,
+    set_session,
 )
-from meerschaum.api.dash.users import is_session_authenticated
+from meerschaum.api.dash.sessions import is_session_authenticated
 from meerschaum.api.dash.connectors import get_web_connector
 from meerschaum.connectors.parse import parse_instance_keys
 from meerschaum.api.dash.pipes import get_pipes_cards, pipe_from_ctx, accordion_items_from_pipe
@@ -30,21 +33,14 @@ from meerschaum.api.dash.webterm import get_webterm
 from meerschaum.api.dash.components import (
     alert_from_success_tuple, console_div, build_cards_grid,
 )
-from meerschaum.api.dash.actions import execute_action, stop_action
-import meerschaum.api.dash.pages as pages
+from meerschaum.api.dash import pages
 from meerschaum.utils.typing import Dict
-from meerschaum.utils.debug import dprint
 from meerschaum.utils.packages import attempt_import, import_html, import_dcc
-from meerschaum.utils.misc import (
-    string_to_dict, get_connector_labels, json_serialize_datetime, filter_keywords,
-    flatten_list,
-)
+from meerschaum.utils.misc import filter_keywords, flatten_list
 from meerschaum.utils.yaml import yaml
 from meerschaum.actions import get_subactions, actions
-from meerschaum._internal.arguments._parser import get_arguments_triggers, parser
+from meerschaum._internal.arguments._parser import parser
 from meerschaum.connectors.sql._fetch import set_pipe_query
-import meerschaum as mrsm
-import json
 dash = attempt_import('dash', lazy=False, check_update=CHECK_UPDATE)
 dbc = attempt_import('dash_bootstrap_components', lazy=False, check_update=CHECK_UPDATE)
 dcc, html = import_dcc(check_update=CHECK_UPDATE), import_html(check_update=CHECK_UPDATE)
@@ -101,6 +97,7 @@ _paths = {
     ''        : pages.dashboard.layout,
     'plugins' : pages.plugins.layout,
     'register': pages.register.layout,
+    'pipes'   : pages.pipes.layout,
 }
 _required_login = {''}
 
@@ -108,21 +105,21 @@ _required_login = {''}
 @dash_app.callback(
     Output('page-layout-div', 'children'),
     Output('session-store', 'data'),
-    Input('location', 'pathname'),
+    Input('mrsm-location', 'pathname'),
     Input('session-store', 'data'),
-    State('location', 'href'),
+    State('mrsm-location', 'href'),
 )
 def update_page_layout_div(
-        pathname: str, 
-        session_store_data: Dict[str, Any],
-        location_href: str,
-    ) -> Tuple[List[Any], Dict[str, Any]]:
+    pathname: str,
+    session_store_data: Dict[str, Any],
+    location_href: str,
+) -> Tuple[List[Any], Dict[str, Any]]:
     """
     Route the user to the correct page.
 
     Parameters
     ----------
-    pathname: str :
+    pathname: str
         The path in the browser.
         
     session_store_data: Dict[str, Any]:
@@ -132,20 +129,19 @@ def update_page_layout_div(
     -------
     A tuple of the page layout and new session store data.
     """
-    ctx = dash.callback_context
     dash_endpoint = endpoints['dash']
     try:
-        session_id = session_store_data.get('session-id', None) 
+        session_id = session_store_data.get('session-id', None)
     except AttributeError:
         session_id = None
 
     ### Bypass login if `--no-auth` is specified.
-    if session_id not in active_sessions and no_auth:
+    if not is_session_active(session_id) and no_auth:
         session_store_data['session-id'] = str(uuid.uuid4())
-        active_sessions[session_store_data['session-id']] = {'username': 'no-auth'}
+        set_session(session_id, {'username': 'no-auth'})
 
         ### Sometimes the href is an empty string, so store it here for later.
-        session_store_data['location.href'] = location_href
+        session_store_data['mrsm-location.href'] = location_href
         session_store_to_return = session_store_data
     else:
         session_store_to_return = dash.no_update
@@ -175,7 +171,7 @@ def update_page_layout_div(
         path_str
         if no_auth or path_str not in _required_login else (
             path_str
-            if session_id in active_sessions
+            if is_session_active(session_id)
             else 'login'
         )
     )
@@ -194,7 +190,7 @@ def update_page_layout_div(
     Input('get-plugins-button', 'n_clicks'),
     Input('get-users-button', 'n_clicks'),
     Input('get-graphs-button', 'n_clicks'),
-    State('location', 'href'),
+    State('mrsm-location', 'href'),
     State('session-store', 'data'),
     State('webterm-div', 'children'),
     *keys_state,
@@ -205,7 +201,6 @@ def update_content(*args):
     and execute the appropriate function.
     """
     ctx = dash.callback_context
-    location_href = ctx.states['session-store.data'].get('location.href', None)
     session_id = ctx.states['session-store.data'].get('session-id', None)
     authenticated = is_session_authenticated(str(session_id))
 
@@ -237,9 +232,6 @@ def update_content(*args):
         'get-pipes-button': 1,
         'get-jobs-button': 2,
     }
-    
-    ### NOTE: stop the running action if it exists
-    stop_action(ctx.states)
 
     content, alerts = triggers[trigger](
         ctx.states,
@@ -311,9 +303,9 @@ dash_app.clientside_callback(
         return url;
     }
     """,
-    Output('location', 'href'),
+    Output('mrsm-location', 'href'),
     Input('go-button', 'n_clicks'),
-    State('location', 'href'),
+    State('mrsm-location', 'href'),
     State('connector-keys-dropdown', 'value'),
     State('metric-keys-dropdown', 'value'),
     State('location-keys-dropdown', 'value'),
@@ -490,12 +482,12 @@ def update_flags(input_flags_dropdown_values, n_clicks, input_flags_texts):
     *keys_state
 )
 def update_keys_options(
-        connector_keys: Optional[List[str]],
-        metric_keys: Optional[List[str]],
-        location_keys: Optional[List[str]],
-        instance_keys: Optional[str],
-        *keys
-    ):
+    connector_keys: Optional[List[str]],
+    metric_keys: Optional[List[str]],
+    location_keys: Optional[List[str]],
+    instance_keys: Optional[str],
+    *keys
+):
     """
     Update the keys dropdown menus' options.
     """
@@ -543,9 +535,9 @@ def update_keys_options(
         _keys = fetch_pipes_keys(
             'registered',
             get_web_connector(ctx.states),
-            connector_keys = _ck_filter,
-            metric_keys = _mk_filter,
-            location_keys = _lk_filter,
+            connector_keys=_ck_filter,
+            metric_keys=_mk_filter,
+            location_keys=_lk_filter,
         )
     except Exception as e:
         instance_alerts += [alert_from_success_tuple((False, str(e)))]
@@ -637,7 +629,7 @@ dash_app.clientside_callback(
         return url;
     }
     """,
-    Output('location', 'href'),
+    Output('mrsm-location', 'href'),
     Input('instance-select', 'value'),
 )
 
@@ -710,11 +702,12 @@ dash_app.clientside_callback(
         return url;
     }
     """,
-    Output('location', 'href'),
+    Output('mrsm-location', 'href'),
     Input('console-pre', 'children'),
-    State('location', 'href'),
+    State('mrsm-location', 'href'),
 )
 
+
 @dash_app.callback(
     Output("download-dataframe-csv", "data"),
     Input({'type': 'pipe-download-csv-button', 'index': ALL}, 'n_clicks'),
@@ -736,7 +729,7 @@ def download_pipe_csv(n_clicks):
     filename = str(pipe.target) + f" {begin} - {end}.csv"
     try:
         df = pipe.get_data(begin=begin, end=end, debug=debug)
-    except Exception as e:
+    except Exception:
         df = None
     if df is not None:
         return dcc.send_data_frame(df.to_csv, filename, index=False)
@@ -749,6 +742,9 @@ def download_pipe_csv(n_clicks):
     State('session-store', 'data'),
 )
 def update_pipe_accordion(item, session_store_data):
+    """
+    Expand the pipe accordion item and lazy load.
+    """
     if item is None:
         raise PreventUpdate
 
@@ -918,7 +914,7 @@ dash_app.clientside_callback(
     """,
     Output('content-div-right', 'children'),
     Input({'type': 'manage-pipe-button', 'index': ALL, 'action': ALL}, 'n_clicks'),
-    State('location', 'href'),
+    State('mrsm-location', 'href'),
 )
 
 @dash_app.callback(
@@ -948,15 +944,15 @@ def toggle_navbar_collapse(n_clicks: Optional[int], is_open: bool) -> bool:
 
 
 @dash_app.callback(
-    Output('location', 'pathname'),
+    Output('mrsm-location', 'pathname'),
     Output('session-store', 'data'),
     Input("sign-out-button", "n_clicks"),
     State('session-store', 'data'),
 )
 def sign_out_button_click(
-        n_clicks: Optional[int],
-        session_store_data: Dict[str, Any],
-    ):
+    n_clicks: Optional[int],
+    session_store_data: Dict[str, Any],
+):
     """
     When the sign out button is clicked, remove the session data and redirect to the login page.
     """
@@ -964,9 +960,7 @@ def sign_out_button_click(
         raise PreventUpdate
     session_id = session_store_data.get('session-id', None)
     if session_id:
-        _ = active_sessions.pop(session_id, None)
-        _ = authenticated_sessions.pop(session_id, None)
-        _ = unauthenticated_sessions.pop(session_id, None)
+        delete_session(session_id)
     return endpoints['dash'], {}
 
 

meerschaum/api/dash/callbacks/jobs.py

@@ -7,36 +7,35 @@ Callbacks for jobs' cards.
 """
 
 from __future__ import annotations
+
 import json
-import functools
 import time
 import traceback
 from datetime import datetime, timezone
-import meerschaum as mrsm
 from meerschaum.utils.typing import Optional, Dict, Any
-from meerschaum.api import get_api_connector, endpoints, CHECK_UPDATE
-from meerschaum.api.dash import dash_app, debug, active_sessions
+from meerschaum.api import CHECK_UPDATE
+from meerschaum.api.dash import dash_app
+from meerschaum.api.dash.sessions import get_username_from_session
 from meerschaum.utils.packages import attempt_import, import_dcc, import_html
-dash = attempt_import('dash', lazy=False, check_update=CHECK_UPDATE)
-from dash.exceptions import PreventUpdate
-from dash.dependencies import Input, Output, State, ALL, MATCH
-from dash import Patch
-html, dcc = import_html(check_update=CHECK_UPDATE), import_dcc(check_update=CHECK_UPDATE)
-import dash_bootstrap_components as dbc
-from meerschaum.api.dash.components import alert_from_success_tuple, build_cards_grid
-from dash.exceptions import PreventUpdate
+from meerschaum.api.dash.components import alert_from_success_tuple
 from meerschaum.api.dash.jobs import (
     build_manage_job_buttons_div_children,
     build_status_children,
     build_process_timestamps_children,
 )
 from meerschaum.jobs import Job
-from meerschaum.api.dash.users import is_session_authenticated
+from meerschaum.api.dash.sessions import is_session_authenticated
+dash = attempt_import('dash', lazy=False, check_update=CHECK_UPDATE)
+html, dcc = import_html(check_update=CHECK_UPDATE), import_dcc(check_update=CHECK_UPDATE)
+from dash.exceptions import PreventUpdate
+from dash.dependencies import Input, Output, State, ALL, MATCH
+import dash_bootstrap_components as dbc
+
 
 @dash_app.callback(
     Output("download-logs", "data"),
     Input({'type': 'job-download-logs-button', 'index': ALL}, 'n_clicks'),
-    prevent_initial_call = True,
+    prevent_initial_call=True,
 )
 def download_job_logs(n_clicks):
     """
@@ -84,7 +83,7 @@ def manage_job_button_click(
         raise PreventUpdate
 
     session_id = session_data.get('session-id', None)
-    username = active_sessions.get(session_id, {}).get('username', None)
+    username = get_username_from_session(session_id)
 
     if not is_session_authenticated(session_id):
         success, msg = False, f"User '{username}' is not authenticated to manage jobs."
@@ -182,7 +181,7 @@ dash_app.clientside_callback(
     """,
     Output('content-div-right', 'children'),
     Input({'type': 'follow-logs-button', 'index': ALL}, 'n_clicks'),
-    State('location', 'href'),
+    State('mrsm-location', 'href'),
 )