meerschaum 2.3.6__py3-none-any.whl → 2.4.0__py3-none-any.whl

meerschaum/api/dash/sessions.py

@@ -0,0 +1,176 @@
+#! /usr/bin/env python3
+# vim:fenc=utf-8
+
+"""
+Define caching functions for session management.
+"""
+
+import json
+
+from meerschaum.utils.typing import Union, Dict, Any, Optional
+from meerschaum.api import debug, no_auth
+from meerschaum.api import get_cache_connector, get_api_connector
+from meerschaum.core import User
+from meerschaum.config import get_config
+from meerschaum.utils.warnings import dprint
+
+SESSION_KEY_TEMPLATE: str = 'mrsm_session_id:{session_id}'
+EXPIRES_SECONDS: int = get_config('system', 'api', 'cache', 'session_expires_minutes') * 60
+_active_sessions: Dict[str, Dict[str, Any]] = {}
+
+
+def get_session_key(session_id: str) -> str:
+    """
+    Return the session key for the cache connector.
+    """
+    return SESSION_KEY_TEMPLATE.format(session_id=session_id)
+
+
+def set_session(session_id: str, session_data: Dict[str, Any]):
+    """
+    Set a `session_id` to a dictionary.
+    """
+    conn = get_cache_connector()
+    if conn is None:
+        _active_sessions[session_id] = session_data
+        if debug:
+            dprint(f"Setting in-memory data for {session_id}:\n{session_data}")
+        return
+
+    session_key = get_session_key(session_id)
+    session_data_str = json.dumps(session_data, separators=(',', ':'))
+    if debug:
+        dprint(f"Setting production data for {session_id=}:\n{session_data_str}")
+
+    conn.set(session_key, session_data_str, ex=EXPIRES_SECONDS)
+
+
+def update_session(session_id: Optional[str], session_data: Dict[str, Any]):
+    """
+    Update the session's data dictionary.
+    """
+    existing_session_data = get_session_data(session_id) or {}
+    existing_session_data.update(session_data)
+    set_session(str(session_id), existing_session_data)
+
+
+def get_session_data(session_id: Optional[str]) -> Union[Dict[str, Any], None]:
+    """
+    Return the session data dictionary.
+    """
+    if debug:
+        dprint(f"Getting session data for {session_id=}")
+    conn = get_cache_connector()
+    if conn is None:
+        return _active_sessions.get(str(session_id), None)
+
+    session_key = get_session_key(str(session_id))
+    session_data_str = conn.get(session_key)
+    if not session_data_str:
+        return None
+
+    return json.loads(session_data_str)
+
+
+def get_username_from_session(session_id: Optional[str]) -> Union[str, None]:
+    """
+    If a `session_id` has been set, return the username.
+    Otherwise return `None`.
+    """
+    if debug:
+        dprint(f"Getting username for {session_id=}")
+    session_data = get_session_data(session_id)
+    if session_data is None:
+        return None
+
+    return session_data.get('username', None)
+
+
+def is_session_active(session_id: Union[str, None]) -> bool:
+    """
+    Return whether a given `session_id` has been set.
+    """
+    return get_username_from_session(str(session_id)) is not None
+
+
+def delete_session(session_id: str):
+    """
+    Delete a session if it's been set.
+    """
+    if debug:
+        dprint(f"Deleting {session_id=}")
+    conn = get_cache_connector()
+    if conn is None:
+        _ = _active_sessions.pop(session_id, None)
+        return
+
+    session_key = get_session_key(session_id)
+    conn.client.delete(session_key)
+
+
+def is_session_authenticated(session_id: Optional[str]) -> bool:
+    """
+    Check is a session ID is active.
+    If running in secure mode, check whether a session ID corresponds to an admin.
+
+    Parameters
+    ----------
+    session_id: str
+        The session UUID.
+
+    Returns
+    -------
+    A bool whether the session is authenticated to perform actions.
+    """
+    if debug:
+        dprint(f"Checking authentication for {session_id=}")
+
+    if no_auth:
+        return True
+
+    if session_id is None:
+        return False
+
+    session_data = get_session_data(session_id)
+    if session_data is None:
+        return False
+    username = session_data.get('username', None)
+    if username is None:
+        return False
+
+    cached_auth = session_data.get('authenticated', None)
+    if cached_auth is not None:
+        return cached_auth
+
+    permissions = get_config('system', 'api', 'permissions')
+    allow_non_admin = permissions.get('actions', {}).get('non_admin', False)
+
+    is_auth = True if allow_non_admin else session_is_admin(session_id)
+    update_session(session_id, {'authenticated': is_auth})
+    return is_auth
+
+
+def session_is_admin(session_id: str) -> bool:
+    """
+    Check whether a session ID corresponds to an admin user.
+    """
+    if debug:
+        dprint(f"Check admin for {session_id=}")
+    session_data = get_session_data(session_id)
+    if session_data is None:
+        return False
+
+    username = session_data.get('username', None)
+    if username is None:
+        return False
+
+    cached_admin = session_data.get('admin', None)
+    if cached_admin is not None:
+        return cached_admin
+
+    conn = get_api_connector()
+    user = User(username, instance=conn)
+    user_type = conn.get_user_type(user, debug=debug)
+    is_admin = user_type == 'admin'
+    update_session(session_id, {'admin': is_admin})
+    return is_admin

meerschaum/api/dash/users.py

@@ -8,16 +8,14 @@ Functions for interacting with users.
 
 from __future__ import annotations
 
-from meerschaum.api import debug, CHECK_UPDATE, get_api_connector, no_auth
-from meerschaum.api.dash import active_sessions, authenticated_sessions, unauthenticated_sessions
+from meerschaum.api import debug, CHECK_UPDATE
 from meerschaum.api.dash.connectors import get_web_connector
-from meerschaum.utils.typing import WebState, SuccessTuple, List, Tuple, Optional
+from meerschaum.utils.typing import WebState, SuccessTuple, List, Tuple
 from meerschaum.utils.packages import attempt_import, import_html, import_dcc
-from meerschaum.config import get_config
-from meerschaum.core import User
 dcc, html = import_dcc(check_update=CHECK_UPDATE), import_html(check_update=CHECK_UPDATE)
 dbc = attempt_import('dash_bootstrap_components', lazy=False, check_update=CHECK_UPDATE)
 
+
 def get_users_cards(state: WebState) -> Tuple[List[dbc.Card], List[SuccessTuple]]:
     """
     Return the cards and alerts for users.
@@ -34,39 +32,3 @@ def get_users_cards(state: WebState) -> Tuple[List[dbc.Card], List[SuccessTuple]
         )
 
     return cards, alerts
-
-
-def is_session_authenticated(session_id: str) -> bool:
-    """
-    Check is a session ID is active.
-    If running in secure mode, check whether a session ID corresponds to an admin.
-
-    Parameters
-    ----------
-    session_id: str
-        The session UUID.
-
-    Returns
-    -------
-    A bool whether the session is authenticated to perform actions.
-    """
-    if no_auth:
-        return True
-    if session_id in unauthenticated_sessions:
-        return False
-    if session_id in authenticated_sessions:
-        return True
-    permissions = get_config('system', 'api', 'permissions')
-    allow_non_admin = permissions.get('actions', {}).get('non_admin', False)
-    if allow_non_admin:
-        return True
-    conn = get_api_connector()
-    username = active_sessions.get(session_id, {}).get('username', None)
-    user = User(username, instance=conn)
-    user_type = conn.get_user_type(user, debug=debug)
-    is_auth = user_type == 'admin'
-    if is_auth:
-        authenticated_sessions[session_id] = username
-    else:
-        unauthenticated_sessions[session_id] = username
-    return is_auth

meerschaum/api/dash/webterm.py

@@ -7,16 +7,12 @@ Functions for interacting with the Webterm via the dashboard.
 """
 
 import time
-from urllib.parse import urlparse
-from meerschaum.config import get_config
-from meerschaum.api import debug, CHECK_UPDATE, get_api_connector, no_auth
-from meerschaum.api.dash import active_sessions
-from meerschaum.api.dash.users import is_session_authenticated
+from meerschaum.api import CHECK_UPDATE, get_api_connector
+from meerschaum.api.dash.sessions import is_session_authenticated, get_username_from_session
 from meerschaum.api.dash.components import alert_from_success_tuple, console_div
-from meerschaum.utils.typing import WebState, SuccessTuple, List, Tuple, Optional, Any
-from meerschaum.utils.packages import attempt_import, import_html, import_dcc, run_python_package
+from meerschaum.utils.typing import WebState, Tuple, Any
+from meerschaum.utils.packages import attempt_import, import_html, import_dcc
 from meerschaum._internal.term.tools import is_webterm_running
-from meerschaum.config.static import STATIC_CONFIG
 from meerschaum.utils.threading import Thread, RLock
 dcc, html = import_dcc(check_update=CHECK_UPDATE), import_html(check_update=CHECK_UPDATE)
 dbc = attempt_import('dash_bootstrap_components', lazy=False, check_update=CHECK_UPDATE)
@@ -30,7 +26,7 @@ def get_webterm(state: WebState) -> Tuple[Any, Any]:
     Start the webterm and return its iframe.
     """
     session_id = state['session-store.data'].get('session-id', None)
-    username = active_sessions.get(session_id, {}).get('username', None)
+    username = get_username_from_session(session_id)
     if not is_session_authenticated(session_id):
         msg = f"User '{username}' is not authorized to access the webterm."
         return (
@@ -57,24 +53,23 @@ def get_webterm(state: WebState) -> Tuple[Any, Any]:
     return console_div, [alert_from_success_tuple((False, "Could not start the webterm server."))]
 
 
-
 webterm_procs = {}
 def start_webterm() -> None:
     """
     Start the webterm thread.
     """
-    from meerschaum._internal.entry import entry
     from meerschaum.utils.packages import run_python_package
 
     def run():
+        conn = get_api_connector()
         _ = run_python_package(
             'meerschaum',
-            ['start', 'webterm'],
-            capture_output = True,
-            as_proc = True,
-            store_proc_dict = webterm_procs,
-            store_proc_key = 'process',
-            venv = None,
+            ['start', 'webterm', '-i', str(conn)],
+            capture_output=True,
+            as_proc=True,
+            store_proc_dict=webterm_procs,
+            store_proc_key='process',
+            venv=None,
         )
 
     with _locks['webterm_thread']:

meerschaum/api/resources/static/js/terminado.js

@@ -15,7 +15,7 @@ function make_terminal(element, size, ws_url) {
   term.attachCustomKeyEventHandler(copyPasteKeyEventHandler);
   term.open(element);
 
-  sessionStore = sessionStorage.getItem("session-store");
+  sessionStore = localStorage.getItem("session-store");
   ws.onopen = function (event) {
     ws.send(sessionStore);
     ws.send(

meerschaum/api/routes/_actions.py

@@ -7,28 +7,13 @@ Execute Meerschaum Actions via the API
 """
 
 from __future__ import annotations
-import asyncio
-import traceback
-import shlex
-from functools import partial
-from datetime import datetime, timezone
-
-from fastapi import WebSocket, WebSocketDisconnect
-from websockets.exceptions import ConnectionClosedError
-
-from meerschaum.utils.misc import generate_password
-from meerschaum.jobs import Job
-from meerschaum.utils.warnings import warn
-from meerschaum.utils.typing import SuccessTuple, Union, List, Dict, Any
+
+from meerschaum.utils.typing import SuccessTuple, List, Dict, Any
 from meerschaum.api import (
     fastapi, app, endpoints, get_api_connector, debug, manager, private, no_auth
 )
 from meerschaum.actions import actions
-import meerschaum.core
 from meerschaum.config import get_config
-from meerschaum._internal.arguments._parse_arguments import parse_dict_to_sysargs, parse_arguments
-from meerschaum.api.routes._jobs import clean_sysargs
-from meerschaum.jobs._Job import StopMonitoringLogs
 
 actions_endpoint = endpoints['actions']
 
@@ -82,7 +67,7 @@ def do_action_legacy(
     ----------
     action: str
         The action to perform.
-        
+
     keywords: Dict[str, Any]
         The keywords dictionary to pass to the action.
 
@@ -91,7 +76,6 @@ def do_action_legacy(
     A `SuccessTuple`.
     """
     if curr_user is not None and curr_user.type != 'admin':
-        from meerschaum.config import get_config
         allow_non_admin = get_config(
             'system', 'api', 'permissions', 'actions', 'non_admin', patch=True
         )
@@ -103,7 +87,7 @@ def do_action_legacy(
                 + "and search for 'permissions'. "
                 + "\nUnder the keys 'api:permissions:actions', "
                 + "you can allow non-admin users to perform actions."
-        )
+            )
 
     if action not in actions:
         return False, f"Invalid action '{action}'."

meerschaum/api/routes/_jobs.py

@@ -9,7 +9,6 @@ Manage jobs via the Meerschaum API.
 from __future__ import annotations
 
 import os
-import select
 import asyncio
 import traceback
 from datetime import datetime
@@ -18,12 +17,10 @@ from functools import partial
 
 from fastapi import WebSocket, WebSocketDisconnect
 
-from meerschaum.utils.typing import Dict, Any, SuccessTuple, List, Optional, Union
+from meerschaum.utils.typing import Dict, Any, SuccessTuple, List, Union
 from meerschaum.jobs import (
     get_jobs as _get_jobs,
     Job,
-    StopMonitoringLogs,
-    get_executor_keys_from_context,
 )
 from meerschaum.utils.warnings import warn
 
@@ -32,14 +29,13 @@ from meerschaum.api import (
     app,
     endpoints,
     manager,
-    debug,
     no_auth,
-    private,
 )
 from meerschaum.config.static import STATIC_CONFIG
 
 JOBS_STDIN_MESSAGE: str = STATIC_CONFIG['api']['jobs']['stdin_message']
 JOBS_STOP_MESSAGE: str = STATIC_CONFIG['api']['jobs']['stop_message']
+NONINTERACTIVE_ENV: str = STATIC_CONFIG['environment']['noninteractive']
 EXECUTOR_KEYS: str = 'local'
 
 
@@ -48,7 +44,8 @@ def _get_job(name: str):
     if systemd_job.exists():
         return systemd_job
 
-    return Job(name, executor_keys=EXECUTOR_KEYS)
+    job = Job(name, executor_keys=EXECUTOR_KEYS)
+    return job
 
 
 @app.get(endpoints['jobs'], tags=['Jobs'])
@@ -150,6 +147,10 @@ def create_job(
         name,
         clean_sysargs(sysargs),
         executor_keys=EXECUTOR_KEYS,
+        env={
+            NONINTERACTIVE_ENV: '1',
+            **dict(os.environ)
+        },
         _properties=properties,
     )
     if job.exists():

meerschaum/api/routes/_login.py

@@ -23,8 +23,8 @@ from meerschaum.api._oauth2 import CustomOAuth2PasswordRequestForm
 
 @manager.user_loader()
 def load_user(
-        username: str
-    ) -> User:
+    username: str
+) -> User:
     """
     Create the `meerschaum.core.User` object from the username.
     """
@@ -33,8 +33,8 @@ def load_user(
 
 @app.post(endpoints['login'], tags=['Users'])
 def login(
-        data: CustomOAuth2PasswordRequestForm = fastapi.Depends()
-    ) -> Dict[str, Any]:
+    data: CustomOAuth2PasswordRequestForm = fastapi.Depends()
+) -> Dict[str, Any]:
     """
     Login and set the session token.
     """

meerschaum/api/routes/_pipes.py

@@ -474,9 +474,9 @@ def get_pipe_data(
         df[col] = df[col].apply(lambda x: f'{x:f}' if isinstance(x, Decimal) else x)
 
     json_content = df.to_json(
-        date_format = 'iso',
-        orient = 'records',
-        date_unit = 'us',
+        date_format='iso',
+        orient='records',
+        date_unit='us',
     )
 
     return fastapi.Response(

meerschaum/api/routes/_webterm.py

@@ -8,10 +8,9 @@ Routes to the Webterm proxy.
 
 import asyncio
 from meerschaum.utils.typing import Optional
-from meerschaum.api import app, no_auth, manager, endpoints
+from meerschaum.api import app, endpoints
 from meerschaum.utils.packages import attempt_import
-from meerschaum.api.dash import active_sessions
-from meerschaum.api.dash.users import is_session_authenticated
+from meerschaum.api.dash.sessions import is_session_authenticated
 fastapi, fastapi_responses = attempt_import('fastapi', 'fastapi.responses')
 import starlette
 httpx = attempt_import('httpx')
@@ -25,9 +24,9 @@ PlainTextResponse = fastapi_responses.PlainTextResponse
 
 @app.get(endpoints['webterm'], tags=["Webterm"])
 async def get_webterm(
-        request: Request,
-        s: Optional[str] = None,
-    ) -> HTMLResponse:
+    request: Request,
+    s: Optional[str] = None,
+) -> HTMLResponse:
     """
     Get the main HTML template for the Webterm.
     """

meerschaum/config/_default.py

@@ -52,6 +52,14 @@ default_meerschaum_config = {
                 'protocol': 'https',
             },
         },
+        'valkey': {
+            'main': {
+                'host': 'localhost',
+                'username': 'default',
+                'password': 'mrsm',
+                'port': 6379,
+            },
+        },
     },
 }
 default_system_config = {
@@ -74,7 +82,7 @@ default_system_config = {
             },
         },
 
-        'api' : {
+        'api': {
         },
     },
     ### not to be confused with system_config['connectors']['api'], this is the configuration
@@ -88,6 +96,10 @@ default_system_config = {
             'proxy_headers': True,
             'forwarded_allow_ips': '*',
         },
+        'cache': {
+            'connector': 'valkey:main',
+            'session_expires_minutes': 43200,
+        },
         'permissions':       {
             'registration': {
                 'users': True,
@@ -97,7 +109,7 @@ default_system_config = {
             'actions': {
                 'non_admin': True,
             },
-            'chaining' : {
+            'chaining': {
                 'insecure_parent_instance': False,
                 'child_apis': False,
             },
@@ -112,6 +124,7 @@ default_system_config = {
         'inplace_sync': True,
         'uv_pip': True,
         'systemd_healthcheck': False,
+        'valkey_session_cache': True,
     },
 }
 default_pipes_config = {

meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "2.3.6"
+__version__ = "2.4.0"

meerschaum/config/stack/__init__.py

@@ -31,21 +31,29 @@ db_host = 'MRSM{stack:' + str(STACK_COMPOSE_FILENAME) + ':services:db:hostname}'
 api_port = "MRSM{meerschaum:connectors:api:main:port}"
 api_host = "api"
 
+valkey_hostname = "valkey"
+valkey_host = 'MRSM{stack:' + str(STACK_COMPOSE_FILENAME) + ':services:valkey:hostname}'
+valkey_port = "MRSM{meerschaum:connectors:valkey:main:port}"
+valkey_username = 'MRSM{meerschaum:connectors:valkey:main:username}'
+valkey_password = 'MRSM{meerschaum:connectors:valkey:main:password}'
+
 env_dict = {
-    'COMPOSE_PROJECT_NAME' : 'mrsm',
-    'TIMESCALEDB_VERSION' : 'latest-pg16-oss',
-    'POSTGRES_USER' : f'{db_user}',
-    'POSTGRES_PASSWORD' : f'{db_pass}',
-    'POSTGRES_DB' : f'{db_base}',
-    'MEERSCHAUM_API_HOSTNAME' : f'{api_host}',
-    'ALLOW_IP_RANGE' : '0.0.0.0/0',
-    'MEERSCHAUM_API_CONFIG_RESOURCES' : '/meerschaum',
+    'COMPOSE_PROJECT_NAME': 'mrsm',
+    'TIMESCALEDB_VERSION': 'latest-pg16-oss',
+    'POSTGRES_USER': db_user,
+    'POSTGRES_PASSWORD': db_pass,
+    'POSTGRES_DB': db_base,
+    'VALKEY_USERNAME': valkey_username,
+    'VALKEY_PASSWORD': valkey_password,
+    'MEERSCHAUM_API_HOSTNAME': api_host,
+    'ALLOW_IP_RANGE': '0.0.0.0/0',
+    'MEERSCHAUM_API_CONFIG_RESOURCES': '/meerschaum',
 }
 ### apply patch to host config to change hostname to the Docker service name
 env_dict['MEERSCHAUM_API_CONFIG'] = json.dumps(
     {
-        'meerschaum' : 'MRSM{!meerschaum}',
-        'system' : 'MRSM{!system}',
+        'meerschaum': 'MRSM{!meerschaum}',
+        'system': 'MRSM{!system}',
     },
     indent = 4,
 ).replace(
@@ -58,6 +66,7 @@ volumes = {
     'api_root': '/meerschaum',
     'meerschaum_db_data': '/var/lib/postgresql/data',
     'grafana_storage': '/var/lib/grafana',
+    'valkey_data': '/bitnami/valkey/data',
 }
 networks = {
     'frontend': None,
@@ -73,13 +82,19 @@ env_dict['MEERSCHAUM_API_PATCH'] = json.dumps(
                         'port': 5432,
                     },
                     'local': {
-                        'database': volumes['api_root'] + '/sqlite/mrsm_local.db'
+                        'database': volumes['api_root'] + '/sqlite/mrsm_local.db',
+                    },
+                },
+                'valkey': {
+                    'main': {
+                        'host': valkey_host,
+                        'port': 6379,
                     },
                 },
             },
         },
     },
-    indent = 4,
+    indent=4,
 )
 
 compose_header = """
@@ -113,19 +128,19 @@ default_docker_compose_config = {
                 ],
                 'interval': '5s',
                 'timeout': '3s',
-                'retries': 3
+                'retries': 5
             },
             'restart': 'always',
-            'image' : 'timescale/timescaledb:' + env_dict['TIMESCALEDB_VERSION'],
-            'ports' : [
+            'image': 'timescale/timescaledb:' + env_dict['TIMESCALEDB_VERSION'],
+            'ports': [
                 f'{db_port}:5432',
             ],
-            'hostname' : f'{db_hostname}',
-            'volumes' : [
+            'hostname': db_hostname,
+            'volumes': [
                 'meerschaum_db_data:' + volumes['meerschaum_db_data'],
             ],
             'shm_size': '1024m',
-            'networks' : [
+            'networks': [
                 'backend',
             ],
         },
@@ -156,11 +171,41 @@ default_docker_compose_config = {
                 'db': {
                     'condition': 'service_healthy',
                 },
+                'valkey': {
+                    'condition': 'service_healthy',
+                },
             },
-            'volumes' : [
+            'volumes': [
                 'api_root:' + volumes['api_root'],
             ],
         },
+        'valkey': {
+            'image': 'bitnami/valkey:latest',
+            'restart': 'always',
+            'environment': {
+                'VALKEY_PASSWORD': '<DOLLAR>VALKEY_PASSWORD',
+                'VALKEY_RDB_POLICY_DISABLED': 'no',
+                'VALKEY_RDB_POLICY': '900#1 600#5 300#10 120#50 60#1000 30#10000',
+            },
+            'hostname': valkey_hostname,
+            'ports': [
+                f'{valkey_port}:6379',
+            ],
+            'volumes': [
+                'valkey_data:' + volumes['valkey_data'],
+            ],
+            'healthcheck': {
+                'test': [
+                    'CMD', 'valkey-cli', 'ping',
+                ],
+                'interval': '5s',
+                'timeout': '3s',
+                'retries': 5,
+            },
+            'networks': [
+                'backend',
+            ],
+        },
         'grafana': {
             'image': 'grafana/grafana:latest',
             'ports': [