meerschaum 2.4.0.dev1__py3-none-any.whl → 2.4.0rc2__py3-none-any.whl

meerschaum/actions/start.py

@@ -304,24 +304,20 @@ def _start_jobs(
 
 
 def _start_gui(
-        action: Optional[List[str]] = None,
-        mrsm_instance: Optional[str] = None,
-        port: Optional[int] = None,
-        debug: bool = False,
-        **kw
-    ) -> SuccessTuple:
+    action: Optional[List[str]] = None,
+    mrsm_instance: Optional[str] = None,
+    port: Optional[int] = None,
+    debug: bool = False,
+    **kw
+) -> SuccessTuple:
     """
     Start the Meerschaum GUI application.
     """
-    from meerschaum.utils.daemon import Daemon
-    from meerschaum.utils.process import run_process
     from meerschaum.utils.venv import venv_exec
-    from meerschaum.utils.packages import (
-        run_python_package, attempt_import, venv_contains_package, pip_install
-    )
+    from meerschaum.utils.packages import attempt_import
     from meerschaum.utils.warnings import warn
     from meerschaum.utils.debug import dprint
-    from meerschaum.utils.networking import find_open_ports, is_port_in_use
+    from meerschaum.utils.networking import find_open_ports
     from meerschaum.connectors.parse import parse_instance_keys
     from meerschaum._internal.term.tools import is_webterm_running
     import platform
@@ -395,13 +391,14 @@ def _start_gui(
 
 
 def _start_webterm(
-        port: Optional[int] = None,
-        host: Optional[str] = None,
-        force: bool = False,
-        nopretty: bool = False,
-        sysargs: Optional[List[str]] = None,
-        **kw
-    ) -> SuccessTuple:
+    port: Optional[int] = None,
+    host: Optional[str] = None,
+    mrsm_instance: Optional[str] = None,
+    force: bool = False,
+    nopretty: bool = False,
+    sysargs: Optional[List[str]] = None,
+    **kw
+) -> SuccessTuple:
     """
     Start the Meerschaum Web Terminal.
     
@@ -409,15 +406,17 @@ def _start_webterm(
         - `-p`, `--port`
             The port to which the webterm binds.
             Defaults to 8765, and `--force` will choose the next available port.
-    
+
         - `--host`
             The host interface to which the webterm binds.
             Defaults to '127.0.0.1'.
+
+        - `-i`, '--instance'
+            The default instance to use for the Webterm shell.
     """
-    from meerschaum._internal.term import get_webterm_app_and_manager, tornado, tornado_ioloop
+    from meerschaum._internal.term import get_webterm_app_and_manager, tornado_ioloop
     from meerschaum._internal.term.tools import is_webterm_running
-    from meerschaum.utils.networking import find_open_ports, is_port_in_use
-    from meerschaum.utils.packages import attempt_import
+    from meerschaum.utils.networking import find_open_ports
     from meerschaum.utils.warnings import info
 
     if host is None:
@@ -426,7 +425,7 @@ def _start_webterm(
         port = 8765
     if sysargs is None:
         sysargs = ['start', 'webterm']
-    tornado_app, term_manager = get_webterm_app_and_manager()
+    tornado_app, term_manager = get_webterm_app_and_manager(instance_keys=mrsm_instance)
 
     if is_webterm_running(host, port):
         if force:
@@ -517,12 +516,12 @@ def _start_connectors(
 
     success = len(fails) == 0
     msg = (
-        f"Successfully started connector" + ('s' if len(successes) != 1 else '')
+        "Successfully started connector" + ('s' if len(successes) != 1 else '')
         + ' ' + items_str(successes) + '.'
     ) if success else f"Failed to start {len(fails)} connectors."
     if len(fails) > 0:
         msg += (
-            f"\n    Failed to start connector" + ('s' if len(fails) != 1 else '')
+            "\n    Failed to start connector" + ('s' if len(fails) != 1 else '')
             + ' ' + items_str(fails) + '.'
         )
 

meerschaum/actions/sync.py

@@ -9,10 +9,12 @@ NOTE: `sync` required a SQL connection and is not intended for client use
 """
 
 from __future__ import annotations
-from datetime import timedelta, datetime, timezone
+from datetime import timedelta
+
 import meerschaum as mrsm
 from meerschaum.utils.typing import SuccessTuple, Any, List, Optional, Tuple, Union
 
+
 def sync(
     action: Optional[List[str]] = None,
     **kw: Any
@@ -38,47 +40,45 @@ def sync(
 
 
 def _pipes_lap(
-        workers: Optional[int] = None,
-        debug: Optional[bool] = None,
-        unblock: bool = False,
-        force: bool = False,
-        min_seconds: int = 1,
-        verify: bool = False,
-        deduplicate: bool = False,
-        bounded: Optional[bool] = None,
-        chunk_interval: Union[timedelta, int, None] = None,
-        mrsm_instance: Optional[str] = None,
-        timeout_seconds: Optional[int] = None,
-        nopretty: bool = False,
-        _progress: Optional['rich.progress.Progress'] = None,
-        **kw: Any
-    ) -> Tuple[List[mrsm.Pipe], List[mrsm.Pipe]]:
+    workers: Optional[int] = None,
+    debug: Optional[bool] = None,
+    unblock: bool = False,
+    force: bool = False,
+    min_seconds: int = 1,
+    verify: bool = False,
+    deduplicate: bool = False,
+    bounded: Optional[bool] = None,
+    chunk_interval: Union[timedelta, int, None] = None,
+    mrsm_instance: Optional[str] = None,
+    timeout_seconds: Optional[int] = None,
+    nopretty: bool = False,
+    _progress: Optional['rich.progress.Progress'] = None,
+    **kw: Any
+) -> Tuple[List[mrsm.Pipe], List[mrsm.Pipe]]:
     """
     Do a lap of syncing pipes.
     """
+    import queue
+    import multiprocessing
+    import time
+    import copy
+    import json
+    import sys
+
     from meerschaum import get_pipes
     from meerschaum.utils.debug import dprint, _checkpoint
-    from meerschaum.utils.packages import attempt_import, import_rich
-    from meerschaum.utils.formatting import print_tuple, ANSI, UNICODE, get_console
+    from meerschaum.utils.packages import attempt_import
+    from meerschaum.utils.formatting import print_tuple
     from meerschaum.utils.warnings import warn
-    from meerschaum.utils.threading import Lock, RLock, Thread, Event
-    from meerschaum.utils.misc import print_options, get_cols_lines
-    from meerschaum.utils.pool import get_pool_executor, get_pool
+    from meerschaum.utils.threading import Lock, Thread, Event
     from meerschaum.connectors.parse import parse_instance_keys
-    from meerschaum import Plugin
-    import queue
-    import multiprocessing
-    import contextlib
-    import time, os, copy
     from meerschaum.utils.packages import venv_exec
     from meerschaum.utils.process import poll_process
-    import json
-    import sys
     dill = attempt_import('dill')
 
     rich_table, rich_text, rich_box = attempt_import(
         'rich.table', 'rich.text', 'rich.box',
-        lazy = False,
+        lazy=False,
     )
     all_kw = copy.deepcopy(kw)
     all_kw.update({
@@ -126,7 +126,7 @@ def _pipes_lap(
             + "may lead to concurrency issues.\n    You can change the pool size with "
             + "`edit config system` under the keys connectors:sql:create_engine:pool_size,\n    "
             + "and a size of 0 will not limit the number of connections.",
-            stack = False,
+            stack=False,
         )
 
 
@@ -155,8 +155,8 @@ def _pipes_lap(
                 ) + msg + '\n'
                 print_tuple(
                     (success, msg),
-                    calm = True,
-                    _progress = _progress,
+                    calm=True,
+                    _progress=_progress,
                 )
             _checkpoint(_progress=_progress, _task=_task)
             if _progress is not None:
@@ -274,25 +274,26 @@ def _sync_pipes(
         - `--debug`
             - Print verbose messages.
     """
-    from meerschaum.utils.debug import dprint
+    import time
+    import os
+    import contextlib
+
     from meerschaum.utils.warnings import warn, info
-    from meerschaum.utils.formatting import UNICODE
-    from meerschaum.utils.formatting._shell import progress, live
-    from meerschaum.utils.formatting._shell import clear_screen, flush_with_newlines
+    from meerschaum.utils.formatting._shell import progress
+    from meerschaum.utils.formatting._shell import clear_screen
     from meerschaum.utils.formatting import print_pipes_results
-    import contextlib
-    import time
-    import sys
-    import json
-    import asyncio
+    from meerschaum.config.static import STATIC_CONFIG
+
+    noninteractive_val = os.environ.get(STATIC_CONFIG['environment']['noninteractive'], None)
+    is_noninterative = noninteractive_val in ('1', 'true', 'True', 'yes')
+
     run = True
     msg = ""
     interrupt_warning_msg = "Syncing was interrupted due to a keyboard interrupt."
     cooldown = 2 * (min_seconds + 1)
-    underline = '\u2015' if UNICODE else '-'
     success_pipes, failure_pipes = [], []
     while run:
-        _progress = progress() if shell else None
+        _progress = progress() if shell and not is_noninterative else None
         cm = _progress if _progress is not None else contextlib.nullcontext()
 
         lap_begin = time.perf_counter()
@@ -301,15 +302,15 @@ def _sync_pipes(
             results_dict = {}
             with cm:
                 results_dict = _pipes_lap(
-                    min_seconds = min_seconds,
-                    _progress = _progress,
-                    verify = verify,
-                    deduplicate = deduplicate,
-                    bounded = bounded,
-                    chunk_interval = chunk_interval,
-                    unblock = unblock,
-                    debug = debug,
-                    nopretty = nopretty,
+                    min_seconds=min_seconds,
+                    _progress=_progress,
+                    verify=verify,
+                    deduplicate=deduplicate,
+                    bounded=bounded,
+                    chunk_interval=chunk_interval,
+                    unblock=unblock,
+                    debug=debug,
+                    nopretty=nopretty,
                     **kw
                 )
                 success_pipes = [

meerschaum/api/__init__.py

@@ -6,7 +6,7 @@
 Meerschaum API backend. Start an API instance with `start api`.
 """
 from __future__ import annotations
-import pathlib, os
+import os
 from meerschaum.utils.typing import Dict, Any, Optional
 
 from meerschaum import __version__ as version
@@ -21,7 +21,7 @@ from meerschaum.utils.packages import attempt_import
 from meerschaum.utils import get_pipes as _get_pipes
 from meerschaum.config._paths import API_UVICORN_CONFIG_PATH, API_UVICORN_RESOURCES_PATH
 from meerschaum.plugins import _api_plugins
-from meerschaum.utils.warnings import warn
+from meerschaum.utils.warnings import warn, dprint
 from meerschaum.utils.threading import RLock
 
 _locks = {'pipes': RLock(), 'connector': RLock(), 'uvicorn_config': RLock()}
@@ -44,8 +44,8 @@ uv = attempt_import('uv', lazy=False, check_update=CHECK_UPDATE)
     'starlette.responses',
     'multipart',
     'packaging.version',
-    lazy = False,
-    check_update = CHECK_UPDATE,
+    lazy=False,
+    check_update=CHECK_UPDATE,
 )
 (
     typing_extensions,
@@ -53,9 +53,9 @@ uv = attempt_import('uv', lazy=False, check_update=CHECK_UPDATE)
 ) = attempt_import(
     'typing_extensions',
     'uvicorn.workers',
-    lazy = False,
-    check_update = CHECK_UPDATE,
-    venv = None,
+    lazy=False,
+    check_update=CHECK_UPDATE,
+    venv=None,
 )
 from meerschaum.api._chain import check_allow_chaining, DISALLOW_CHAINING_MESSAGE
 uvicorn_config_path = API_UVICORN_RESOURCES_PATH / SERVER_ID / 'config.json'
@@ -75,7 +75,7 @@ def get_uvicorn_config() -> Dict[str, Any]:
     with _locks['uvicorn_config']:
         if uvicorn_config is None:
             try:
-                with open(uvicorn_config_path, 'r') as f:
+                with open(uvicorn_config_path, 'r', encoding='utf-8') as f:
                     uvicorn_config = json.load(f)
                 _uvicorn_config = uvicorn_config
             except Exception as e:
@@ -93,12 +93,12 @@ debug = get_uvicorn_config().get('debug', False)
 no_dash = get_uvicorn_config().get('no_dash', False)
 no_auth = get_uvicorn_config().get('no_auth', False)
 private = get_uvicorn_config().get('private', False)
+production = get_uvicorn_config().get('production', False)
 _include_dash = (not no_dash)
 
 connector = None
 def get_api_connector(instance_keys: Optional[str] = None):
     """Create the instance connector."""
-    from meerschaum.utils.debug import dprint
     global connector
     with _locks['connector']:
         if connector is None:
@@ -111,6 +111,40 @@ def get_api_connector(instance_keys: Optional[str] = None):
         dprint(f"API instance connector: {connector}")
     return connector
 
+cache_connector = None
+def get_cache_connector(connector_keys: Optional[str] = None):
+    """Return the `valkey` connector if running in production."""
+    global cache_connector
+    if cache_connector is not None:
+        return cache_connector
+
+    if not production:
+        return None
+
+    enable_valkey_cache = get_config('system', 'experimental', 'valkey_session_cache')
+    if not enable_valkey_cache:
+        return None
+
+    connector_keys = connector_keys or get_config(
+        'system', 'api', 'cache', 'connector',
+        warn=False,
+    )
+    if connector_keys is None:
+        return None
+
+    if not connector_keys.startswith('valkey'):
+        warn(f"Invalid cache connector '{connector_keys}'.")
+        return None
+
+    if cache_connector is None:
+        from meerschaum.connectors.parse import parse_instance_keys
+        cache_connector = parse_instance_keys(connector_keys)
+
+    if debug:
+        dprint(f"Cache connector: {cache_connector}")
+
+    return cache_connector
+
 
 _pipes = None
 def pipes(refresh=False):
@@ -138,17 +172,17 @@ def get_pipe(connector_keys, metric_key, location_key, refresh=False):
 
 app = fastapi.FastAPI(
     title = 'Meerschaum API',
-    description = __doc__,
-    version = __version__,
-    contact = {
+    description=__doc__,
+    version=__version__,
+    contact={
         'name': 'Bennett Meares',
         'url': 'https://meerschaum.io',
     },
-    license_info = {
+    license_info={
         'name': 'Apache 2.0',
         'url': 'https://www.apache.org/licenses/LICENSE-2.0.html',
     },
-    open_api_tags = [
+    open_api_tags=[
         {
             'name': 'Pipes',
             'description': 'Access pipes by indexing their keys.',

meerschaum/api/_events.py

@@ -6,29 +6,30 @@
 Declare FastAPI events in this module (startup, shutdown, etc.).
 """
 
-import sys, os, time
+import sys
+import os
+import time
 from meerschaum.api import (
     app,
     get_api_connector,
+    get_cache_connector,
     get_uvicorn_config,
     debug,
     no_dash,
-    uvicorn_config_path,
 )
 from meerschaum.utils.debug import dprint
 from meerschaum.connectors.poll import retry_connect
 from meerschaum.utils.warnings import warn
-from meerschaum._internal.term.tools import is_webterm_running
 from meerschaum.jobs import (
     get_jobs,
     start_check_jobs_thread,
     stop_check_jobs_thread,
-    get_executor_keys_from_context,
 )
 from meerschaum.config.static import STATIC_CONFIG
 
 TEMP_PREFIX: str = STATIC_CONFIG['api']['jobs']['temp_prefix']
 
+
 @app.on_event("startup")
 async def startup():
     """
@@ -41,10 +42,17 @@ async def startup():
 
         connected = retry_connect(
             get_api_connector(),
-            workers = get_uvicorn_config().get('workers', None),
-            debug = debug
+            workers=get_uvicorn_config().get('workers', None),
+            debug=debug
         )
-    except Exception as e:
+        cache_connector = get_cache_connector()
+        if cache_connector is not None:
+            connected = retry_connect(
+                cache_connector,
+                workers=get_uvicorn_config().get('workers', None),
+                debug=debug,
+            )
+    except Exception:
         import traceback
         traceback.print_exc()
         connected = False
@@ -78,8 +86,5 @@ async def shutdown():
 
     ### Terminate any running jobs left over.
     if 'meerschaum.api.dash' in sys.modules:
-        from meerschaum.api.dash.actions import running_jobs, stop_action
         from meerschaum.api.dash.webterm import stop_webterm
         stop_webterm()
-        for session_id in running_jobs:
-            stop_action({'session-store.data': {'session-id': session_id}})

meerschaum/api/_oauth2.py

@@ -39,10 +39,10 @@ def generate_secret_key() -> str:
     from meerschaum.config._paths import API_SECRET_KEY_PATH
     if not API_SECRET_KEY_PATH.exists():
         secret_key = os.urandom(24).hex()
-        with open(API_SECRET_KEY_PATH, 'w+') as f:
+        with open(API_SECRET_KEY_PATH, 'w+', encoding='utf-8') as f:
             f.write(secret_key)
     else:
-        with open(API_SECRET_KEY_PATH, 'r') as f:
+        with open(API_SECRET_KEY_PATH, 'r', encoding='utf-8') as f:
             secret_key = f.read()
 
     return secret_key.encode('utf-8')

meerschaum/api/_websockets.py

@@ -6,13 +6,13 @@
 Implement WebSockets for the Meerschaum API via FastAPI.
 """
 
-import time, uuid
 from datetime import datetime, timezone
+
 from meerschaum.api import (
-    app, get_api_connector, get_uvicorn_config, debug, fastapi, endpoints
+    app,
+    fastapi,
+    endpoints,
 )
-from meerschaum.api.dash.users import is_session_authenticated
-from meerschaum.utils.typing import Optional
 
 _websocket_endpoint = endpoints['websocket']
 
@@ -28,6 +28,7 @@ async def websocket_endpoint(
     """
     Communicate with the Web Interface over a websocket.
     """
+    from meerschaum.api.dash.sessions import is_session_authenticated
     await websocket.accept()
     try:
         initial_data = await websocket.receive_json()

meerschaum/api/dash/__init__.py

@@ -16,7 +16,6 @@ flask_compress = attempt_import('flask_compress', lazy=False)
 _monkey_patch_get_distribution('flask-compress', flask_compress.__version__)
 dash = attempt_import('dash', lazy=False)
 
-
 from meerschaum.utils.typing import List, Optional
 from meerschaum.config.static import _static_config
 from meerschaum.api import (
@@ -38,19 +37,10 @@ with warnings.catch_warnings():
 html, dcc = import_html(), import_dcc()
 from meerschaum.api.dash.components import location
 
-active_sessions = {}
-authenticated_sessions = {}
-unauthenticated_sessions = {}
-running_jobs = {}
-running_monitors = {}
-stopped_jobs = {}
-stopped_monitors = {}
-
 stylesheets = [
     '/static/css/bootstrap.min.css',
     '/static/css/dbc_dark.css',
     '/static/css/dash.css',
-    '/static/css/bootstrap-icons/font/bootstrap-icons.min.css',
 ]
 scripts = ['/static/js/node_modules/xterm/lib/xterm.js']
 dash_app = enrich.DashProxy(
@@ -68,7 +58,7 @@ dash_app = enrich.DashProxy(
 
 dash_app.layout = html.Div([
     location,
-    dcc.Store(id='session-store', storage_type='session', data={}),
+    dcc.Store(id='session-store', storage_type='local', data={}),
     html.Div([], id='page-layout-div'),
 ])