meerschaum 3.0.0rc3__py3-none-any.whl → 3.0.0rc7__py3-none-any.whl

meerschaum/api/__init__.py

@@ -21,7 +21,6 @@ from meerschaum.config._paths import API_UVICORN_CONFIG_PATH, API_UVICORN_RESOUR
 from meerschaum.plugins import _api_plugins
 from meerschaum.utils.warnings import warn, dprint
 from meerschaum.utils.threading import RLock
-from meerschaum.utils.misc import is_pipe_registered
 from meerschaum.connectors.parse import parse_instance_keys
 
 from meerschaum import __version__ as version
@@ -67,16 +66,13 @@ from meerschaum.api._exceptions import APIPermissionError
 uvicorn_config_path = API_UVICORN_RESOURCES_PATH / SERVER_ID / 'config.json'
 
 uvicorn_config = None
-sys_config = get_config('system', 'api')
-permissions_config = get_config('system', 'api', 'permissions')
+sys_config = get_config('api')
+permissions_config = get_config('api', 'permissions')
 
 def get_uvicorn_config() -> Dict[str, Any]:
     """Read the Uvicorn configuration JSON and return a dictionary."""
     global uvicorn_config
     import json
-    runtime = os.environ.get(STATIC_CONFIG['environment']['runtime'], None)
-    if runtime == 'api':
-        return get_config('system', 'api', 'uvicorn')
     _uvicorn_config = uvicorn_config
     with _locks['uvicorn_config']:
         if uvicorn_config is None:
@@ -85,6 +81,8 @@ def get_uvicorn_config() -> Dict[str, Any]:
                     uvicorn_config = json.load(f)
                 _uvicorn_config = uvicorn_config
             except Exception:
+                import traceback
+                traceback.print_exc()
                 _uvicorn_config = sys_config.get('uvicorn', None)
 
             if _uvicorn_config is None:
@@ -102,6 +100,10 @@ production = get_uvicorn_config().get('production', False)
 _include_dash = (not no_dash)
 _include_webterm = (not no_webterm) and _include_dash
 docs_enabled = not production or sys_config.get('endpoints', {}).get('docs_in_production', True)
+webterm_port = (
+    get_uvicorn_config().get('webterm_port', None)
+    or mrsm.get_config('api', 'webterm', 'port')
+)
 
 default_instance_keys = None
 _instance_connectors = defaultdict(lambda: None)
@@ -129,7 +131,7 @@ def get_api_connector(instance_keys: Optional[str] = None):
     )
     found_match: bool = False
     for allowed_keys_pattern in allowed_instance_keys:
-        if fnmatch(instance_keys, allowed_keys_pattern):
+        if fnmatch(str(instance_keys), allowed_keys_pattern):
             found_match = True
             break
     if not found_match:
@@ -141,7 +143,9 @@ def get_api_connector(instance_keys: Optional[str] = None):
         if _instance_connectors[instance_keys] is None:
             try:
                 is_valid_connector = True
-                _instance_connectors[instance_keys] = parse_instance_keys(instance_keys, debug=debug)
+                instance_connector = parse_instance_keys(instance_keys, debug=debug)
+                instance_connector._cache_connector = get_cache_connector()
+                _instance_connectors[instance_keys] = instance_connector
             except Exception:
                 is_valid_connector = False
 
@@ -168,7 +172,7 @@ def get_cache_connector(connector_keys: Optional[str] = None):
         return None
 
     connector_keys = connector_keys or get_config(
-        'system', 'api', 'cache', 'connector',
+        'api', 'cache', 'connector',
         warn=False,
     )
     if connector_keys is None:
@@ -196,7 +200,11 @@ def pipes(instance_keys: Optional[str] = None, refresh: bool = False) -> PipesDi
     with _locks['pipes-' + instance_keys]:
         pipes = _instance_pipes[instance_keys]
         if pipes is None or refresh:
-            pipes = _get_pipes(mrsm_instance=instance_keys)
+            pipes = _get_pipes(
+                mrsm_instance=instance_keys,
+                cache=True,
+                cache_connector_keys=get_cache_connector(),
+            )
             _instance_pipes[instance_keys] = pipes
     return pipes
 
@@ -218,9 +226,23 @@ def get_pipe(
             detail="Unable to serve any pipes with connector keys `mrsm` over the API.",
         )
 
-    pipe = mrsm.Pipe(connector_keys, metric_key, location_key, mrsm_instance=instance_keys)
-    if is_pipe_registered(pipe, pipes(instance_keys, refresh=False)):
-        return pipes(instance_keys, refresh=refresh)[connector_keys][metric_key][location_key]
+    pipes_dict = pipes(instance_keys)
+    if (
+        not refresh
+        and connector_keys in pipes_dict
+        and metric_key in pipes_dict[connector_keys]
+        and location_key in pipes_dict[connector_keys][metric_key]
+    ):
+        return pipes_dict[connector_keys][metric_key][location_key]
+
+    pipe = mrsm.Pipe(
+        connector_keys,
+        metric_key,
+        location_key,
+        mrsm_instance=instance_keys,
+        cache=True,
+        cache_connector_keys=get_cache_connector(),
+    )
     return pipe
 
 

meerschaum/api/_events.py

@@ -8,13 +8,13 @@ Declare FastAPI events in this module (startup, shutdown, etc.).
 
 import sys
 import os
-import time
 from meerschaum.api import (
     app,
     get_api_connector,
     get_cache_connector,
     get_uvicorn_config,
     debug,
+    webterm_port,
     no_dash,
     _include_dash,
     _include_webterm,
@@ -75,7 +75,7 @@ async def startup():
     try:
         if _include_webterm:
             from meerschaum.api.dash.webterm import start_webterm
-            start_webterm()
+            start_webterm(webterm_port=webterm_port)
 
         connected = retry_connect(
             get_api_connector(),
@@ -98,6 +98,11 @@ async def startup():
         await shutdown()
         os._exit(1)
 
+    conn = get_api_connector()
+    if conn.type == 'sql':
+        from meerschaum.connectors.sql.tables import get_tables
+        _ = get_tables(conn, refresh=True, create=True, debug=debug)
+
     start_check_jobs_thread()
 
 

meerschaum/api/_oauth2.py

@@ -21,6 +21,7 @@ from meerschaum.core import User, Token
 fastapi, starlette = attempt_import('fastapi', 'starlette', lazy=False, check_update=CHECK_UPDATE)
 fastapi_responses = attempt_import('fastapi.responses', lazy=False, check_update=CHECK_UPDATE)
 fastapi_login = attempt_import('fastapi_login', check_update=CHECK_UPDATE)
+jose_jwt, jose_exceptions = attempt_import('jose.jwt', 'jose.exceptions', lazy=False, check_update=CHECK_UPDATE)
 from fastapi import Depends, HTTPException, Request
 from starlette import status
 
@@ -91,6 +92,7 @@ async def load_user_or_token(
             status_code=status.HTTP_401_UNAUTHORIZED,
             detail="Not authenticated.",
         )
+
     authorization = authorization.replace('Basic ', '').replace('Bearer ', '')
     if not authorization.startswith('mrsm-key:'):
         if not users:
@@ -98,12 +100,15 @@ async def load_user_or_token(
                 status=status.HTTP_401_UNAUTHORIZED,
                 detail="Users not authenticated for this endpoint.",
             )
+
         return await manager(request)
+
     if not tokens:
         raise HTTPException(
             status_code=status.HTTP_401_UNAUTHORIZED,
             detail="Tokens not authenticated for this endpoint.",
         )
+
     return get_token_from_authorization(authorization)
 
 
@@ -112,6 +117,7 @@ def ScopedAuth(scopes: List[str]):
     Dependency factory for authenticating with either a user session or a scoped token.
     """
     async def _authenticate(
+        request: Request,
         user_or_token: Union[User, Token, None] = Depends(
             load_user_or_token,
         ),
@@ -126,12 +132,45 @@ def ScopedAuth(scopes: List[str]):
                 headers={"WWW-Authenticate": "Basic"},
             )
 
-        fresh_scopes = user_or_token.get_scopes(refresh=True, debug=debug)
-        if '*' in fresh_scopes:
+        authorization = request.headers.get('authorization', request.headers.get('Authorization', None))
+        is_long_lived = authorization and 'mrsm-key:' in authorization
+
+        current_scopes = []
+        ### For long-lived API tokens, always hit the database.
+        if is_long_lived:
+            current_scopes = user_or_token.get_scopes(refresh=True, debug=debug)
+        
+        ### For JWTs, trust the scopes in the token.
+        else:
+            if not authorization:
+                # This should be caught by `load_user_or_token` but we can be safe.
+                raise HTTPException(
+                    status_code=status.HTTP_401_UNAUTHORIZED,
+                    detail="Not authenticated.",
+                )
+            
+            scheme, _, token_str = authorization.partition(' ')
+            if not token_str or scheme.lower() != 'bearer':
+                raise HTTPException(
+                    status_code=status.HTTP_401_UNAUTHORIZED,
+                    detail="Unsupported authentication scheme.",
+                )
+
+            try:
+                payload = jose_jwt.decode(token_str, SECRET, algorithms=['HS256'])
+                current_scopes = payload.get('scopes', [])
+            except jose_exceptions.JWTError:
+                raise HTTPException(
+                    status_code=status.HTTP_401_UNAUTHORIZED,
+                    detail="Invalid access token.",
+                    headers={"WWW-Authenticate": "Bearer"},
+                )
+
+        if '*' in current_scopes:
             return user_or_token
 
         for scope in scopes:
-            if scope not in fresh_scopes:
+            if scope not in current_scopes:
                 raise HTTPException(
                     status_code=status.HTTP_403_FORBIDDEN,
                     detail=f"Missing required scope: '{scope}'",
@@ -159,4 +198,8 @@ def generate_secret_key() -> bytes:
 
 
 SECRET = generate_secret_key()
-manager = LoginManager(SECRET, token_url=endpoints['login'])
+manager = LoginManager(
+    SECRET,
+    token_url=endpoints['login'],
+    scopes=STATIC_CONFIG['tokens']['scopes'],
+)

meerschaum/api/dash/callbacks/dashboard.py

@@ -498,13 +498,9 @@ def update_flags(input_flags_dropdown_values, n_clicks, input_flags_texts):
 
 @dash_app.callback(
     Output('connector-keys-dropdown', 'options'),
-    Output('connector-keys-dropdown', 'value'),
     Output('metric-keys-dropdown', 'options'),
-    Output('metric-keys-dropdown', 'value'),
     Output('location-keys-dropdown', 'options'),
-    Output('location-keys-dropdown', 'value'),
     Output('tags-dropdown', 'options'),
-    Output('tags-dropdown', 'value'),
     Output('instance-select', 'value'),
     Output('instance-alert-div', 'children'),
     Input('connector-keys-dropdown', 'value'),
@@ -546,15 +542,6 @@ def update_keys_options(
     except Exception as e:
         instance_alerts += [alert_from_success_tuple((False, str(e)))]
 
-    ### Update the keys filters.
-    if connector_keys is None:
-        connector_keys = []
-    if metric_keys is None:
-        metric_keys = []
-    if location_keys is None:
-        location_keys = []
-    if tags is None:
-        tags = []
     num_filter = 0
     if connector_keys:
         num_filter += 1
@@ -565,10 +552,6 @@ def update_keys_options(
     if tags:
         num_filter += 1
 
-    _ck_filter = connector_keys
-    _mk_filter = metric_keys
-    _lk_filter = location_keys
-    _tags_filter = tags
     _ck_alone = (connector_keys and num_filter == 1) or instance_click
     _mk_alone = (metric_keys and num_filter == 1) or instance_click
     _lk_alone = (location_keys and num_filter == 1) or instance_click
@@ -581,101 +564,95 @@ def update_keys_options(
         _keys = fetch_pipes_keys(
             'registered',
             get_web_connector(ctx.states),
-            connector_keys=_ck_filter,
-            metric_keys=_mk_filter,
-            location_keys=_lk_filter,
-            tags=_tags_filter,
+            connector_keys=connector_keys,
+            metric_keys=metric_keys,
+            location_keys=location_keys,
+            tags=tags,
         )
         _tags_pipes = mrsm.get_pipes(
-            connector_keys=_ck_filter,
-            metric_keys=_mk_filter,
-            location_keys=_lk_filter,
-            tags=_tags_filter,
+            connector_keys=connector_keys,
+            metric_keys=metric_keys,
+            location_keys=location_keys,
+            tags=tags,
             instance=get_web_connector(ctx.states),
             as_tags_dict=True,
         )
         _all_tags = list(
-            mrsm.get_pipes(
-                instance=get_web_connector(ctx.states),
-                as_tags_dict=True,
-            )
+            set(
+                mrsm.get_pipes(
+                    instance=get_web_connector(ctx.states),
+                    as_tags_dict=True,
+                )
+            ).union(tags or [])
         ) if _tags_alone else []
     except Exception as e:
         instance_alerts += [alert_from_success_tuple((False, str(e)))]
-        _all_keys, _keys = [], []
-    _connectors_options = []
-    _metrics_options = []
-    _locations_options = []
-    _tags_options = []
-
-    _seen_keys = {'ck' : set(), 'mk' : set(), 'lk' : set(), 'tags': set()}
-
-    def add_options(options, keys, key_type):
-        for ck, mk, lk in keys:
-            k = locals()[key_type]
-            if k not in _seen_keys[key_type]:
-                _k = 'None' if k in (None, '[None]', 'None', 'null') else k
-                options.append({'label': _k, 'value': _k})
-                _seen_keys[key_type].add(k)
-
-    add_options(_connectors_options, _all_keys if _ck_alone else _keys, 'ck')
-    add_options(_metrics_options, _all_keys if _mk_alone else _keys, 'mk')
-    add_options(_locations_options, _all_keys if _lk_alone else _keys, 'lk')
-
-    _tags_options = [
-        {'label': tag, 'value': tag}
-        for tag in (_all_tags if _tags_alone else _tags_pipes)
-    ]
+        _all_keys, _all_tags, _keys = [], [], []
+
+    connectors_options = sorted(
+        list(
+            set(
+                keys_tuple[0] for keys_tuple in (_all_keys if _ck_alone else _keys)
+            ).union(set(connector_keys or []))
+        ),
+        key=(lambda x: str(x).lower()),
+    )
+    metrics_options = sorted(
+        list(
+            set(
+                keys_tuple[1] for keys_tuple in (_all_keys if _mk_alone else _keys)
+            ).union(set(metric_keys or []))
+        ),
+        key=(lambda x: str(x).lower()),
+    )
+    locations_options = sorted(
+        list(
+            set(
+                (
+                    str(keys_tuple[2])
+                    for keys_tuple in (_all_keys if _lk_alone else _keys)
+                )
+            ).union(set((str(_lk) for _lk in (location_keys or []))))
+        ),
+        key=(lambda x: str(x).lower()),
+    )
+
+    tags_options = sorted(
+        list(
+            set(
+                (_all_tags if _tags_alone else _tags_pipes)
+            ).union(set(tags or []))
+        ),
+        key=(lambda x: str(x).lower()),
+    )
 
-    _connectors_options.sort(key=lambda x: str(x).lower())
-    _metrics_options.sort(key=lambda x: str(x).lower())
-    _locations_options.sort(key=lambda x: str(x).lower())
-    _tags_options.sort(key=lambda x: str(x).lower())
-    connector_keys = [
-        ck
-        for ck in connector_keys
-        if ck in [
-            _ck['value']
-            for _ck in _connectors_options
-        ]
-    ]
-    metric_keys = [
-        mk
-        for mk in metric_keys
-        if mk in [
-            _mk['value']
-            for _mk in _metrics_options
-        ]
-    ]
-    location_keys = [
-        lk
-        for lk in location_keys
-        if lk in [
-            _lk['value']
-            for _lk in _locations_options
-        ]
-    ]
-    tags = [
-        tag
-        for tag in tags
-        if tag in [
-            _tag['value']
-            for _tag in _tags_options
-        ]
-    ]
     return (
-        _connectors_options,
-        connector_keys,
-        _metrics_options,
-        metric_keys,
-        _locations_options,
-        location_keys,
-        _tags_options,
-        tags,
+        connectors_options,
+        metrics_options,
+        locations_options,
+        tags_options,
         (instance_keys if update_instance_keys else dash.no_update),
         instance_alerts,
     )
 
+
+@dash_app.callback(
+    Output('connector-keys-dropdown', 'value'),
+    Output('metric-keys-dropdown', 'value'),
+    Output('location-keys-dropdown', 'value'),
+    Output('tags-dropdown', 'value'),
+    Input('clear-all-keys-button', 'n_clicks'),
+    prevent_initial_call=True,
+)
+def clear_all_keys_button_click(n_clicks):
+    """
+    Clear the keys dropdowns when the `Clear all` button is clicked.
+    """
+    if not n_clicks:
+        raise PreventUpdate
+
+    return [], [], [], []
+
 dash_app.clientside_callback(
     """
     function(
@@ -1216,3 +1193,32 @@ def toggle_pages_offcanvas(n_clicks: Optional[int], is_open: bool):
     if n_clicks:
         return not is_open, pages_children
     return is_open, pages_children
+
+
+@dash_app.callback(
+    Output({'type': 'calculate-rowcount-div', 'index': MATCH}, 'children'),
+    Input({'type': 'calculate-rowcount-button', 'index': MATCH}, 'n_clicks'),
+    prevent_initial_call=True,
+)
+def calculate_rowcount_button_click(n_clicks: int):
+    """
+    Calculate the rowcount for the pipe.
+    """
+    if not n_clicks:
+        raise PreventUpdate
+
+    triggered = dash.callback_context.triggered
+    if triggered[0]['value'] is None:
+        raise PreventUpdate
+
+    pipe = pipe_from_ctx(triggered, 'n_clicks')
+    if pipe is None:
+        raise PreventUpdate
+
+    try:
+        rowcount = pipe.get_rowcount(debug=debug)
+        return f"{rowcount:,}"
+    except Exception as e:
+        return (
+            alert_from_success_tuple((False, f"Failed to calculate row count: {e}"))
+        )

meerschaum/api/dash/callbacks/jobs.py

@@ -13,6 +13,7 @@ import time
 import traceback
 from datetime import datetime, timezone
 
+import meerschaum as mrsm
 from meerschaum.jobs import get_jobs
 from meerschaum.utils.typing import Optional, Dict, Any
 from meerschaum.api import CHECK_UPDATE
@@ -136,12 +137,12 @@ def manage_job_button_click(
     old_status = job.status
     try:
         success, msg = manage_functions[manage_job_action]()
-    except Exception as e:
+    except Exception:
         success, msg = False, traceback.format_exc()
 
     ### Wait for a status change before building the elements.
     timeout_seconds = 1.0
-    check_interval_seconds = 0.01
+    check_interval_seconds = mrsm.get_config('system', 'cli', 'refresh_seconds')
     begin = time.perf_counter()
     while (time.perf_counter() - begin) < timeout_seconds:
         if job.status != old_status:

meerschaum/api/dash/callbacks/login.py

@@ -16,11 +16,14 @@ from meerschaum.utils.typing import Optional
 dash = attempt_import('dash', lazy=False, check_update=CHECK_UPDATE)
 from dash.exceptions import PreventUpdate
 from dash.dependencies import Input, Output, State
+
 from meerschaum.api.dash import dash_app, debug, pipes, _get_pipes
 from meerschaum.api.dash.sessions import set_session
 from meerschaum.api.dash.connectors import get_web_connector
 from meerschaum.api.routes._login import login
 from meerschaum.api.dash.components import alert_from_success_tuple
+from meerschaum.api._oauth2 import CustomOAuth2PasswordRequestForm
+from meerschaum._internal.static import STATIC_CONFIG
 from fastapi_login.exceptions import InvalidCredentialsException
 from fastapi.exceptions import HTTPException
 dbc = attempt_import('dash_bootstrap_components', lazy=False, check_update=CHECK_UPDATE)
@@ -73,7 +76,13 @@ def login_button_click(
         raise PreventUpdate
 
     try:
-        _ = login({'username': username, 'password': password})
+        form = CustomOAuth2PasswordRequestForm(
+            grant_type='password',
+            username=username,
+            password=password,
+            scope=' '.join(STATIC_CONFIG['tokens']['scopes'])
+        )
+        _ = login(form)
         session_id = str(uuid.uuid4())
         session_data = {
             'session-id': session_id,