meerschaum 2.3.6__py3-none-any.whl → 2.4.0.dev1__py3-none-any.whl

meerschaum/actions/bootstrap.py

@@ -234,7 +234,13 @@ def _bootstrap_connectors(
     Prompt the user for the details necessary to create a Connector.
     """
     from meerschaum.connectors.parse import is_valid_connector_keys
-    from meerschaum.connectors import connectors, get_connector, types, custom_types
+    from meerschaum.connectors import (
+        connectors,
+        get_connector,
+        types,
+        custom_types,
+        _load_builtin_custom_connectors,
+    )
     from meerschaum.utils.prompt import prompt, yes_no, choose
     from meerschaum.config import get_config
     from meerschaum.config._edit import write_config
@@ -246,6 +252,7 @@ def _bootstrap_connectors(
 
     abort_tuple = False, "No connectors bootstrapped."
     _clear = get_config('shell', 'clear_screen', patch=True)
+    _load_builtin_custom_connectors()
 
     if action is None:
         action = []
@@ -262,8 +269,8 @@ def _bootstrap_connectors(
                 + '    See https://meerschaum.io/reference/connectors '
                 + 'for documentation on connectors.\n'
             ),
-            sorted(list(connectors)),
-            default = 'sql'
+            sorted([k for k in connectors if k != 'plugin']),
+            default='sql',
         )
     except KeyboardInterrupt:
         return abort_tuple
@@ -288,21 +295,30 @@ def _bootstrap_connectors(
             warn(f"Connector '{_type}:{_label}' already exists.", stack=False)
             overwrite = yes_no(
                 f"Do you want to overwrite connector '{_type}:{_label}'?",
-                default = 'n',
-                yes = yes,
-                noask = noask,
+                default='n',
+                yes=yes,
+                noask=noask,
             )
             if not overwrite and not force:
-                return False, f"No changes made to connector configuration."
+                return False, "No changes made to connector configuration."
                 break
         elif _label == "":
-            warn(f"Please enter a label.", stack=False)
+            warn("Please enter a label.", stack=False)
         else:
             break
 
     cls = types.get(_type)
     cls_required_attrs = getattr(cls, 'REQUIRED_ATTRIBUTES', [])
-    type_attributes = connector_attributes.get(_type, {'required': cls_required_attrs})
+    cls_optional_attrs = getattr(cls, 'OPTIONAL_ATTRIBUTES', [])
+    cls_default_attrs = getattr(cls, 'DEFAULT_ATTRIBUTES', {})
+    type_attributes = connector_attributes.get(
+        _type,
+        {
+            'required': cls_required_attrs,
+            'optional': cls_optional_attrs,
+            'default': cls_default_attrs,
+        }
+    )
 
     new_attributes = {}
     if 'flavors' in type_attributes:
@@ -323,6 +339,7 @@ def _bootstrap_connectors(
         default = type_attributes['flavors'][flavor].get('defaults', {})
     else:
         required = sorted(list(type_attributes.get('required', {})))
+        optional = sorted(list(type_attributes.get('optional', {})))
         default = type_attributes.get('default', {})
     info(
         f"Please answer the following questions to configure the new connector '{_type}:{_label}'."
@@ -330,13 +347,22 @@ def _bootstrap_connectors(
     )
     for r in required:
         try:
-            val = prompt(f"Value for {r}:")
+            default_val = str(default.get(r)) if r in default else None
+            val = prompt(f"Value for {r}:", default=default_val)
         except KeyboardInterrupt:
             continue
         if is_int(val):
             val = int(val)
         new_attributes[r] = val
 
+    for o in optional:
+        try:
+            val = prompt(f"Value for {o} (optional; empty to omit):")
+        except KeyboardInterrupt:
+            continue
+        if val:
+            new_attributes[o] = val
+
     for k, v in default.items():
         ### skip already configured attributes, (e.g. flavor or from required)
         if k in new_attributes:

meerschaum/actions/copy.py

@@ -115,11 +115,11 @@ def _copy_pipes(
         ):
             _new_pipe.sync(
                 p.get_data(
-                    debug = debug,
-                    as_iterator = True,
+                    debug=debug,
+                    as_iterator=True,
                     **kw
                 ),
-                debug = debug,
+                debug=debug,
                 **kw
             )
 

meerschaum/actions/start.py

@@ -455,12 +455,12 @@ def _start_webterm(
 
 
 def _start_connectors(
-        action: Optional[List[str]] = None,
-        connector_keys: Optional[List[str]] = None,
-        min_seconds: int = 3,
-        debug: bool = False,
-        **kw
-    ) -> SuccessTuple:
+    action: Optional[List[str]] = None,
+    connector_keys: Optional[List[str]] = None,
+    min_seconds: int = 3,
+    debug: bool = False,
+    **kw
+) -> SuccessTuple:
     """
     Start polling connectors to verify a connection can be made.
     """
@@ -480,7 +480,7 @@ def _start_connectors(
     for keys in unique_keys:
         try:
             conn = parse_instance_keys(keys)
-        except Exception as e:
+        except Exception:
             warn(f"Invalid connector keys: '{keys}'. Skipping...", stack=False)
             continue
         valid_conns.append(conn)
@@ -488,20 +488,19 @@ def _start_connectors(
     if not valid_conns:
         return False, "No valid connector keys were provided."
 
-
     connected = {}
     def connect(conn):
         success = retry_connect(
             conn,
-            retry_wait = min_seconds,
-            enforce_chaining = False,
-            enforce_login = False,
-            print_on_connect = True,
-            debug = debug,
+            retry_wait=min_seconds,
+            enforce_chaining=False,
+            enforce_login=False,
+            print_on_connect=True,
+            debug=debug,
         )
         connected[conn] = success
         return success
-    
+
     pool = get_pool()
     try:
         pool.map(connect, valid_conns)

meerschaum/api/dash/__init__.py

@@ -50,16 +50,17 @@ stylesheets = [
     '/static/css/bootstrap.min.css',
     '/static/css/dbc_dark.css',
     '/static/css/dash.css',
+    '/static/css/bootstrap-icons/font/bootstrap-icons.min.css',
 ]
 scripts = ['/static/js/node_modules/xterm/lib/xterm.js']
 dash_app = enrich.DashProxy(
     __name__,
-    title = 'Meerschaum Web',
-    requests_pathname_prefix = endpoints['dash'] + '/',
-    external_stylesheets = stylesheets,
-    update_title = None,
-    suppress_callback_exceptions = True,
-    transforms = [
+    title='Meerschaum Web',
+    requests_pathname_prefix=endpoints['dash'] + '/',
+    external_stylesheets=stylesheets,
+    update_title=None,
+    suppress_callback_exceptions=True,
+    transforms=[
         enrich.TriggerTransform(),
         enrich.MultiplexerTransform(),
     ],

meerschaum/api/dash/callbacks/__init__.py

@@ -11,6 +11,7 @@ import meerschaum.api.dash.callbacks.login
 import meerschaum.api.dash.callbacks.plugins
 import meerschaum.api.dash.callbacks.jobs
 import meerschaum.api.dash.callbacks.register
+import meerschaum.api.dash.callbacks.pipes
 from meerschaum.api.dash.callbacks.custom import init_dash_plugins, add_plugin_pages
 
 init_dash_plugins()

meerschaum/api/dash/callbacks/dashboard.py

@@ -101,6 +101,7 @@ _paths = {
     ''        : pages.dashboard.layout,
     'plugins' : pages.plugins.layout,
     'register': pages.register.layout,
+    'pipes'   : pages.pipes.layout,
 }
 _required_login = {''}
 
@@ -113,10 +114,10 @@ _required_login = {''}
     State('location', 'href'),
 )
 def update_page_layout_div(
-        pathname: str, 
-        session_store_data: Dict[str, Any],
-        location_href: str,
-    ) -> Tuple[List[Any], Dict[str, Any]]:
+    pathname: str, 
+    session_store_data: Dict[str, Any],
+    location_href: str,
+) -> Tuple[List[Any], Dict[str, Any]]:
     """
     Route the user to the correct page.
 
@@ -135,7 +136,7 @@ def update_page_layout_div(
     ctx = dash.callback_context
     dash_endpoint = endpoints['dash']
     try:
-        session_id = session_store_data.get('session-id', None) 
+        session_id = session_store_data.get('session-id', None)
     except AttributeError:
         session_id = None
 
@@ -760,6 +761,7 @@ def update_pipe_accordion(item, session_store_data):
         raise PreventUpdate
 
     session_id = session_store_data.get('session-id', None)
+    print(f"{session_id=}")
     authenticated = is_session_authenticated(str(session_id))
     return accordion_items_from_pipe(pipe, active_items=[item], authenticated=authenticated)
 

meerschaum/api/dash/callbacks/pipes.py

@@ -0,0 +1,42 @@
+#! /usr/bin/env python3
+# vim:fenc=utf-8
+
+"""
+Define callbacks for the `/dash/pipes/` page.
+"""
+
+from dash.exceptions import PreventUpdate
+from dash.dependencies import Input, Output, State
+from dash import no_update
+
+import meerschaum as mrsm
+from meerschaum.api.dash import dash_app, debug, active_sessions
+from meerschaum.api.dash.pipes import build_pipe_card
+from meerschaum.api import get_api_connector, CHECK_UPDATE
+from meerschaum.utils.packages import import_html, import_dcc
+html, dcc = import_html(check_update=CHECK_UPDATE), import_dcc(check_update=CHECK_UPDATE)
+
+
+@dash_app.callback(
+    Output('pipe-output-div', 'children'),
+    Input('pipes-location', 'pathname'),
+)
+def render_page_from_url(pathname):
+    if not str(pathname).startswith('/dash/pipes'):
+        return no_update
+
+    keys = pathname.replace('/dash/pipes', '').lstrip('/').rstrip('/').split('/')
+    if len(keys) not in (2, 3):
+        return no_update
+
+    ck = keys[0]
+    mk = keys[1]
+    lk = keys[2] if len(keys) == 3 else None
+
+    pipe = mrsm.Pipe(ck, mk, lk)
+    ### TODO Check if logged in
+    return [
+        html.Br(),
+        build_pipe_card(pipe, authenticated=False),
+        html.Br(),
+    ]

meerschaum/api/dash/pages/__init__.py

@@ -10,3 +10,4 @@ import meerschaum.api.dash.pages.login
 import meerschaum.api.dash.pages.dashboard
 import meerschaum.api.dash.pages.plugins
 import meerschaum.api.dash.pages.register
+import meerschaum.api.dash.pages.pipes

meerschaum/api/dash/pages/pipes.py

@@ -0,0 +1,16 @@
+#! /usr/bin/env python3
+# vim:fenc=utf-8
+
+"""
+Display pipes via a shareable URL.
+"""
+
+from meerschaum.api import get_api_connector, CHECK_UPDATE
+from meerschaum.utils.packages import import_html, import_dcc
+html, dcc = import_html(check_update=CHECK_UPDATE), import_dcc(check_update=CHECK_UPDATE)
+import dash_bootstrap_components as dbc
+
+layout = dbc.Container([
+    dcc.Location('pipes-location'),
+    html.Div(id='pipe-output-div'),
+])

meerschaum/api/dash/pipes.py

@@ -48,13 +48,14 @@ def pipe_from_ctx(ctx, trigger_property: str = 'n_clicks') -> Union[mrsm.Pipe, N
         return None
     return mrsm.Pipe(**meta)
 
+
 def keys_from_state(
-        state: Dict[str, Any],
-        with_params: bool = False
-    ) -> Union[
-        Tuple[List[str], List[str], List[str]],
-        Tuple[List[str], List[str], List[str], str],
-    ]:
+    state: Dict[str, Any],
+    with_params: bool = False
+) -> Union[
+    Tuple[List[str], List[str], List[str]],
+    Tuple[List[str], List[str], List[str], str],
+]:
     """
     Read the current state and return the selected keys lists.
     """
@@ -85,10 +86,11 @@ def keys_from_state(
         keys.append(params)
     return tuple(keys)
 
+
 def pipes_from_state(
-        state: Dict[str, Any],
-        **kw
-    ):
+    state: Dict[str, Any],
+    **kw
+):
     _ck, _mk, _lk, _params = keys_from_state(state, with_params=True)
     try:
         _pipes = _get_pipes(
@@ -103,10 +105,10 @@ def pipes_from_state(
 
 
 def build_pipe_card(
-        pipe: mrsm.Pipe,
-        authenticated: bool = False,
-        _build_children_num: int = 10,
-    ) -> 'dbc.Card':
+    pipe: mrsm.Pipe,
+    authenticated: bool = False,
+    _build_children_num: int = 10,
+) -> 'dbc.Card':
     """
     Return a card for the given pipe.
 
@@ -128,11 +130,11 @@ def build_pipe_card(
             dbc.Col(
                 (
                     dbc.DropdownMenu(
-                        label = "Manage",
-                        children = [
+                        label="Manage",
+                        children=[
                             dbc.DropdownMenuItem(
                                 'Open in Python',
-                                id = {
+                                id={
                                     'type': 'manage-pipe-button',
                                     'index': meta_str,
                                     'action': 'python',
@@ -140,7 +142,7 @@ def build_pipe_card(
                             ),
                             dbc.DropdownMenuItem(
                                 'Delete',
-                                id = {
+                                id={
                                     'type': 'manage-pipe-button',
                                     'index': meta_str,
                                     'action': 'delete',
@@ -148,7 +150,7 @@ def build_pipe_card(
                             ),
                             dbc.DropdownMenuItem(
                                 'Drop',
-                                id = {
+                                id={
                                     'type': 'manage-pipe-button',
                                     'index': meta_str,
                                     'action': 'drop',
@@ -156,7 +158,7 @@ def build_pipe_card(
                             ),
                             dbc.DropdownMenuItem(
                                 'Clear',
-                                id = {
+                                id={
                                     'type': 'manage-pipe-button',
                                     'index': meta_str,
                                     'action': 'clear',
@@ -164,7 +166,7 @@ def build_pipe_card(
                             ),
                             dbc.DropdownMenuItem(
                                 'Verify',
-                                id = {
+                                id={
                                     'type': 'manage-pipe-button',
                                     'index': meta_str,
                                     'action': 'verify',
@@ -172,56 +174,86 @@ def build_pipe_card(
                             ),
                             dbc.DropdownMenuItem(
                                 'Sync',
-                                id = {
+                                id={
                                     'type': 'manage-pipe-button',
                                     'index': meta_str,
                                     'action': 'sync',
                                 },
                             ),
                         ],
-                        direction = "up",
-                        menu_variant = "dark",
-                        size = 'sm',
-                        color = 'secondary',
+                        direction="up",
+                        menu_variant="dark",
+                        size='sm',
+                        color='secondary',
                     )
                 ) if authenticated else [],
-                width = 2,
+                width=2,
             ),
             dbc.Col(width=6),
             dbc.Col(
                 dbc.Button(
                     'Download CSV',
-                    size = 'sm',
-                    color = 'link',
-                    style = {'float': 'right'},
-                    id = {'type': 'pipe-download-csv-button', 'index': meta_str},
+                    size='sm',
+                    color='link',
+                    style={'float': 'right'},
+                    id={'type': 'pipe-download-csv-button', 'index': meta_str},
                 ),
-                width = 4,
+                width=4,
             ),
         ],
-        justify = 'start',
+        justify='start',
     )
     card_body_children = [
-        html.H5(
-            html.B(str(pipe)),
-            className = 'card-title',
-            style = {'font-family': ['monospace']}
-        ),
         html.Div(
             dbc.Accordion(
                 accordion_items_from_pipe(
                     pipe,
-                    authenticated = authenticated,
-                    _build_children_num = _build_children_num,
+                    authenticated=authenticated,
+                    _build_children_num=_build_children_num,
                 ),
-                flush = True,
-                start_collapsed = True,
-                id = {'type': 'pipe-accordion', 'index': meta_str},
+                flush=True,
+                start_collapsed=True,
+                id={'type': 'pipe-accordion', 'index': meta_str},
             )
         )
 
     ]
+
+    pipe_url = (
+        f"/dash/pipes/{pipe.connector_keys}/{pipe.metric_key}/{pipe.location_key}"
+    )
+
+    card_header_children = dbc.Row(
+        [
+            dbc.Col(
+                html.H5(
+                    html.B(str(pipe)),
+                    className='card-title',
+                    style={'font-family': ['monospace']}
+                ),
+                width=11,
+            ),
+            dbc.Col(
+                dbc.Button(
+                    [
+                        html.I(
+                            className="bi bi-share",
+                        ),
+                    ],
+                    #  href=pipe_url,
+                    style={'float': 'right'},
+                    outline=True,
+                    color='link',
+                    id={'type': 'share-pipe-button', 'index': meta_str},
+                ),
+                width=1,
+            ),
+        ],
+        justify='start',
+    )
+
     return dbc.Card([
+        dbc.CardHeader(children=card_header_children),
         dbc.CardBody(children=card_body_children),
         dbc.CardFooter(children=footer_children),
     ])
@@ -270,11 +302,11 @@ def get_pipes_cards(*keys, session_data: Optional[Dict[str, Any]] = None):
 
 
 def accordion_items_from_pipe(
-        pipe: mrsm.Pipe,
-        active_items: Optional[List[str]] = None,
-        authenticated: bool = False,
-        _build_children_num: int = 10,
-    ) -> 'List[dbc.AccordionItem]':
+    pipe: mrsm.Pipe,
+    active_items: Optional[List[str]] = None,
+    authenticated: bool = False,
+    _build_children_num: int = 10,
+) -> 'List[dbc.AccordionItem]':
     """
     Build the accordion items for a given pipe.
     """

meerschaum/api/dash/users.py

@@ -18,6 +18,7 @@ from meerschaum.core import User
 dcc, html = import_dcc(check_update=CHECK_UPDATE), import_html(check_update=CHECK_UPDATE)
 dbc = attempt_import('dash_bootstrap_components', lazy=False, check_update=CHECK_UPDATE)
 
+
 def get_users_cards(state: WebState) -> Tuple[List[dbc.Card], List[SuccessTuple]]:
     """
     Return the cards and alerts for users.
@@ -52,21 +53,33 @@ def is_session_authenticated(session_id: str) -> bool:
     """
     if no_auth:
         return True
+    if session_id not in active_sessions:
+        return False
     if session_id in unauthenticated_sessions:
         return False
     if session_id in authenticated_sessions:
         return True
+
     permissions = get_config('system', 'api', 'permissions')
     allow_non_admin = permissions.get('actions', {}).get('non_admin', False)
-    if allow_non_admin:
-        return True
-    conn = get_api_connector()
+
+    is_auth = True if allow_non_admin else session_is_admin(session_id)
     username = active_sessions.get(session_id, {}).get('username', None)
-    user = User(username, instance=conn)
-    user_type = conn.get_user_type(user, debug=debug)
-    is_auth = user_type == 'admin'
+
     if is_auth:
         authenticated_sessions[session_id] = username
     else:
         unauthenticated_sessions[session_id] = username
+
     return is_auth
+
+
+def session_is_admin(session_id: str) -> bool:
+    """
+    Check whether a session ID corresponds to an admin user.
+    """
+    conn = get_api_connector()
+    username = active_sessions.get(session_id, {}).get('username', None)
+    user = User(username, instance=conn)
+    user_type = conn.get_user_type(user, debug=debug)
+    return user_type == 'admin'

meerschaum/api/routes/_login.py

@@ -23,8 +23,8 @@ from meerschaum.api._oauth2 import CustomOAuth2PasswordRequestForm
 
 @manager.user_loader()
 def load_user(
-        username: str
-    ) -> User:
+    username: str
+) -> User:
     """
     Create the `meerschaum.core.User` object from the username.
     """
@@ -33,8 +33,8 @@ def load_user(
 
 @app.post(endpoints['login'], tags=['Users'])
 def login(
-        data: CustomOAuth2PasswordRequestForm = fastapi.Depends()
-    ) -> Dict[str, Any]:
+    data: CustomOAuth2PasswordRequestForm = fastapi.Depends()
+) -> Dict[str, Any]:
     """
     Login and set the session token.
     """

meerschaum/api/routes/_pipes.py

@@ -474,9 +474,9 @@ def get_pipe_data(
         df[col] = df[col].apply(lambda x: f'{x:f}' if isinstance(x, Decimal) else x)
 
     json_content = df.to_json(
-        date_format = 'iso',
-        orient = 'records',
-        date_unit = 'us',
+        date_format='iso',
+        orient='records',
+        date_unit='us',
     )
 
     return fastapi.Response(

meerschaum/config/_default.py

@@ -52,6 +52,14 @@ default_meerschaum_config = {
                 'protocol': 'https',
             },
         },
+        'valkey': {
+            'main': {
+                'host': 'localhost',
+                'username': 'default',
+                'password': 'mrsm',
+                'port': 6379,
+            },
+        },
     },
 }
 default_system_config = {
@@ -74,7 +82,7 @@ default_system_config = {
             },
         },
 
-        'api' : {
+        'api': {
         },
     },
     ### not to be confused with system_config['connectors']['api'], this is the configuration

meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "2.3.6"
+__version__ = "2.4.0.dev1"