meerschaum 3.0.0rc3__tar.gz → 3.0.0rc4__tar.gz

{meerschaum-3.0.0rc3/meerschaum.egg-info → meerschaum-3.0.0rc4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: meerschaum
-Version: 3.0.0rc3
+Version: 3.0.0rc4
 Summary: Sync Time-Series Pipes with Meerschaum
 Author-email: Bennett Meares <bennett.meares@gmail.com>
 Maintainer-email: Bennett Meares <bennett.meares@gmail.com>

{meerschaum-3.0.0rc3 → meerschaum-3.0.0rc4}/meerschaum/api/_events.py

@@ -98,6 +98,11 @@ async def startup():
         await shutdown()
         os._exit(1)
 
+    conn = get_api_connector()
+    if conn.type == 'sql':
+        from meerschaum.connectors.sql.tables import get_tables
+        _ = get_tables(conn, refresh=True, create=True, debug=debug)
+
     start_check_jobs_thread()
 
 

{meerschaum-3.0.0rc3 → meerschaum-3.0.0rc4}/meerschaum/api/dash/callbacks/dashboard.py

@@ -498,13 +498,9 @@ def update_flags(input_flags_dropdown_values, n_clicks, input_flags_texts):
 
 @dash_app.callback(
     Output('connector-keys-dropdown', 'options'),
-    Output('connector-keys-dropdown', 'value'),
     Output('metric-keys-dropdown', 'options'),
-    Output('metric-keys-dropdown', 'value'),
     Output('location-keys-dropdown', 'options'),
-    Output('location-keys-dropdown', 'value'),
     Output('tags-dropdown', 'options'),
-    Output('tags-dropdown', 'value'),
     Output('instance-select', 'value'),
     Output('instance-alert-div', 'children'),
     Input('connector-keys-dropdown', 'value'),
@@ -546,15 +542,6 @@ def update_keys_options(
     except Exception as e:
         instance_alerts += [alert_from_success_tuple((False, str(e)))]
 
-    ### Update the keys filters.
-    if connector_keys is None:
-        connector_keys = []
-    if metric_keys is None:
-        metric_keys = []
-    if location_keys is None:
-        location_keys = []
-    if tags is None:
-        tags = []
     num_filter = 0
     if connector_keys:
         num_filter += 1
@@ -565,10 +552,6 @@ def update_keys_options(
     if tags:
         num_filter += 1
 
-    _ck_filter = connector_keys
-    _mk_filter = metric_keys
-    _lk_filter = location_keys
-    _tags_filter = tags
     _ck_alone = (connector_keys and num_filter == 1) or instance_click
     _mk_alone = (metric_keys and num_filter == 1) or instance_click
     _lk_alone = (location_keys and num_filter == 1) or instance_click
@@ -581,101 +564,95 @@ def update_keys_options(
         _keys = fetch_pipes_keys(
             'registered',
             get_web_connector(ctx.states),
-            connector_keys=_ck_filter,
-            metric_keys=_mk_filter,
-            location_keys=_lk_filter,
-            tags=_tags_filter,
+            connector_keys=connector_keys,
+            metric_keys=metric_keys,
+            location_keys=location_keys,
+            tags=tags,
         )
         _tags_pipes = mrsm.get_pipes(
-            connector_keys=_ck_filter,
-            metric_keys=_mk_filter,
-            location_keys=_lk_filter,
-            tags=_tags_filter,
+            connector_keys=connector_keys,
+            metric_keys=metric_keys,
+            location_keys=location_keys,
+            tags=tags,
             instance=get_web_connector(ctx.states),
             as_tags_dict=True,
         )
         _all_tags = list(
-            mrsm.get_pipes(
-                instance=get_web_connector(ctx.states),
-                as_tags_dict=True,
-            )
+            set(
+                mrsm.get_pipes(
+                    instance=get_web_connector(ctx.states),
+                    as_tags_dict=True,
+                )
+            ).union(tags or [])
         ) if _tags_alone else []
     except Exception as e:
         instance_alerts += [alert_from_success_tuple((False, str(e)))]
-        _all_keys, _keys = [], []
-    _connectors_options = []
-    _metrics_options = []
-    _locations_options = []
-    _tags_options = []
-
-    _seen_keys = {'ck' : set(), 'mk' : set(), 'lk' : set(), 'tags': set()}
-
-    def add_options(options, keys, key_type):
-        for ck, mk, lk in keys:
-            k = locals()[key_type]
-            if k not in _seen_keys[key_type]:
-                _k = 'None' if k in (None, '[None]', 'None', 'null') else k
-                options.append({'label': _k, 'value': _k})
-                _seen_keys[key_type].add(k)
-
-    add_options(_connectors_options, _all_keys if _ck_alone else _keys, 'ck')
-    add_options(_metrics_options, _all_keys if _mk_alone else _keys, 'mk')
-    add_options(_locations_options, _all_keys if _lk_alone else _keys, 'lk')
-
-    _tags_options = [
-        {'label': tag, 'value': tag}
-        for tag in (_all_tags if _tags_alone else _tags_pipes)
-    ]
+        _all_keys, _all_tags, _keys = [], [], []
+
+    connectors_options = sorted(
+        list(
+            set(
+                keys_tuple[0] for keys_tuple in (_all_keys if _ck_alone else _keys)
+            ).union(set(connector_keys or []))
+        ),
+        key=(lambda x: str(x).lower()),
+    )
+    metrics_options = sorted(
+        list(
+            set(
+                keys_tuple[1] for keys_tuple in (_all_keys if _mk_alone else _keys)
+            ).union(set(metric_keys or []))
+        ),
+        key=(lambda x: str(x).lower()),
+    )
+    locations_options = sorted(
+        list(
+            set(
+                (
+                    str(keys_tuple[2])
+                    for keys_tuple in (_all_keys if _lk_alone else _keys)
+                )
+            ).union(set((str(_lk) for _lk in (location_keys or []))))
+        ),
+        key=(lambda x: str(x).lower()),
+    )
+
+    tags_options = sorted(
+        list(
+            set(
+                (_all_tags if _tags_alone else _tags_pipes)
+            ).union(set(tags or []))
+        ),
+        key=(lambda x: str(x).lower()),
+    )
 
-    _connectors_options.sort(key=lambda x: str(x).lower())
-    _metrics_options.sort(key=lambda x: str(x).lower())
-    _locations_options.sort(key=lambda x: str(x).lower())
-    _tags_options.sort(key=lambda x: str(x).lower())
-    connector_keys = [
-        ck
-        for ck in connector_keys
-        if ck in [
-            _ck['value']
-            for _ck in _connectors_options
-        ]
-    ]
-    metric_keys = [
-        mk
-        for mk in metric_keys
-        if mk in [
-            _mk['value']
-            for _mk in _metrics_options
-        ]
-    ]
-    location_keys = [
-        lk
-        for lk in location_keys
-        if lk in [
-            _lk['value']
-            for _lk in _locations_options
-        ]
-    ]
-    tags = [
-        tag
-        for tag in tags
-        if tag in [
-            _tag['value']
-            for _tag in _tags_options
-        ]
-    ]
     return (
-        _connectors_options,
-        connector_keys,
-        _metrics_options,
-        metric_keys,
-        _locations_options,
-        location_keys,
-        _tags_options,
-        tags,
+        connectors_options,
+        metrics_options,
+        locations_options,
+        tags_options,
         (instance_keys if update_instance_keys else dash.no_update),
         instance_alerts,
     )
 
+
+@dash_app.callback(
+    Output('connector-keys-dropdown', 'value'),
+    Output('metric-keys-dropdown', 'value'),
+    Output('location-keys-dropdown', 'value'),
+    Output('tags-dropdown', 'value'),
+    Input('clear-all-keys-button', 'n_clicks'),
+    prevent_initial_call=True,
+)
+def clear_all_keys_button_click(n_clicks):
+    """
+    Clear the keys dropdowns when the `Clear all` button is clicked.
+    """
+    if not n_clicks:
+        raise PreventUpdate
+
+    return [], [], [], []
+
 dash_app.clientside_callback(
     """
     function(

meerschaum-3.0.0rc4/meerschaum/api/dash/callbacks/pipes.py

@@ -0,0 +1,235 @@
+#! /usr/bin/env python3
+# vim:fenc=utf-8
+
+"""
+Define callbacks for the `/dash/pipes/` page.
+"""
+
+from urllib.parse import parse_qs, quote_plus
+from typing import List, Optional, Dict, Any
+
+import dash
+from dash.dependencies import Input, Output, State
+from dash import no_update
+from dash.exceptions import PreventUpdate
+import dash_bootstrap_components as dbc
+
+import meerschaum as mrsm
+from meerschaum.api.dash import dash_app
+from meerschaum.api.dash.components import (
+    alert_from_success_tuple,
+    build_cards_grid,
+)
+from meerschaum.api.dash.pipes import (
+    build_pipe_card,
+    build_pipes_dropdown_keys_row,
+    build_pipes_tags_dropdown,
+    build_pipes_navbar,
+)
+from meerschaum.api import CHECK_UPDATE, get_api_connector
+from meerschaum.utils.packages import import_html, import_dcc
+from meerschaum.api.dash.sessions import is_session_authenticated
+html, dcc = import_html(check_update=CHECK_UPDATE), import_dcc(check_update=CHECK_UPDATE)
+
+
+@dash_app.callback(
+    Output('pipes-navbar-div', 'children'),
+    Output('pipe-output-div', 'children'),
+    Input('pipes-location', 'pathname'),
+    State('pipes-location', 'search'),
+    State('session-store', 'data'),
+)
+def render_pipe_page_from_url(
+    pathname: str,
+    pipe_search: str,
+    session_data: Optional[Dict[str, Any]],
+):
+    if not str(pathname).startswith('/dash/pipes'):
+        raise PreventUpdate
+
+    session_id = (session_data or {}).get('session-id', None)
+    authenticated = is_session_authenticated(str(session_id))
+    query_params = parse_qs(pipe_search.lstrip('?')) if pipe_search else {}
+    instance = query_params.get('instance', [None])[0] or str(get_api_connector())
+    tags = query_params.get('tags', [None])[0] or []
+    if isinstance(tags, str):
+        tags = tags.split(',')
+
+    connector_keys = query_params.get('connector_keys', [None])[0] or []
+    if isinstance(connector_keys, str):
+        connector_keys = connector_keys.split(',')
+
+    metric_keys = query_params.get('metric_keys', [None])[0] or []
+    if isinstance(metric_keys, str):
+        metric_keys = metric_keys.split(',')
+
+    location_keys = query_params.get('location_keys', [None])[0] or []
+    if isinstance(location_keys, str):
+        location_keys = location_keys.split(',')
+
+    keys = pathname.replace('/dash/pipes', '').lstrip('/').rstrip('/').split('/')
+    instance_connector = mrsm.get_connector(instance)
+    viewing_single_pipe = len(keys) in (2, 3)
+    if instance_connector is None:
+        return (
+            build_pipes_navbar(instance, with_instance_select=(not viewing_single_pipe)),
+            [
+                html.Br(),
+                alert_from_success_tuple((False, f"Invalid instance keys '{instance}'.")),
+                html.Br(),
+            ]
+        )
+
+    if not viewing_single_pipe:
+        try:
+            pipes = mrsm.get_pipes(
+                as_list=True,
+                connector_keys=connector_keys,
+                metric_keys=metric_keys,
+                location_keys=location_keys,
+                tags=tags,
+                instance=instance_connector,
+            )
+        except Exception as e:
+            return (
+                build_pipes_navbar(instance, with_instance_select=False),
+                [
+                    html.Br(),
+                    alert_from_success_tuple(
+                        (False, f"Failed to get pipes for instance '{instance}':\n{e}")
+                    ),
+                    html.Br(),
+                    dbc.Row(
+                        [
+                            dbc.Button(
+                                "Reload",
+                                id='pipes-reload-button',
+                                size='lg',
+                                href=(
+                                    "/dash/pipes"
+                                    if pathname.startswith('/dash/pipes/')
+                                    else "/dash/pipes/" 
+                                )
+                            ),
+                        ],
+                        justify='center',
+                        align='center',
+                        className='h-50',
+                    ),
+                ]
+            )
+
+        cards = [
+            build_pipe_card(pipe, authenticated=authenticated, include_manage=False)
+            for pipe in pipes
+        ]
+        return (
+            build_pipes_navbar(instance, with_instance_select=True),
+            [
+                html.Div([
+                    html.Br(),
+                    build_pipes_dropdown_keys_row(
+                        connector_keys,
+                        metric_keys,
+                        location_keys,
+                        tags,
+                        pipes,
+                        instance_connector,
+                    ),
+                    html.Br(),
+                    build_pipes_tags_dropdown(
+                        connector_keys,
+                        metric_keys,
+                        location_keys,
+                        tags,
+                        instance,
+                    ),
+                ]),
+                html.Br(),
+                build_cards_grid(cards, 1),
+                html.Br(),
+            ]
+        )
+
+    ck = keys[0]
+    mk = keys[1]
+    lk = keys[2] if len(keys) == 3 else None
+
+    pipe = mrsm.Pipe(ck, mk, lk, instance=instance)
+    return (
+        build_pipes_navbar(instance, with_instance_select=False),
+        [
+            html.Br(),
+            build_pipe_card(pipe, authenticated=authenticated, include_manage=False),
+            html.Br(),
+        ]
+    )
+
+
+@dash_app.callback(
+    Output('pipes-location', 'search'),
+    Input('pipes-connector-keys-dropdown', 'value'),
+    Input('pipes-metric-keys-dropdown', 'value'),
+    Input('pipes-location-keys-dropdown', 'value'),
+    Input('pipes-tags-dropdown', 'value'),
+    Input('instance-select', 'value'),
+    Input('pipes-clear-all-button', 'n_clicks'),
+)
+def update_location_on_pipes_filter_change(
+    connector_keys: Optional[List[str]],
+    metric_keys: Optional[List[str]],
+    location_keys: Optional[List[str]],
+    tags: Optional[List[str]],
+    instance_keys: str,
+    clear_all_button_n_clicks: Optional[int],
+):
+    """
+    Update the URL parameters when clicking the dropdowns.
+    """
+    ctx = dash.callback_context.triggered
+    if len(ctx) != 1:
+        raise PreventUpdate
+
+    if not any(
+        (connector_keys or [])
+        + (metric_keys or [])
+        + (location_keys or [])
+        + (tags or [])
+        + ([instance_keys] if instance_keys else [])
+    ):
+        return ''
+
+    if ctx[0].get('prop_id', None) == 'pipes-clear-all-button.n_clicks':
+        connector_keys = []
+        metric_keys = []
+        location_keys = []
+        tags = []
+
+    include_instance_keys = instance_keys and instance_keys != str(get_api_connector())
+    search_str = ""
+    
+    if connector_keys:
+        search_str += "connector_keys=" + ','.join((quote_plus(ck) for ck in connector_keys))
+        if metric_keys or location_keys or tags or include_instance_keys:
+            search_str += '&'
+
+    if metric_keys:
+        search_str += "metric_keys=" + ','.join((quote_plus(mk) for mk in metric_keys))
+        if location_keys or tags or include_instance_keys:
+            search_str += '&'
+
+    if location_keys:
+        search_str += "location_keys=" + ','.join((quote_plus(str(lk)) for lk in location_keys))
+        if tags or include_instance_keys:
+            search_str += '&'
+
+    if tags:
+        search_str += "tags=" + ','.join((quote_plus(tag) for tag in tags))
+        if include_instance_keys:
+            search_str += '&'
+
+    if instance_keys:
+        if include_instance_keys:
+            search_str += "instance=" + quote_plus(instance_keys)
+
+    return ('?' + search_str) if search_str else ''

{meerschaum-3.0.0rc3 → meerschaum-3.0.0rc4}/meerschaum/api/dash/callbacks/tokens.py

@@ -54,7 +54,8 @@ def refresh_tokens_button_click(
                 html.H4('No tokens registered.'),
                 html.P('Click the `+` button to register a new token.'),
             ],
-            alerts
+            build_tokens_register_input_modal(),
+            alerts,
         )
 
     return tokens_table, build_tokens_register_input_modal(), alerts

{meerschaum-3.0.0rc3 → meerschaum-3.0.0rc4}/meerschaum/api/dash/components.py

@@ -13,7 +13,7 @@ from meerschaum.utils.typing import SuccessTuple, List
 from meerschaum._internal.static import STATIC_CONFIG
 from meerschaum.utils.misc import remove_ansi
 from meerschaum._internal.shell.Shell import get_shell_intro
-from meerschaum.api import endpoints, CHECK_UPDATE, docs_enabled
+from meerschaum.api import endpoints, CHECK_UPDATE, docs_enabled, get_api_connector
 from meerschaum.connectors import instance_types, _load_builtin_custom_connectors
 from meerschaum.utils.misc import get_connector_labels
 from meerschaum.config import __doc__ as doc
@@ -104,7 +104,7 @@ instance_select = dbc.Select(
     id='instance-select',
     size='sm',
     options=[
-        {'label': i, 'value': i}
+        {'label': (i[:32] + '…') if len(i) > 32 else i, 'value': i}
         for i in get_connector_labels(*instance_types)
     ],
     class_name='dbc_dark custom-select custom-select-sm',
@@ -163,6 +163,7 @@ pages_navbar = html.Div(
     id='pages-navbar-div',
 )
 
+
 navbar = dbc.Navbar(
     dbc.Container(
         [
@@ -171,13 +172,11 @@ navbar = dbc.Navbar(
             dbc.Collapse(
                 dbc.Row(
                     [
-                        dbc.Col(instance_select),
-                        dbc.Col(
-                            sign_out_button,
-                            className="ms-auto",
-                        ),
+                        dbc.Col(instance_select, width="auto"),
+                        dbc.Col(sign_out_button, width="auto"),
                     ],
                     className="g-0 ms-auto flex-nowrap mt-3 mt-md-0",
+                    align="center",
                 ),
                 id='navbar-collapse',
                 is_open=False,

{meerschaum-3.0.0rc3 → meerschaum-3.0.0rc4}/meerschaum/api/dash/keys.py

@@ -227,7 +227,23 @@ dropdown_tab_content = html.Div([
             [
                 dropdown_keys_row,
                 html.Br(),
-                tags_dropdown,
+                dbc.Row(
+                    [
+                        dbc.Col(tags_dropdown, width=True),
+                        dbc.Col(
+                            dbc.Button(
+                                "Clear all",
+                                id='clear-all-keys-button',
+                                color='link',
+                                size='sm',
+                                style={'text-decoration': 'none'},
+                            ),
+                            width='auto',
+                        ),
+                    ],
+                    className='g-0',
+                    align='center',
+                ),
             ], ### end of card children
             className='card-text',
         )

{meerschaum-3.0.0rc3 → meerschaum-3.0.0rc4}/meerschaum/api/dash/pages/pipes.py

@@ -7,17 +7,27 @@ Display pipes via a shareable URL.
 
 from meerschaum.api import CHECK_UPDATE
 from meerschaum.utils.packages import import_html, import_dcc
-from meerschaum.api.dash.components import download_dataframe, pages_navbar, navbar
+from meerschaum.api.dash.components import (
+    download_dataframe,
+)
 
 html, dcc = import_html(check_update=CHECK_UPDATE), import_dcc(check_update=CHECK_UPDATE)
 import dash_bootstrap_components as dbc
 
-
 layout = [
-    pages_navbar,
+    html.Div(id='pipes-navbar-div'),
     dcc.Location('pipes-location'),
     download_dataframe,
     dbc.Container([
-        html.Div(id='pipe-output-div'),
+        dcc.Loading(
+            html.Div(id='pipe-output-div'),
+            id='pipes-loading',
+            type='circle',
+            delay_hide=1000,
+            delay_show=1000,
+            style={
+                'padding-top': '100px',
+            },
+        ),
     ])
 ]