meerschaum 3.0.0rc2__py3-none-any.whl → 3.0.0rc4__py3-none-any.whl

meerschaum/_internal/shell/Shell.py

@@ -781,10 +781,11 @@ class Shell(cmd.Cmd):
             instance_keys += ':main'
 
         conn_attrs = parse_instance_keys(instance_keys, construct=False, debug=debug)
-        if conn_attrs is None or not conn_attrs:
-            conn_keys = str(get_connector(debug=debug))
-        else:
-            conn_keys = instance_keys
+        conn_keys = (
+            str(get_connector(debug=debug))
+            if conn_attrs is None
+            else instance_keys
+        )
 
         shell_attrs['instance_keys'] = conn_keys
 

meerschaum/actions/bootstrap.py

@@ -429,7 +429,7 @@ def _bootstrap_plugins(
         action = [prompt("Enter the name of your new plugin:")]
 
     for plugin_name in action:
-        bootstrap_success, bootstrap_msg = bootstrap_plugin(plugin_name)
+        bootstrap_success, bootstrap_msg = bootstrap_plugin(plugin_name, debug=debug)
         if not bootstrap_success:
             return bootstrap_success, bootstrap_msg
 

meerschaum/actions/edit.py

@@ -552,7 +552,12 @@ def _edit_tokens(
     dateutil_parser = mrsm.attempt_import('dateutil.parser')
 
     if not action:
-        return False, "Provide token labels or IDs for the tokens to edit."
+        return (
+            False, (
+                "Provide token labels or IDs for the tokens to edit\n"
+                "    (run `show tokens` to see registered tokens)."
+            )
+        )
 
     conn = parse_instance_keys(mrsm_instance)
 
@@ -628,8 +633,6 @@ def _edit_tokens(
 
     return True, msg
 
-        
-
 
 ### NOTE: This must be the final statement of the module.
 ###       Any subactions added below these lines will not

meerschaum/actions/start.py

@@ -676,7 +676,7 @@ def _start_pipeline(
 
     try:
         run_loop()
-    except KeyboardInterrupt:
+    except (KeyboardInterrupt, SystemExit):
         warn("Cancelled pipeline.", stack=False)
         if proc is not None:
             _stop_process(proc)

meerschaum/api/_events.py

@@ -98,6 +98,11 @@ async def startup():
         await shutdown()
         os._exit(1)
 
+    conn = get_api_connector()
+    if conn.type == 'sql':
+        from meerschaum.connectors.sql.tables import get_tables
+        _ = get_tables(conn, refresh=True, create=True, debug=debug)
+
     start_check_jobs_thread()
 
 

meerschaum/api/dash/callbacks/__init__.py

@@ -10,6 +10,7 @@ from meerschaum.api import debug as _debug
 import meerschaum.api.dash.callbacks.dashboard
 import meerschaum.api.dash.callbacks.login
 import meerschaum.api.dash.callbacks.plugins
+import meerschaum.api.dash.callbacks.tokens
 import meerschaum.api.dash.callbacks.jobs
 import meerschaum.api.dash.callbacks.register
 import meerschaum.api.dash.callbacks.pipes

meerschaum/api/dash/callbacks/dashboard.py

@@ -72,8 +72,6 @@ keys_state = (
     State({'type': 'input-flags-dropdown', 'index': ALL}, 'value'),
     State({'type': 'input-flags-dropdown-text', 'index': ALL}, 'value'),
     State('instance-select', 'value'),
-    State('content-div-right', 'children'),
-    State('success-alert-div', 'children'),
     State('session-store', 'data'),
 )
 
@@ -103,15 +101,20 @@ omit_actions = {
 _paths = {
     'login'   : pages.login.layout,
     ''        : pages.dashboard.layout,
+    'pipes'   : pages.pipes.layout,
     'plugins' : pages.plugins.layout,
+    'tokens'  : pages.tokens.layout,
     'register': pages.register.layout,
     'pipes'   : pages.pipes.layout,
-    'job'     : pages.job.layout,
+    'jobs'    : pages.jobs.layout,
 }
-_required_login = {''}
+_required_login = {'', 'tokens', 'jobs', 'pipes'}
 _pages = {
     'Web Console': '/dash/',
+    'Pipes': '/dash/pipes',
     'Plugins': '/dash/plugins',
+    'Tokens': '/dash/tokens',
+    'Jobs': '/dash/jobs',
 }
 
 
@@ -495,17 +498,9 @@ def update_flags(input_flags_dropdown_values, n_clicks, input_flags_texts):
 
 @dash_app.callback(
     Output('connector-keys-dropdown', 'options'),
-    Output('connector-keys-list', 'children'),
-    Output('connector-keys-dropdown', 'value'),
     Output('metric-keys-dropdown', 'options'),
-    Output('metric-keys-list', 'children'),
-    Output('metric-keys-dropdown', 'value'),
     Output('location-keys-dropdown', 'options'),
-    Output('location-keys-list', 'children'),
-    Output('location-keys-dropdown', 'value'),
     Output('tags-dropdown', 'options'),
-    Output('tags-list', 'children'),
-    Output('tags-dropdown', 'value'),
     Output('instance-select', 'value'),
     Output('instance-alert-div', 'children'),
     Input('connector-keys-dropdown', 'value'),
@@ -547,15 +542,6 @@ def update_keys_options(
     except Exception as e:
         instance_alerts += [alert_from_success_tuple((False, str(e)))]
 
-    ### Update the keys filters.
-    if connector_keys is None:
-        connector_keys = []
-    if metric_keys is None:
-        metric_keys = []
-    if location_keys is None:
-        location_keys = []
-    if tags is None:
-        tags = []
     num_filter = 0
     if connector_keys:
         num_filter += 1
@@ -566,10 +552,6 @@ def update_keys_options(
     if tags:
         num_filter += 1
 
-    _ck_filter = connector_keys
-    _mk_filter = metric_keys
-    _lk_filter = location_keys
-    _tags_filter = tags
     _ck_alone = (connector_keys and num_filter == 1) or instance_click
     _mk_alone = (metric_keys and num_filter == 1) or instance_click
     _lk_alone = (location_keys and num_filter == 1) or instance_click
@@ -582,109 +564,95 @@ def update_keys_options(
         _keys = fetch_pipes_keys(
             'registered',
             get_web_connector(ctx.states),
-            connector_keys=_ck_filter,
-            metric_keys=_mk_filter,
-            location_keys=_lk_filter,
-            tags=_tags_filter,
+            connector_keys=connector_keys,
+            metric_keys=metric_keys,
+            location_keys=location_keys,
+            tags=tags,
         )
         _tags_pipes = mrsm.get_pipes(
-            connector_keys=_ck_filter,
-            metric_keys=_mk_filter,
-            location_keys=_lk_filter,
-            tags=_tags_filter,
+            connector_keys=connector_keys,
+            metric_keys=metric_keys,
+            location_keys=location_keys,
+            tags=tags,
             instance=get_web_connector(ctx.states),
             as_tags_dict=True,
         )
         _all_tags = list(
-            mrsm.get_pipes(
-                instance=get_web_connector(ctx.states),
-                as_tags_dict=True,
-            )
+            set(
+                mrsm.get_pipes(
+                    instance=get_web_connector(ctx.states),
+                    as_tags_dict=True,
+                )
+            ).union(tags or [])
         ) if _tags_alone else []
     except Exception as e:
         instance_alerts += [alert_from_success_tuple((False, str(e)))]
-        _all_keys, _keys = [], []
-    _connectors_options = []
-    _metrics_options = []
-    _locations_options = []
-    _tags_options = []
-
-    _seen_keys = {'ck' : set(), 'mk' : set(), 'lk' : set(), 'tags': set()}
-
-    def add_options(options, keys, key_type):
-        for ck, mk, lk in keys:
-            k = locals()[key_type]
-            if k not in _seen_keys[key_type]:
-                _k = 'None' if k in (None, '[None]', 'None', 'null') else k
-                options.append({'label': _k, 'value': _k})
-                _seen_keys[key_type].add(k)
-
-    add_options(_connectors_options, _all_keys if _ck_alone else _keys, 'ck')
-    add_options(_metrics_options, _all_keys if _mk_alone else _keys, 'mk')
-    add_options(_locations_options, _all_keys if _lk_alone else _keys, 'lk')
-
-    _tags_options = [
-        {'label': tag, 'value': tag}
-        for tag in (_all_tags if _tags_alone else _tags_pipes)
-    ]
+        _all_keys, _all_tags, _keys = [], [], []
+
+    connectors_options = sorted(
+        list(
+            set(
+                keys_tuple[0] for keys_tuple in (_all_keys if _ck_alone else _keys)
+            ).union(set(connector_keys or []))
+        ),
+        key=(lambda x: str(x).lower()),
+    )
+    metrics_options = sorted(
+        list(
+            set(
+                keys_tuple[1] for keys_tuple in (_all_keys if _mk_alone else _keys)
+            ).union(set(metric_keys or []))
+        ),
+        key=(lambda x: str(x).lower()),
+    )
+    locations_options = sorted(
+        list(
+            set(
+                (
+                    str(keys_tuple[2])
+                    for keys_tuple in (_all_keys if _lk_alone else _keys)
+                )
+            ).union(set((str(_lk) for _lk in (location_keys or []))))
+        ),
+        key=(lambda x: str(x).lower()),
+    )
+
+    tags_options = sorted(
+        list(
+            set(
+                (_all_tags if _tags_alone else _tags_pipes)
+            ).union(set(tags or []))
+        ),
+        key=(lambda x: str(x).lower()),
+    )
 
-    _connectors_options.sort(key=lambda x: str(x).lower())
-    _metrics_options.sort(key=lambda x: str(x).lower())
-    _locations_options.sort(key=lambda x: str(x).lower())
-    _tags_options.sort(key=lambda x: str(x).lower())
-    connector_keys = [
-        ck
-        for ck in connector_keys
-        if ck in [
-            _ck['value']
-            for _ck in _connectors_options
-        ]
-    ]
-    metric_keys = [
-        mk
-        for mk in metric_keys
-        if mk in [
-            _mk['value']
-            for _mk in _metrics_options
-        ]
-    ]
-    location_keys = [
-        lk
-        for lk in location_keys
-        if lk in [
-            _lk['value']
-            for _lk in _locations_options
-        ]
-    ]
-    tags = [
-        tag
-        for tag in tags
-        if tag in [
-            _tag['value']
-            for _tag in _tags_options
-        ]
-    ]
-    _connectors_datalist = [html.Option(value=o['value']) for o in _connectors_options]
-    _metrics_datalist = [html.Option(value=o['value']) for o in _metrics_options]
-    _locations_datalist = [html.Option(value=o['value']) for o in _locations_options]
-    _tags_datalist = [html.Option(value=o['value']) for o in _tags_options]
     return (
-        _connectors_options,
-        _connectors_datalist,
-        connector_keys,
-        _metrics_options,
-        _metrics_datalist,
-        metric_keys,
-        _locations_options,
-        _locations_datalist,
-        location_keys,
-        _tags_options,
-        _tags_datalist,
-        tags,
+        connectors_options,
+        metrics_options,
+        locations_options,
+        tags_options,
         (instance_keys if update_instance_keys else dash.no_update),
         instance_alerts,
     )
 
+
+@dash_app.callback(
+    Output('connector-keys-dropdown', 'value'),
+    Output('metric-keys-dropdown', 'value'),
+    Output('location-keys-dropdown', 'value'),
+    Output('tags-dropdown', 'value'),
+    Input('clear-all-keys-button', 'n_clicks'),
+    prevent_initial_call=True,
+)
+def clear_all_keys_button_click(n_clicks):
+    """
+    Clear the keys dropdowns when the `Clear all` button is clicked.
+    """
+    if not n_clicks:
+        raise PreventUpdate
+
+    return [], [], [], []
+
 dash_app.clientside_callback(
     """
     function(
@@ -856,6 +824,7 @@ dash_app.clientside_callback(
 @dash_app.callback(
     Output("download-dataframe-csv", "data"),
     Input({'type': 'pipe-download-csv-button', 'index': ALL}, 'n_clicks'),
+    prevent_initial_call=True,
 )
 def download_pipe_csv(n_clicks):
     """
@@ -885,6 +854,7 @@ def download_pipe_csv(n_clicks):
     Output({'type': 'pipe-accordion', 'index': MATCH}, 'children'),
     Input({'type': 'pipe-accordion', 'index': MATCH}, 'active_item'),
     State('session-store', 'data'),
+    prevent_initial_call=True,
 )
 def update_pipe_accordion(item, session_store_data):
     """
@@ -908,7 +878,8 @@ def update_pipe_accordion(item, session_store_data):
 @dash_app.callback(
     Output({'type': 'update-parameters-success-div', 'index': MATCH}, 'children'),
     Input({'type': 'update-parameters-button', 'index': MATCH}, 'n_clicks'),
-    State({'type': 'parameters-editor', 'index': MATCH}, 'value')
+    State({'type': 'parameters-editor', 'index': MATCH}, 'value'),
+    prevent_initial_call=True,
 )
 def update_pipe_parameters_click(n_clicks, parameters_editor_text):
     if not n_clicks:
@@ -942,6 +913,7 @@ def update_pipe_parameters_click(n_clicks, parameters_editor_text):
     Output({'type': 'update-sql-success-div', 'index': MATCH}, 'children'),
     Input({'type': 'update-sql-button', 'index': MATCH}, 'n_clicks'),
     State({'type': 'sql-editor', 'index': MATCH}, 'value'),
+    prevent_initial_call=True,
 )
 def update_pipe_sql_click(n_clicks, sql_editor_text):
     if not n_clicks:
@@ -968,7 +940,8 @@ def update_pipe_sql_click(n_clicks, sql_editor_text):
 @dash_app.callback(
     Output({'type': 'sync-success-div', 'index': MATCH}, 'children'),
     Input({'type': 'update-sync-button', 'index': MATCH}, 'n_clicks'),
-    State({'type': 'sync-editor', 'index': MATCH}, 'value')
+    State({'type': 'sync-editor', 'index': MATCH}, 'value'),
+    prevent_initial_call=True,
 )
 def sync_documents_click(n_clicks, sync_editor_text):
     if not n_clicks:
@@ -1014,6 +987,7 @@ def sync_documents_click(n_clicks, sync_editor_text):
     State({'type': 'limit-input', 'index': MATCH}, 'value'),
     State({'type': 'query-data-begin-input', 'index': MATCH}, 'value'),
     State({'type': 'query-data-end-input', 'index': MATCH}, 'value'),
+    prevent_initial_call=True,
 )
 def query_data_click(n_clicks, query_editor_text, limit_value, begin, end):
     triggered = dash.callback_context.triggered
@@ -1130,6 +1104,7 @@ def toggle_navbar_collapse(n_clicks: Optional[int], is_open: bool) -> bool:
     Output('session-store', 'data'),
     Input("sign-out-button", "n_clicks"),
     State('session-store', 'data'),
+    prevent_initial_call=True,
 )
 def sign_out_button_click(
     n_clicks: Optional[int],
@@ -1150,6 +1125,7 @@ def sign_out_button_click(
     Output({'type': 'parameters-editor', 'index': MATCH}, 'value'),
     Input({'type': 'parameters-as-yaml-button', 'index': MATCH}, 'n_clicks'),
     Input({'type': 'parameters-as-json-button', 'index': MATCH}, 'n_clicks'),
+    prevent_initial_call=True,
 )
 def parameters_as_yaml_or_json_click(
     yaml_n_clicks: Optional[int],
@@ -1178,6 +1154,7 @@ def parameters_as_yaml_or_json_click(
     Output({'type': 'sync-editor', 'index': MATCH}, 'value'),
     Input({'type': 'sync-as-json-button', 'index': MATCH}, 'n_clicks'),
     Input({'type': 'sync-as-lines-button', 'index': MATCH}, 'n_clicks'),
+    prevent_initial_call=True,
 )
 def sync_as_json_or_lines_click(
     json_n_clicks: Optional[int],
@@ -1206,6 +1183,7 @@ def sync_as_json_or_lines_click(
     Output('pages-offcanvas', 'children'),
     Input('logo-img', 'n_clicks'),
     State('pages-offcanvas', 'is_open'),
+    prevent_initial_call=True,
 )
 def toggle_pages_offcanvas(n_clicks: Optional[int], is_open: bool):
     """

meerschaum/api/dash/callbacks/jobs.py

@@ -13,12 +13,13 @@ import time
 import traceback
 from datetime import datetime, timezone
 
+from meerschaum.jobs import get_jobs
 from meerschaum.utils.typing import Optional, Dict, Any
 from meerschaum.api import CHECK_UPDATE
 from meerschaum.api.dash import dash_app
 from meerschaum.api.dash.sessions import get_username_from_session
 from meerschaum.utils.packages import attempt_import, import_dcc, import_html
-from meerschaum.api.dash.components import alert_from_success_tuple
+from meerschaum.api.dash.components import alert_from_success_tuple, build_cards_grid
 from meerschaum.api.dash.jobs import (
     build_job_card,
     build_manage_job_buttons_div_children,
@@ -251,17 +252,22 @@ def render_job_page_from_url(
     session_data: Optional[Dict[str, Any]],
 ):
     """
-    Load the `/job/{name}` page.
+    Load the `/dash/jobs/{name}` page.
     """
-    if not str(pathname).startswith('/dash/job'):
+    if not str(pathname).startswith('/dash/jobs'):
         return no_update
 
     session_id = (session_data or {}).get('session-id', None)
     authenticated = is_session_authenticated(str(session_id))
 
-    job_name = pathname.replace('/dash/job', '').lstrip('/').rstrip('/')
+    job_name = pathname.replace('/dash/jobs', '').lstrip('/').rstrip('/')
     if not job_name:
-        return no_update
+        jobs = get_jobs(executor_keys='local', combine_local_and_systemd=True)
+        cards = [
+            build_job_card(job, authenticated=authenticated, include_follow=False)
+            for job in jobs.values()
+        ]
+        return [html.Br(), build_cards_grid(cards, 3), html.Br()]
 
     job = _get_job(job_name)
     if not job.exists():