meerschaum 2.1.7__py3-none-any.whl → 2.2.0__py3-none-any.whl

meerschaum/api/dash/callbacks/jobs.py

@@ -11,6 +11,7 @@ import json
 import functools
 import time
 import traceback
+from datetime import datetime, timezone
 import meerschaum as mrsm
 from meerschaum.utils.typing import Optional, Dict, Any
 from meerschaum.api import get_api_connector, endpoints, CHECK_UPDATE
@@ -54,9 +55,14 @@ def download_job_logs(n_clicks):
     component_dict = json.loads(ctx[0]['prop_id'].split('.' + 'n_clicks')[0])
     daemon_id = component_dict['index']
     daemon = Daemon(daemon_id=daemon_id)
+    now = datetime.now(timezone.utc)
+    filename = (
+        daemon.rotating_log.file_path.name[:(-1 * len('.log'))]
+        + '_' + str(int(now.timestamp())) + '.log'
+    )
     return {
         'content': daemon.log_text,
-        'filename': daemon.rotating_log.file_path.name,
+        'filename': filename,
     }
 
 
@@ -74,7 +80,7 @@ def manage_job_button_click(
         session_data: Optional[Dict[str, Any]] = None,
     ):
     """
-    Start, stop, or pause the given job.
+    Start, stop, pause, or delete the given job.
     """
     if not n_clicks:
         raise PreventUpdate
@@ -98,12 +104,18 @@ def manage_job_button_click(
     component_dict = json.loads(ctx[0]['prop_id'].split('.' + 'n_clicks')[0])
     daemon_id = component_dict['index']
     manage_job_action = component_dict['action']
-    daemon = Daemon(daemon_id=daemon_id)
+    try:
+        daemon = Daemon(daemon_id=daemon_id)
+    except Exception as e:
+        daemon = None
+    if daemon is None:
+        raise PreventUpdate
 
     manage_functions = {
         'start': functools.partial(daemon.run, allow_dirty_run=True),
         'stop': daemon.quit,
         'pause': daemon.pause,
+        'delete': daemon.cleanup,
     }
     if manage_job_action not in manage_functions:
         return (
@@ -135,6 +147,46 @@ def manage_job_button_click(
         build_process_timestamps_children(daemon),
     )
 
+dash_app.clientside_callback(
+    """
+    function(n_clicks_arr, url){
+        display_block = {"display": "block"};
+
+        var clicked = false;
+        for (var i = 0; i < n_clicks_arr.length; i++){
+            if (n_clicks_arr[i]){
+                clicked = true;
+                break;
+            }
+        }
+
+        if (!clicked){
+            return dash_clientside.no_update;
+        }
+
+        const triggered_id = dash_clientside.callback_context.triggered_id;
+        const job_daemon_id = triggered_id["index"];
+
+        iframe = document.getElementById('webterm-iframe');
+        if (!iframe){ return dash_clientside.no_update; }
+
+        iframe.contentWindow.postMessage(
+            {
+               action: "show",
+               subaction: "logs",
+               subaction_text: job_daemon_id,
+            },
+            url
+        );
+        dash_clientside.set_props("webterm-div", {"style": display_block});
+        return [];
+    }
+    """,
+    Output('content-div-right', 'children'),
+    Input({'type': 'follow-logs-button', 'index': ALL}, 'n_clicks'),
+    State('location', 'href'),
+)
+
 
 @dash_app.callback(
     Output({'type': 'manage-job-buttons-div', 'index': ALL}, 'children'),

meerschaum/api/dash/jobs.py

@@ -49,6 +49,17 @@ def get_jobs_cards(state: WebState):
             build_process_timestamps_children(d),
             id = {'type': 'process-timestamps-div', 'index': d.daemon_id},
         )
+        follow_logs_button = dbc.DropdownMenuItem(
+            "Follow logs",
+            id = {'type': 'follow-logs-button', 'index': d.daemon_id},
+        )
+        download_logs_button = dbc.DropdownMenuItem(
+            "Download logs",
+            id = {'type': 'job-download-logs-button', 'index': d.daemon_id},
+        )
+        logs_menu_children = (
+            ([follow_logs_button] if is_authenticated else []) + [download_logs_button]
+        )
         header_children = [
             html.Div(
                 build_status_children(d),
@@ -56,11 +67,13 @@ def get_jobs_cards(state: WebState):
                 style = {'float': 'left'},
             ),
             html.Div(
-                dbc.Button(
-                    'Download logs',
-                    size = 'sm',
-                    color = 'link',
-                    id = {'type': 'job-download-logs-button', 'index': d.daemon_id},
+                dbc.DropdownMenu(
+                    logs_menu_children,
+                    label = "Logs",
+                    size = "sm",
+                    align_end = True,
+                    color = "secondary",
+                    menu_variant = 'dark',
                 ),
                 style = {'float': 'right'},
             ),
@@ -74,7 +87,7 @@ def get_jobs_cards(state: WebState):
                     className = "card-text job-card-text", 
                     style = {"word-wrap": "break-word"}
                 ),
-                style={"white-space": "pre-wrap"},
+                style = {"white-space": "pre-wrap"},
             ),
             html.Div(
                 (
@@ -82,7 +95,7 @@ def get_jobs_cards(state: WebState):
                     if is_authenticated
                     else []
                 ),
-                id={'type': 'manage-job-buttons-div', 'index': d.daemon_id}
+                id = {'type': 'manage-job-buttons-div', 'index': d.daemon_id},
             ),
             html.Div(id={'type': 'manage-job-alert-div', 'index': d.daemon_id}),
         ]
@@ -108,7 +121,7 @@ def build_manage_job_buttons_div_children(daemon: Daemon):
     return [
         html.Br(),
         dbc.Row([
-            dbc.Col(button, width=4)
+            dbc.Col(button, width=6)
             for button in buttons
         ])
     ]
@@ -153,9 +166,22 @@ def build_manage_job_buttons(daemon: Daemon):
             'index': daemon.daemon_id,
         },
     )
+    delete_button = dbc.Button(
+        'Delete',
+        size = 'sm',
+        color = 'danger',
+        style = {'width': '100%'},
+        id = {
+            'type': 'manage-job-button',
+            'action': 'delete',
+            'index': daemon.daemon_id,
+        },
+    )
     buttons = []
     if daemon.status in ('stopped', 'paused'):
         buttons.append(start_button)
+    if daemon.status == 'stopped':
+        buttons.append(delete_button)
     if daemon.status in ('running',):
         buttons.append(pause_button)
     if daemon.status in ('running', 'paused'):

meerschaum/api/dash/keys.py

@@ -108,7 +108,7 @@ action_dropdown_row = html.Div(
                             id = 'flags-dropdown',
                             multi = True,
                             placeholder = 'Boolean flags',
-                            options = [],
+                            options = ['--yes'],
                             value = ['--yes'],
                         ),
                         id = 'flags-dropdown-div',

meerschaum/api/dash/pages/dashboard.py

@@ -11,12 +11,22 @@ from meerschaum.config import __doc__ as doc, get_config
 from meerschaum.utils.misc import get_connector_labels
 from meerschaum.utils.packages import attempt_import, import_html, import_dcc, import_pandas
 from meerschaum.api import endpoints, CHECK_UPDATE
-dex = attempt_import('dash_extensions', lazy=False, check_update=CHECK_UPDATE)
-dbc = attempt_import('dash_bootstrap_components', lazy=False, check_update=CHECK_UPDATE)
+(
+    dex,
+    px,
+    daq,
+    dbc,
+) = attempt_import(
+    'dash_extensions',
+    'plotly.express',
+    'dash_daq',
+    'dash_bootstrap_components',
+    lazy = False,
+    warn = False,
+    check_update = CHECK_UPDATE,
+)
 html, dcc = import_html(check_update=CHECK_UPDATE), import_dcc(check_update=CHECK_UPDATE)
 pd = import_pandas(check_update=CHECK_UPDATE)
-px = attempt_import('plotly.express', warn=False, check_update=CHECK_UPDATE)
-daq = attempt_import('dash_daq', warn=False, check_update=CHECK_UPDATE)
 
 from meerschaum.api.dash.components import (
     go_button,

meerschaum/api/dash/pipes.py

@@ -24,6 +24,7 @@ from meerschaum.api.dash import (
 from meerschaum.api.dash.connectors import get_web_connector
 from meerschaum.api.dash.components import alert_from_success_tuple
 from meerschaum.api.dash.users import is_session_authenticated
+from meerschaum.config import get_config
 import meerschaum as mrsm
 dbc = attempt_import('dash_bootstrap_components', lazy=False, check_update=CHECK_UPDATE)
 dash_ace = attempt_import('dash_ace', lazy=False, check_update=CHECK_UPDATE)
@@ -110,43 +111,127 @@ def get_pipes_cards(*keys, session_data: Optional[Dict[str, Any]] = None):
     session_id = (session_data or {}).get('session-id', None)
     authenticated = is_session_authenticated(str(session_id))
 
-    _pipes = pipes_from_state(*keys, as_list=True)
-    alerts = [alert_from_success_tuple(_pipes)]
-    if not isinstance(_pipes, list):
-        _pipes = []
-    for p in _pipes:
-        footer_children = dbc.Row([
-            dbc.Col(
-                dbc.Button(
-                    'Download recent data',
-                    size = 'sm',
-                    color = 'link',
-                    id = {'type': 'pipe-download-csv-button', 'index': json.dumps(p.meta)},
-                )
-            ),
-        ])
+    pipes = pipes_from_state(*keys, as_list=True)
+    alerts = [alert_from_success_tuple(pipes)]
+    if not isinstance(pipes, list):
+        pipes = []
+
+    max_num_pipes_cards = get_config('dash', 'max_num_pipes_cards')
+    overflow_pipes = pipes[max_num_pipes_cards:]
+
+    for pipe in pipes[:max_num_pipes_cards]:
+        meta_str = json.dumps(pipe.meta)
+        footer_children = dbc.Row(
+            [
+                dbc.Col(
+                    (
+                        dbc.DropdownMenu(
+                            label = "Manage",
+                            children = [
+                                dbc.DropdownMenuItem(
+                                    'Delete',
+                                    id = {
+                                        'type': 'manage-pipe-button',
+                                        'index': meta_str,
+                                        'action': 'delete',
+                                    },
+                                ),
+                                dbc.DropdownMenuItem(
+                                    'Drop',
+                                    id = {
+                                        'type': 'manage-pipe-button',
+                                        'index': meta_str,
+                                        'action': 'drop',
+                                    },
+                                ),
+                                dbc.DropdownMenuItem(
+                                    'Clear',
+                                    id = {
+                                        'type': 'manage-pipe-button',
+                                        'index': meta_str,
+                                        'action': 'clear',
+                                    },
+                                ),
+                                dbc.DropdownMenuItem(
+                                    'Verify',
+                                    id = {
+                                        'type': 'manage-pipe-button',
+                                        'index': meta_str,
+                                        'action': 'verify',
+                                    },
+                                ),
+                                dbc.DropdownMenuItem(
+                                    'Sync',
+                                    id = {
+                                        'type': 'manage-pipe-button',
+                                        'index': meta_str,
+                                        'action': 'sync',
+                                    },
+                                ),
+                            ],
+                            direction = "up",
+                            menu_variant = "dark",
+                            size = 'sm',
+                            color = 'secondary',
+                        )
+                    ) if authenticated else [],
+                    width = 2,
+                ),
+                dbc.Col(width=6),
+                dbc.Col(
+                    dbc.Button(
+                        'Download CSV',
+                        size = 'sm',
+                        color = 'link',
+                        style = {'float': 'right'},
+                        id = {'type': 'pipe-download-csv-button', 'index': meta_str},
+                    ),
+                    width = 4,
+                ),
+            ],
+            justify = 'start',
+        )
         card_body_children = [
             html.H5(
-                html.B(str(p)),
+                html.B(str(pipe)),
                 className = 'card-title',
                 style = {'font-family': ['monospace']}
             ),
             html.Div(
                 dbc.Accordion(
-                    accordion_items_from_pipe(p, authenticated=authenticated),
+                    accordion_items_from_pipe(pipe, authenticated=authenticated),
                     flush = True,
                     start_collapsed = True,
-                    id = {'type': 'pipe-accordion', 'index': json.dumps(p.meta)},
+                    id = {'type': 'pipe-accordion', 'index': meta_str},
                 )
             )
 
         ]
         cards.append(
-            dbc.Card(children=[
+            dbc.Card([
                 dbc.CardBody(children=card_body_children),
                 dbc.CardFooter(children=footer_children),
             ])
         )
+
+    if overflow_pipes:
+        cards.append(
+            dbc.Card([
+                dbc.CardBody(
+                    html.Ul(
+                        [
+                            html.Li(html.H5(
+                                html.B(str(pipe)),
+                                className = 'card-title',
+                                style = {'font-family': ['monospace']}
+                            ))
+                            for pipe in overflow_pipes
+                        ]
+                    )
+                )
+            ])
+        )
+
     return cards, alerts
 
 
@@ -188,14 +273,40 @@ def accordion_items_from_pipe(
         overview_header = [html.Thead(html.Tr([html.Th("Attribute"), html.Th("Value")]))]
         dt_name, id_name, val_name = pipe.get_columns('datetime', 'id', 'value', error=False)
         overview_rows = [
-            html.Tr([html.Td("Connector"), html.Td(f"{pipe.connector_keys}")]),
-            html.Tr([html.Td("Metric"), html.Td(f"{pipe.metric_key}")]),
-            html.Tr([html.Td("Location"), html.Td(f"{pipe.location_key}")]),
-            html.Tr([html.Td("Instance"), html.Td(f"{pipe.instance_keys}")]),
-            html.Tr([html.Td("Target Table"), html.Td(f"{pipe.target}")]),
+            html.Tr([html.Td("Connector"), html.Td(html.Pre(f"{pipe.connector_keys}"))]),
+            html.Tr([html.Td("Metric"), html.Td(html.Pre(f"{pipe.metric_key}"))]),
+            html.Tr([html.Td("Location"), html.Td(html.Pre(f"{pipe.location_key}"))]),
+            html.Tr([html.Td("Instance"), html.Td(html.Pre(f"{pipe.instance_keys}"))]),
+            html.Tr([html.Td("Target Table"), html.Td(html.Pre(f"{pipe.target}"))]),
         ]
-        for col_key, col in pipe.columns.items():
-            overview_rows.append(html.Tr([html.Td(f"'{col_key}' Index"), html.Td(col)]))
+        columns = pipe.columns.copy()
+        if columns:
+            datetime_index = columns.pop('datetime', None)
+            columns_items = []
+            if datetime_index:
+                columns_items.append(html.Li(f"{datetime_index} (datetime)"))
+            columns_items.extend([
+                html.Li(f"{col}")
+                for col_key, col in columns.items()
+            ])
+            overview_rows.append(
+                html.Tr([
+                    html.Td("Indices" if len(columns_items) != 1 else "Index"),
+                    html.Td(html.Pre(html.Ul(columns_items))),
+                ])
+            )
+        tags = pipe.tags
+        if tags:
+            tags_items = html.Ul([
+                html.Li(tag)
+                for tag in tags
+            ])
+            overview_rows.append(
+                html.Tr([
+                    html.Td("Tags"),
+                    html.Td(html.Pre(tags_items)),
+                ])
+            )
 
         items_bodies['overview'] = dbc.Table(
             overview_header + [html.Tbody(overview_rows)],

meerschaum/api/dash/plugins.py

@@ -32,17 +32,22 @@ def get_plugins_cards(
         desc = get_api_connector().get_plugin_attributes(plugin).get(
             'description', 'No description provided.'
         )
-        desc_textarea_kw = dict(
-            value=desc, readOnly=True, debounce=False, className='plugin-description',
-            draggable=False, wrap='overflow',
-            id={'type': 'description-textarea', 'index': plugin_name},
-        )
+        desc_textarea_kw = {
+            'value': desc,
+            'readOnly': True,
+            'debounce': False,
+            'className': 'plugin-description',
+            'draggable': False,
+            'wrap': 'overflow',
+            'placeholder': "Edit the plugin's description",
+            'id': {'type': 'description-textarea', 'index': plugin_name},
+        }
 
         card_body_children = [html.H4(plugin_name)]
 
         if is_plugin_owner(plugin_name, session_data):
             desc_textarea_kw['readOnly'] = False
-        card_body_children += [dbc.Textarea(**desc_textarea_kw)]
+        card_body_children.append(dbc.Textarea(**desc_textarea_kw))
         if not desc_textarea_kw['readOnly']:
             card_body_children += [
                 dbc.Button(
@@ -53,12 +58,23 @@ def get_plugins_cards(
                 ),
                 html.Div(id={'type': 'edit-alert-div', 'index': plugin_name}),
             ]
-        _plugin_username = get_api_connector().get_plugin_username(plugin, debug=debug)
+        plugin_username = get_api_connector().get_plugin_username(plugin, debug=debug)
+        plugin_version = get_api_connector().get_plugin_version(plugin, debug=debug) or ' '
         card_children = [
-            dbc.CardHeader([html.A('👤 ' + str(_plugin_username), href='#')]),
+            dbc.CardHeader(
+                [
+                    dbc.Row(
+                        [
+                            dbc.Col(html.A('👤 ' + str(plugin_username), href='#')),
+                            dbc.Col(html.Pre(str(plugin_version), style={'text-align': 'right'})),
+                        ],
+                        justify = 'between',
+                    ),
+                ],
+            ),
             dbc.CardBody(card_body_children),
             dbc.CardFooter([
-                html.A('⬇️ Download source', href=(endpoints['plugins'] + '/' + plugin_name))
+                html.A('⬇️ Download', href=(endpoints['plugins'] + '/' + plugin_name))
             ]),
         ]
         cards.append(