meerschaum 2.1.7__py3-none-any.whl → 2.2.0.dev2__py3-none-any.whl

meerschaum/_internal/arguments/_parser.py

@@ -346,6 +346,9 @@ groups['misc'].add_argument(
 groups['misc'].add_argument(
     '--nopretty', action="store_true", help="Print elements without 'pretty' formatting"
 )
+groups['misc'].add_argument(
+    '--skip-deps', action="store_true", help="Skip dependencies when installing plugins.",
+)
 groups['misc'].add_argument(
     '-P', '--params', type=string_to_dict, help=(
         "Parameters dictionary in JSON format or simple format. " +

meerschaum/_internal/entry.py

@@ -1,6 +1,7 @@
 #! /usr/bin/env python
 # -*- coding: utf-8 -*-
 # vim:fenc=utf-8
+# type: ignore
 
 """
 The entry point for launching Meerschaum actions.
@@ -61,7 +62,7 @@ def entry_with_args(
     """
     import sys
     from meerschaum.plugins import Plugin
-    from meerschaum.actions import get_shell, get_action, get_main_action_name
+    from meerschaum.actions import get_action, get_main_action_name
     from meerschaum._internal.arguments import remove_leading_action
     from meerschaum.utils.venv import Venv, active_venvs, deactivate_venv
     if kw.get('trace', None):

meerschaum/actions/install.py

@@ -54,6 +54,7 @@ def _complete_install(
 def _install_plugins(
         action: Optional[List[str]] = None,
         repository: Optional[str] = None,
+        skip_deps: bool = False,
         force: bool = False,
         debug: bool = False,
         **kw: Any
@@ -87,11 +88,14 @@ def _install_plugins(
 
     repo_connector = parse_repo_keys(repository)
 
-    successes = {}
     for name in action:
         info(f"Installing plugin '{name}' from Meerschaum repository '{repo_connector}'...")
-        success, msg = repo_connector.install_plugin(name, force=force, debug=debug)
-        successes[name] = (success, msg)
+        success, msg = repo_connector.install_plugin(
+            name,
+            force = force,
+            skip_deps = skip_deps,
+            debug = debug,
+        )
         print_tuple((success, msg))
 
     reload_plugins(debug=debug)

meerschaum/actions/sync.py

@@ -453,16 +453,20 @@ def _wrap_pipe(
                 return False, msg
         return True, "Success"
 
-    hook_results = []
+    pre_hook_results, post_hook_results = [], []
     def apply_hooks(is_pre_sync: bool):
         _sync_hooks = (_pre_sync_hooks if is_pre_sync else _post_sync_hooks)
+        _hook_results = (pre_hook_results if is_pre_sync else post_hook_results)
         for module_name, sync_hooks in _sync_hooks.items():
             plugin_name = module_name.split('.')[-1] if module_name.startswith('plugins.') else None
             for sync_hook in sync_hooks:
                 hook_result = pool.apply_async(call_sync_hook, (plugin_name, sync_hook))
-                hook_results.append(hook_result)
+                _hook_results.append(hook_result)
 
     apply_hooks(True)
+    for hook_result in pre_hook_results:
+        hook_success, hook_msg = hook_result.get()
+        mrsm.pprint((hook_success, hook_msg))
    
     try:
         with Venv(get_connector_plugin(pipe.connector), debug=debug):
@@ -480,7 +484,7 @@ def _wrap_pipe(
         'sync_complete_timestamp': datetime.now(timezone.utc),
     })
     apply_hooks(False)
-    for hook_result in hook_results:
+    for hook_result in post_hook_results:
         hook_success, hook_msg = hook_result.get()
         mrsm.pprint((hook_success, hook_msg))
 

meerschaum/api/dash/callbacks/dashboard.py

@@ -488,11 +488,11 @@ def update_keys_options(
     """
     ctx = dash.callback_context
     trigger = ctx.triggered[0]['prop_id'].split('.')[0]
+    instance_click = trigger == 'instance-select'
 
     ### Update the instance first.
     update_instance_keys = False
     if not instance_keys:
-        #  instance_keys = get_config('meerschaum', 'web_instance')
         instance_keys = str(get_api_connector())
         update_instance_keys = True
     instance_alerts = []
@@ -516,20 +516,23 @@ def update_keys_options(
     if location_keys:
         num_filter += 1
 
-    _ck_alone, _mk_alone, _lk_alone = False, False, False
-    _ck_filter, _mk_filter, _lk_filter = connector_keys, metric_keys, location_keys
-
-    _ck_alone = connector_keys and num_filter == 1
-    _mk_alone = metric_keys and num_filter == 1
-    _lk_alone = location_keys and num_filter == 1
+    _ck_filter = connector_keys
+    _mk_filter = metric_keys
+    _lk_filter = location_keys
+    _ck_alone = (connector_keys and num_filter == 1) or instance_click
+    _mk_alone = (metric_keys and num_filter == 1) or instance_click
+    _lk_alone = (location_keys and num_filter == 1) or instance_click
 
     from meerschaum.utils import fetch_pipes_keys
 
     try:
         _all_keys = fetch_pipes_keys('registered', get_web_connector(ctx.states))
         _keys = fetch_pipes_keys(
-            'registered', get_web_connector(ctx.states),
-            connector_keys=_ck_filter, metric_keys=_mk_filter, location_keys=_lk_filter
+            'registered',
+            get_web_connector(ctx.states),
+            connector_keys = _ck_filter,
+            metric_keys = _mk_filter,
+            location_keys = _lk_filter,
         )
     except Exception as e:
         instance_alerts += [alert_from_success_tuple((False, str(e)))]
@@ -545,15 +548,39 @@ def update_keys_options(
             k = locals()[key_type]
             if k not in _seen_keys[key_type]:
                 _k = 'None' if k in (None, '[None]', 'None', 'null') else k
-                options.append({'label' : _k, 'value' : _k})
+                options.append({'label': _k, 'value': _k})
                 _seen_keys[key_type].add(k)
 
     add_options(_connectors_options, _all_keys if _ck_alone else _keys, 'ck')
     add_options(_metrics_options, _all_keys if _mk_alone else _keys, 'mk')
     add_options(_locations_options, _all_keys if _lk_alone else _keys, 'lk')
-    connector_keys = [ck for ck in connector_keys if ck in [_ck['value'] for _ck in _connectors_options]]
-    metric_keys = [mk for mk in metric_keys if mk in [_mk['value'] for _mk in _metrics_options]]
-    location_keys = [lk for lk in location_keys if lk in [_lk['value'] for _lk in _locations_options]]
+    _connectors_options.sort(key=lambda x: str(x).lower())
+    _metrics_options.sort(key=lambda x: str(x).lower())
+    _locations_options.sort(key=lambda x: str(x).lower())
+    connector_keys = [
+        ck
+        for ck in connector_keys
+        if ck in [
+            _ck['value']
+            for _ck in _connectors_options
+        ]
+    ]
+    metric_keys = [
+        mk
+        for mk in metric_keys
+        if mk in [
+            _mk['value']
+            for _mk in _metrics_options
+        ]
+    ]
+    location_keys = [
+        lk
+        for lk in location_keys
+        if lk in [
+            _lk['value']
+            for _lk in _locations_options
+        ]
+    ]
     _connectors_datalist = [html.Option(value=o['value']) for o in _connectors_options]
     _metrics_datalist = [html.Option(value=o['value']) for o in _metrics_options]
     _locations_datalist = [html.Option(value=o['value']) for o in _locations_options]
@@ -680,6 +707,9 @@ dash_app.clientside_callback(
     Input({'type': 'pipe-download-csv-button', 'index': ALL}, 'n_clicks'),
 )
 def download_pipe_csv(n_clicks):
+    """
+    Download the most recent chunk as a CSV file.
+    """
     if not n_clicks:
         raise PreventUpdate
     ctx = dash.callback_context.triggered
@@ -818,6 +848,51 @@ def sync_documents_click(n_clicks, sync_editor_text):
     return alert_from_success_tuple((success, msg))
 
 
+dash_app.clientside_callback(
+    """
+    function(n_clicks_arr, url){
+        display_block = {"display": "block"};
+
+        var clicked = false;
+        for (var i = 0; i < n_clicks_arr.length; i++){
+            if (n_clicks_arr[i]){
+                clicked = true;
+                break;
+            }
+        }
+        if (!clicked){ return dash_clientside.no_update; }
+
+        const triggered_id = dash_clientside.callback_context.triggered_id;
+        const action = triggered_id["action"];
+        const pipe_meta = JSON.parse(triggered_id["index"]);
+
+        iframe = document.getElementById('webterm-iframe');
+        if (!iframe){ return dash_clientside.no_update; }
+        var location = pipe_meta.location;
+        if (!pipe_meta.location){
+            location = "None";
+        }
+
+        iframe.contentWindow.postMessage(
+            {
+                action: action,
+                subaction: "pipes",
+                connector_keys: [pipe_meta.connector],
+                metric_keys: [pipe_meta.metric],
+                location_keys: [location],
+                instance: pipe_meta.instance,
+            },
+            url
+        );
+        dash_clientside.set_props("webterm-div", {"style": display_block});
+        return [];
+    }
+    """,
+    Output('content-div-right', 'children'),
+    Input({'type': 'manage-pipe-button', 'index': ALL, 'action': ALL}, 'n_clicks'),
+    State('location', 'href'),
+)
+
 @dash_app.callback(
     Output("navbar-collapse", "is_open"),
     [Input("navbar-toggler", "n_clicks")],

meerschaum/api/dash/callbacks/jobs.py

@@ -11,6 +11,7 @@ import json
 import functools
 import time
 import traceback
+from datetime import datetime, timezone
 import meerschaum as mrsm
 from meerschaum.utils.typing import Optional, Dict, Any
 from meerschaum.api import get_api_connector, endpoints, CHECK_UPDATE
@@ -54,9 +55,14 @@ def download_job_logs(n_clicks):
     component_dict = json.loads(ctx[0]['prop_id'].split('.' + 'n_clicks')[0])
     daemon_id = component_dict['index']
     daemon = Daemon(daemon_id=daemon_id)
+    now = datetime.now(timezone.utc)
+    filename = (
+        daemon.rotating_log.file_path.name[:(-1 * len('.log'))]
+        + '_' + str(int(now.timestamp())) + '.log'
+    )
     return {
         'content': daemon.log_text,
-        'filename': daemon.rotating_log.file_path.name,
+        'filename': filename,
     }
 
 
@@ -74,7 +80,7 @@ def manage_job_button_click(
         session_data: Optional[Dict[str, Any]] = None,
     ):
     """
-    Start, stop, or pause the given job.
+    Start, stop, pause, or delete the given job.
     """
     if not n_clicks:
         raise PreventUpdate
@@ -98,12 +104,18 @@ def manage_job_button_click(
     component_dict = json.loads(ctx[0]['prop_id'].split('.' + 'n_clicks')[0])
     daemon_id = component_dict['index']
     manage_job_action = component_dict['action']
-    daemon = Daemon(daemon_id=daemon_id)
+    try:
+        daemon = Daemon(daemon_id=daemon_id)
+    except Exception as e:
+        daemon = None
+    if daemon is None:
+        raise PreventUpdate
 
     manage_functions = {
         'start': functools.partial(daemon.run, allow_dirty_run=True),
         'stop': daemon.quit,
         'pause': daemon.pause,
+        'delete': daemon.cleanup,
     }
     if manage_job_action not in manage_functions:
         return (
@@ -135,6 +147,46 @@ def manage_job_button_click(
         build_process_timestamps_children(daemon),
     )
 
+dash_app.clientside_callback(
+    """
+    function(n_clicks_arr, url){
+        display_block = {"display": "block"};
+
+        var clicked = false;
+        for (var i = 0; i < n_clicks_arr.length; i++){
+            if (n_clicks_arr[i]){
+                clicked = true;
+                break;
+            }
+        }
+
+        if (!clicked){
+            return dash_clientside.no_update;
+        }
+
+        const triggered_id = dash_clientside.callback_context.triggered_id;
+        const job_daemon_id = triggered_id["index"];
+
+        iframe = document.getElementById('webterm-iframe');
+        if (!iframe){ return dash_clientside.no_update; }
+
+        iframe.contentWindow.postMessage(
+            {
+               action: "show",
+               subaction: "logs",
+               subaction_text: job_daemon_id,
+            },
+            url
+        );
+        dash_clientside.set_props("webterm-div", {"style": display_block});
+        return [];
+    }
+    """,
+    Output('content-div-right', 'children'),
+    Input({'type': 'follow-logs-button', 'index': ALL}, 'n_clicks'),
+    State('location', 'href'),
+)
+
 
 @dash_app.callback(
     Output({'type': 'manage-job-buttons-div', 'index': ALL}, 'children'),

meerschaum/api/dash/jobs.py

@@ -49,6 +49,17 @@ def get_jobs_cards(state: WebState):
             build_process_timestamps_children(d),
             id = {'type': 'process-timestamps-div', 'index': d.daemon_id},
         )
+        follow_logs_button = dbc.DropdownMenuItem(
+            "Follow logs",
+            id = {'type': 'follow-logs-button', 'index': d.daemon_id},
+        )
+        download_logs_button = dbc.DropdownMenuItem(
+            "Download logs",
+            id = {'type': 'job-download-logs-button', 'index': d.daemon_id},
+        )
+        logs_menu_children = (
+            ([follow_logs_button] if is_authenticated else []) + [download_logs_button]
+        )
         header_children = [
             html.Div(
                 build_status_children(d),
@@ -56,11 +67,13 @@ def get_jobs_cards(state: WebState):
                 style = {'float': 'left'},
             ),
             html.Div(
-                dbc.Button(
-                    'Download logs',
-                    size = 'sm',
-                    color = 'link',
-                    id = {'type': 'job-download-logs-button', 'index': d.daemon_id},
+                dbc.DropdownMenu(
+                    logs_menu_children,
+                    label = "Logs",
+                    size = "sm",
+                    align_end = True,
+                    color = "secondary",
+                    menu_variant = 'dark',
                 ),
                 style = {'float': 'right'},
             ),
@@ -74,7 +87,7 @@ def get_jobs_cards(state: WebState):
                     className = "card-text job-card-text", 
                     style = {"word-wrap": "break-word"}
                 ),
-                style={"white-space": "pre-wrap"},
+                style = {"white-space": "pre-wrap"},
             ),
             html.Div(
                 (
@@ -82,7 +95,7 @@ def get_jobs_cards(state: WebState):
                     if is_authenticated
                     else []
                 ),
-                id={'type': 'manage-job-buttons-div', 'index': d.daemon_id}
+                id = {'type': 'manage-job-buttons-div', 'index': d.daemon_id},
             ),
             html.Div(id={'type': 'manage-job-alert-div', 'index': d.daemon_id}),
         ]
@@ -108,7 +121,7 @@ def build_manage_job_buttons_div_children(daemon: Daemon):
     return [
         html.Br(),
         dbc.Row([
-            dbc.Col(button, width=4)
+            dbc.Col(button, width=6)
             for button in buttons
         ])
     ]
@@ -153,9 +166,22 @@ def build_manage_job_buttons(daemon: Daemon):
             'index': daemon.daemon_id,
         },
     )
+    delete_button = dbc.Button(
+        'Delete',
+        size = 'sm',
+        color = 'danger',
+        style = {'width': '100%'},
+        id = {
+            'type': 'manage-job-button',
+            'action': 'delete',
+            'index': daemon.daemon_id,
+        },
+    )
     buttons = []
     if daemon.status in ('stopped', 'paused'):
         buttons.append(start_button)
+    if daemon.status == 'stopped':
+        buttons.append(delete_button)
     if daemon.status in ('running',):
         buttons.append(pause_button)
     if daemon.status in ('running', 'paused'):

meerschaum/api/dash/pipes.py

@@ -115,16 +115,77 @@ def get_pipes_cards(*keys, session_data: Optional[Dict[str, Any]] = None):
     if not isinstance(_pipes, list):
         _pipes = []
     for p in _pipes:
-        footer_children = dbc.Row([
-            dbc.Col(
-                dbc.Button(
-                    'Download recent data',
-                    size = 'sm',
-                    color = 'link',
-                    id = {'type': 'pipe-download-csv-button', 'index': json.dumps(p.meta)},
-                )
-            ),
-        ])
+        meta_str = json.dumps(p.meta)
+        footer_children = dbc.Row(
+            [
+                dbc.Col(
+                    (
+                        dbc.DropdownMenu(
+                            label = "Manage",
+                            children = [
+                                dbc.DropdownMenuItem(
+                                    'Delete',
+                                    id = {
+                                        'type': 'manage-pipe-button',
+                                        'index': meta_str,
+                                        'action': 'delete',
+                                    },
+                                ),
+                                dbc.DropdownMenuItem(
+                                    'Drop',
+                                    id = {
+                                        'type': 'manage-pipe-button',
+                                        'index': meta_str,
+                                        'action': 'drop',
+                                    },
+                                ),
+                                dbc.DropdownMenuItem(
+                                    'Clear',
+                                    id = {
+                                        'type': 'manage-pipe-button',
+                                        'index': meta_str,
+                                        'action': 'clear',
+                                    },
+                                ),
+                                dbc.DropdownMenuItem(
+                                    'Verify',
+                                    id = {
+                                        'type': 'manage-pipe-button',
+                                        'index': meta_str,
+                                        'action': 'verify',
+                                    },
+                                ),
+                                dbc.DropdownMenuItem(
+                                    'Sync',
+                                    id = {
+                                        'type': 'manage-pipe-button',
+                                        'index': meta_str,
+                                        'action': 'sync',
+                                    },
+                                ),
+                            ],
+                            direction = "up",
+                            menu_variant = "dark",
+                            size = 'sm',
+                            color = 'secondary',
+                        )
+                    ) if authenticated else [],
+                    width = 2,
+                ),
+                dbc.Col(width=6),
+                dbc.Col(
+                    dbc.Button(
+                        'Download CSV',
+                        size = 'sm',
+                        color = 'link',
+                        style = {'float': 'right'},
+                        id = {'type': 'pipe-download-csv-button', 'index': meta_str},
+                    ),
+                    width = 4,
+                ),
+            ],
+            justify = 'start',
+        )
         card_body_children = [
             html.H5(
                 html.B(str(p)),
@@ -136,7 +197,7 @@ def get_pipes_cards(*keys, session_data: Optional[Dict[str, Any]] = None):
                     accordion_items_from_pipe(p, authenticated=authenticated),
                     flush = True,
                     start_collapsed = True,
-                    id = {'type': 'pipe-accordion', 'index': json.dumps(p.meta)},
+                    id = {'type': 'pipe-accordion', 'index': meta_str},
                 )
             )
 
@@ -188,14 +249,40 @@ def accordion_items_from_pipe(
         overview_header = [html.Thead(html.Tr([html.Th("Attribute"), html.Th("Value")]))]
         dt_name, id_name, val_name = pipe.get_columns('datetime', 'id', 'value', error=False)
         overview_rows = [
-            html.Tr([html.Td("Connector"), html.Td(f"{pipe.connector_keys}")]),
-            html.Tr([html.Td("Metric"), html.Td(f"{pipe.metric_key}")]),
-            html.Tr([html.Td("Location"), html.Td(f"{pipe.location_key}")]),
-            html.Tr([html.Td("Instance"), html.Td(f"{pipe.instance_keys}")]),
-            html.Tr([html.Td("Target Table"), html.Td(f"{pipe.target}")]),
+            html.Tr([html.Td("Connector"), html.Td(html.Pre(f"{pipe.connector_keys}"))]),
+            html.Tr([html.Td("Metric"), html.Td(html.Pre(f"{pipe.metric_key}"))]),
+            html.Tr([html.Td("Location"), html.Td(html.Pre(f"{pipe.location_key}"))]),
+            html.Tr([html.Td("Instance"), html.Td(html.Pre(f"{pipe.instance_keys}"))]),
+            html.Tr([html.Td("Target Table"), html.Td(html.Pre(f"{pipe.target}"))]),
         ]
-        for col_key, col in pipe.columns.items():
-            overview_rows.append(html.Tr([html.Td(f"'{col_key}' Index"), html.Td(col)]))
+        columns = pipe.columns.copy()
+        if columns:
+            datetime_index = columns.pop('datetime', None)
+            columns_items = []
+            if datetime_index:
+                columns_items.append(html.Li(f"{datetime_index} (datetime)"))
+            columns_items.extend([
+                html.Li(f"{col}")
+                for col_key, col in columns.items()
+            ])
+            overview_rows.append(
+                html.Tr([
+                    html.Td("Indices" if len(columns_items) != 1 else "Index"),
+                    html.Td(html.Pre(html.Ul(columns_items))),
+                ])
+            )
+        tags = pipe.tags
+        if tags:
+            tags_items = html.Ul([
+                html.Li(tag)
+                for tag in tags
+            ])
+            overview_rows.append(
+                html.Tr([
+                    html.Td("Tags"),
+                    html.Td(html.Pre(tags_items)),
+                ])
+            )
 
         items_bodies['overview'] = dbc.Table(
             overview_header + [html.Tbody(overview_rows)],