meerschaum 2.1.7__py3-none-any.whl → 2.2.0__py3-none-any.whl

meerschaum/__main__.py

@@ -4,7 +4,7 @@
 # vim:fenc=utf-8
 
 """
-Copyright 2021 Bennett Meares
+Copyright 2024 Bennett Meares
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

meerschaum/_internal/arguments/_parser.py

@@ -346,6 +346,9 @@ groups['misc'].add_argument(
 groups['misc'].add_argument(
     '--nopretty', action="store_true", help="Print elements without 'pretty' formatting"
 )
+groups['misc'].add_argument(
+    '--skip-deps', action="store_true", help="Skip dependencies when installing plugins.",
+)
 groups['misc'].add_argument(
     '-P', '--params', type=string_to_dict, help=(
         "Parameters dictionary in JSON format or simple format. " +

meerschaum/_internal/entry.py

@@ -1,6 +1,7 @@
 #! /usr/bin/env python
 # -*- coding: utf-8 -*-
 # vim:fenc=utf-8
+# type: ignore
 
 """
 The entry point for launching Meerschaum actions.
@@ -48,7 +49,7 @@ def entry(sysargs: Optional[List[str]] = None) -> SuccessTuple:
 
     if args.get('schedule', None):
         from meerschaum.utils.schedule import schedule_function
-        return schedule_function(entry_with_args, args['schedule'], **args)
+        return schedule_function(entry_with_args, **args)
     return entry_with_args(**args)
 
 
@@ -61,7 +62,7 @@ def entry_with_args(
     """
     import sys
     from meerschaum.plugins import Plugin
-    from meerschaum.actions import get_shell, get_action, get_main_action_name
+    from meerschaum.actions import get_action, get_main_action_name
     from meerschaum._internal.arguments import remove_leading_action
     from meerschaum.utils.venv import Venv, active_venvs, deactivate_venv
     if kw.get('trace', None):

meerschaum/actions/install.py

@@ -54,6 +54,7 @@ def _complete_install(
 def _install_plugins(
         action: Optional[List[str]] = None,
         repository: Optional[str] = None,
+        skip_deps: bool = False,
         force: bool = False,
         debug: bool = False,
         **kw: Any
@@ -87,11 +88,14 @@ def _install_plugins(
 
     repo_connector = parse_repo_keys(repository)
 
-    successes = {}
     for name in action:
         info(f"Installing plugin '{name}' from Meerschaum repository '{repo_connector}'...")
-        success, msg = repo_connector.install_plugin(name, force=force, debug=debug)
-        successes[name] = (success, msg)
+        success, msg = repo_connector.install_plugin(
+            name,
+            force = force,
+            skip_deps = skip_deps,
+            debug = debug,
+        )
         print_tuple((success, msg))
 
     reload_plugins(debug=debug)

meerschaum/actions/show.py

@@ -41,6 +41,7 @@ def show(
         'jobs'       : _show_jobs,
         'logs'       : _show_logs,
         'tags'       : _show_tags,
+        'schedules'  : _show_schedules,
     }
     return choose_subaction(action, show_options, **kw)
 
@@ -577,6 +578,7 @@ def _show_logs(
         `show logs myjob myotherjob`
     """
     import os, pathlib, random, asyncio
+    from datetime import datetime, timezone
     from meerschaum.utils.packages import attempt_import, import_rich
     from meerschaum.utils.daemon import get_filtered_daemons, Daemon
     from meerschaum.utils.warnings import warn, info
@@ -587,72 +589,106 @@ def _show_logs(
     if not ANSI:
         info = print
     colors = get_config('jobs', 'logs', 'colors')
+    timestamp_format = get_config('jobs', 'logs', 'timestamps', 'format')
+    follow_timestamp_format = get_config('jobs', 'logs', 'timestamps', 'follow_format')
     daemons = get_filtered_daemons(action)
-
-    def _build_buffer_spaces(daemons) -> Dict[str, str]:
-        _max_len_id = max([len(d.daemon_id) for d in daemons]) if daemons else 0
-        _buffer_len = max(get_config('jobs', 'logs', 'min_buffer_len'), _max_len_id + 2)
+    now = datetime.now(timezone.utc)
+    now_str = now.strftime(timestamp_format)
+    now_follow_str = now.strftime(follow_timestamp_format)
+
+    def build_buffer_spaces(daemons) -> Dict[str, str]:
+        max_len_id = max(len(d.daemon_id) for d in daemons) if daemons else 0
+        buffer_len = max(
+            get_config('jobs', 'logs', 'min_buffer_len'),
+            max_len_id 
+        )
         return {
-            d.daemon_id: ''.join([' ' for i in range(_buffer_len - len(d.daemon_id))])
+            d.daemon_id: ''.join([' ' for i in range(buffer_len - len(d.daemon_id))])
             for d in daemons
         }
 
-    def _build_job_colors(daemons, _old_job_colors = None) -> Dict[str, str]:
+    def build_job_colors(daemons, _old_job_colors = None) -> Dict[str, str]:
         return {d.daemon_id: colors[i % len(colors)] for i, d in enumerate(daemons)}
 
-    _buffer_spaces = _build_buffer_spaces(daemons)
-    _job_colors = _build_job_colors(daemons)
+    buffer_spaces = build_buffer_spaces(daemons)
+    job_colors = build_job_colors(daemons)
 
-    def _get_buffer_spaces(daemon_id):
-        nonlocal _buffer_spaces, daemons
-        if daemon_id not in _buffer_spaces:
+    def get_buffer_spaces(daemon_id):
+        nonlocal buffer_spaces, daemons
+        if daemon_id not in buffer_spaces:
             d = Daemon(daemon_id=daemon_id)
             if d not in daemons:
                 daemons = get_filtered_daemons(action)
-            _buffer_spaces = _build_buffer_spaces(daemons)
-        return _buffer_spaces[daemon_id]
+            buffer_spaces = build_buffer_spaces(daemons)
+        return buffer_spaces[daemon_id] or ' '
 
-    def _get_job_colors(daemon_id):
-        nonlocal _job_colors, daemons
-        if daemon_id not in _job_colors:
+    def get_job_colors(daemon_id):
+        nonlocal job_colors, daemons
+        if daemon_id not in job_colors:
             d = Daemon(daemon_id=daemon_id)
             if d not in daemons:
                 daemons = get_filtered_daemons(action)
-            _job_colors = _build_job_colors(daemons)
-        return _job_colors[daemon_id]
+            job_colors = build_job_colors(daemons)
+        return job_colors[daemon_id]
 
-    def _follow_pretty_print():
-        watchgod = attempt_import('watchgod')
+    def follow_pretty_print():
+        watchfiles = attempt_import('watchfiles')
         rich = import_rich()
         rich_text = attempt_import('rich.text')
-        _watch_daemon_ids = {d.daemon_id: d for d in daemons}
+        watch_daemon_ids = {d.daemon_id: d for d in daemons}
         info("Watching log files...")
 
-        def _print_job_line(daemon, line):
+        previous_line_timestamp = None
+        def print_job_line(daemon, line):
+            nonlocal previous_line_timestamp
+            date_prefix_str = line[:len(now_str)]
+            try:
+                line_timestamp = datetime.strptime(date_prefix_str, timestamp_format)
+                previous_line_timestamp = line_timestamp
+            except Exception as e:
+                line_timestamp = None
+            if line_timestamp:
+                line = line[(len(now_str) + 3):]
+            else:
+                line_timestamp = previous_line_timestamp
+
+            if len(line) == 0 or line == '\n':
+                return
+
             text = rich_text.Text(daemon.daemon_id)
-            text.append(
-                _get_buffer_spaces(daemon.daemon_id) + '| '
-                + (line[:-1] if line[-1] == '\n' else line)
+            line_prefix = (
+                get_buffer_spaces(daemon.daemon_id)
+                + (line_timestamp.strftime(follow_timestamp_format) if line_timestamp else '')
+                + ' | '
             )
+            text.append(line_prefix + (line[:-1] if line[-1] == '\n' else line))
             if ANSI:
                 text.stylize(
-                    _get_job_colors(daemon.daemon_id),
+                    get_job_colors(daemon.daemon_id),
                     0,
-                    len(daemon.daemon_id) + len(_get_buffer_spaces(daemon.daemon_id)) + 1
+                    len(daemon.daemon_id) + len(line_prefix)
                 )
             get_console().print(text)
 
 
-        def _print_log_lines(daemon):
+        def print_log_lines(daemon):
             for line in daemon.readlines():
-                _print_job_line(daemon, line)
+                print_job_line(daemon, line)
 
-        def _seek_back_offset(d) -> bool:
+        def seek_back_offset(d) -> bool:
             if d.log_offset_path.exists():
                 d.log_offset_path.unlink()
 
             latest_subfile_path = d.rotating_log.get_latest_subfile_path()
             latest_subfile_index = d.rotating_log.get_latest_subfile_index()
+
+            ### Sometimes the latest file is empty.
+            if os.stat(latest_subfile_path).st_size == 0 and latest_subfile_index > 0:
+                latest_subfile_index -= 1
+                latest_subfile_path = d.rotating_log.get_subfile_path_from_index(
+                    latest_subfile_index
+                )
+
             with open(latest_subfile_path, 'r', encoding='utf-8') as f:
                 latest_lines = f.readlines()
 
@@ -672,12 +708,12 @@ def _show_logs(
             for d in daemons
         }
         for d in daemons:
-            _seek_back_offset(d)
-            _print_log_lines(d)
+            seek_back_offset(d)
+            print_log_lines(d)
 
         _quit = False
-        async def _watch_logs():
-            async for changes in watchgod.awatch(LOGS_RESOURCES_PATH):
+        def watch_logs():
+            for changes in watchfiles.watch(LOGS_RESOURCES_PATH):
                 if _quit:
                     return
                 for change in changes:
@@ -688,7 +724,7 @@ def _show_logs(
                     if not file_path.exists():
                         continue
                     daemon_id = file_path.name.split('.log')[0]
-                    if daemon_id not in _watch_daemon_ids and action:
+                    if daemon_id not in watch_daemon_ids and action:
                         continue
                     try:
                         daemon = Daemon(daemon_id=daemon_id)
@@ -697,22 +733,21 @@ def _show_logs(
                         warn(f"Seeing new logs for non-existent job '{daemon_id}'.", stack=False)
 
                     if daemon is not None:
-                        _print_log_lines(daemon)
+                        print_log_lines(daemon)
 
-        loop = asyncio.new_event_loop()
         try:
-            loop.run_until_complete(_watch_logs())
-        except KeyboardInterrupt:
+            watch_logs()
+        except KeyboardInterrupt as ki:
             _quit = True
 
-    def _print_nopretty_log_text():
+    def print_nopretty_log_text():
         for d in daemons:
             log_text = d.log_text
             print(d.daemon_id)
             print(log_text)
 
-    _print_log_text = _follow_pretty_print if not nopretty else _print_nopretty_log_text
-    _print_log_text()
+    print_log_text = follow_pretty_print if not nopretty else print_nopretty_log_text
+    print_log_text()
 
     return True, "Success"
 
@@ -817,6 +852,57 @@ def _show_tags(
     return True, "Success"
 
 
+def _show_schedules(
+        action: Optional[List[str]] = None,
+        nopretty: bool = False,
+        **kwargs: Any
+    ) -> SuccessTuple:
+    """
+    Print the upcoming timestamps according to the given schedule.
+
+    Examples:
+        show schedule 'daily starting 00:00'
+        show schedule 'every 12 hours and mon-fri starting 2024-01-01'
+    """
+    from meerschaum.utils.schedule import parse_schedule
+    from meerschaum.utils.misc import is_int
+    from meerschaum.utils.formatting import print_options
+    if not action:
+        return False, "Provide a schedule to be parsed."
+    schedule = action[0]
+    default_num_timestamps = 5
+    num_timestamps_str = action[1] if len(action) >= 2 else str(default_num_timestamps)
+    num_timestamps = (
+        int(num_timestamps_str)
+        if is_int(num_timestamps_str)
+        else default_num_timestamps
+    )
+    try:
+        trigger = parse_schedule(schedule)
+    except ValueError as e:
+        return False, str(e)
+
+    next_datetimes = []
+    for _ in range(num_timestamps):
+        try:
+            next_dt = trigger.next()
+            next_datetimes.append(next_dt)
+        except Exception as e:
+            break
+
+    print_options(
+        next_datetimes,
+        num_cols = 1,
+        nopretty = nopretty,
+        header = (
+            f"Next {min(num_timestamps, len(next_datetimes))} timestamps "
+            + f"for schedule '{schedule}':"
+        ),
+    )
+
+    return True, "Success"
+        
+
 
 ### NOTE: This must be the final statement of the module.
 ###       Any subactions added below these lines will not

meerschaum/actions/sync.py

@@ -453,16 +453,20 @@ def _wrap_pipe(
                 return False, msg
         return True, "Success"
 
-    hook_results = []
+    pre_hook_results, post_hook_results = [], []
     def apply_hooks(is_pre_sync: bool):
         _sync_hooks = (_pre_sync_hooks if is_pre_sync else _post_sync_hooks)
+        _hook_results = (pre_hook_results if is_pre_sync else post_hook_results)
         for module_name, sync_hooks in _sync_hooks.items():
             plugin_name = module_name.split('.')[-1] if module_name.startswith('plugins.') else None
             for sync_hook in sync_hooks:
                 hook_result = pool.apply_async(call_sync_hook, (plugin_name, sync_hook))
-                hook_results.append(hook_result)
+                _hook_results.append(hook_result)
 
     apply_hooks(True)
+    for hook_result in pre_hook_results:
+        hook_success, hook_msg = hook_result.get()
+        mrsm.pprint((hook_success, hook_msg))
    
     try:
         with Venv(get_connector_plugin(pipe.connector), debug=debug):
@@ -480,7 +484,7 @@ def _wrap_pipe(
         'sync_complete_timestamp': datetime.now(timezone.utc),
     })
     apply_hooks(False)
-    for hook_result in hook_results:
+    for hook_result in post_hook_results:
         hook_success, hook_msg = hook_result.get()
         mrsm.pprint((hook_success, hook_msg))
 

meerschaum/api/__init__.py

@@ -30,25 +30,35 @@ _locks = {'pipes': RLock(), 'connector': RLock(), 'uvicorn_config': RLock()}
 CHECK_UPDATE = os.environ.get(STATIC_CONFIG['environment']['runtime'], None) != 'docker'
 
 endpoints = STATIC_CONFIG['api']['endpoints']
-aiofiles = attempt_import('aiofiles', lazy=False, check_update=CHECK_UPDATE)
-typing_extensions = attempt_import(
-    'typing_extensions', lazy=False, check_update=CHECK_UPDATE,
-    venv = None,
-)
-pydantic_dataclasses = attempt_import(
-    'pydantic.dataclasses', lazy=False, check_update=CHECK_UPDATE,
+
+(
+    fastapi,
+    aiofiles,
+    starlette_responses,
+    multipart,
+    packaging_version,
+) = attempt_import(
+    'fastapi',
+    'aiofiles',
+    'starlette.responses',
+    'multipart',
+    'packaging.version',
+    lazy = False,
+    check_update = CHECK_UPDATE,
 )
-fastapi = attempt_import('fastapi', lazy=False, check_update=CHECK_UPDATE)
-starlette_reponses = attempt_import(
-    'starlette.responses', warn=False, lazy=False,
-    check_update=CHECK_UPDATE,
+(
+    typing_extensions,
+    uvicorn_workers,
+) = attempt_import(
+    'typing_extensions',
+    'uvicorn.workers',
+    lazy = False,
+    check_update = CHECK_UPDATE,
+    venv = None,
 )
-python_multipart = attempt_import('multipart', lazy=False, check_update=CHECK_UPDATE)
-packaging_version = attempt_import('packaging.version', check_update=CHECK_UPDATE)
 from meerschaum.api._chain import check_allow_chaining, DISALLOW_CHAINING_MESSAGE
 uvicorn_config_path = API_UVICORN_RESOURCES_PATH / SERVER_ID / 'config.json'
 
-uvicorn_workers = attempt_import('uvicorn.workers', venv=None, check_update=CHECK_UPDATE)
 uvicorn_config = None
 sys_config = get_config('system', 'api')
 permissions_config = get_config('system', 'api', 'permissions')

meerschaum/api/_oauth2.py

@@ -7,11 +7,11 @@ Define JWT authorization here.
 """
 
 import os
-from meerschaum.api import app, endpoints
+from meerschaum.api import app, endpoints, CHECK_UPDATE
 from meerschaum.utils.packages import attempt_import
-fastapi = attempt_import('fastapi', lazy=False)
-fastapi_responses = attempt_import('fastapi.responses', lazy=False)
-fastapi_login = attempt_import('fastapi_login')
+fastapi = attempt_import('fastapi', lazy=False, check_update=CHECK_UPDATE)
+fastapi_responses = attempt_import('fastapi.responses', lazy=False, check_update=CHECK_UPDATE)
+fastapi_login = attempt_import('fastapi_login', check_update=CHECK_UPDATE)
 
 LoginManager = fastapi_login.LoginManager
 def generate_secret_key() -> str:

meerschaum/api/dash/callbacks/dashboard.py

@@ -440,18 +440,13 @@ def update_flags(input_flags_dropdown_values, n_clicks, input_flags_texts):
             className = 'input-text',
         )
 
+    remove_index = trigger_dict['index'] if trigger_type == 'input-flags-remove-button' else None
     rows = [
         build_row(i, val, val_text)
         for i, (val, val_text) in enumerate(zip(input_flags_dropdown_values, input_flags_texts))
+        if i != remove_index
     ]
 
-    if trigger_type == 'input-flags-remove-button':
-        remove_index = trigger_dict['index']
-        try:
-            del rows[remove_index]
-        except IndexError:
-            pass
-
     if not rows or input_flags_dropdown_values[-1]:
         rows.append(build_row(len(rows), None, None))
 
@@ -488,11 +483,11 @@ def update_keys_options(
     """
     ctx = dash.callback_context
     trigger = ctx.triggered[0]['prop_id'].split('.')[0]
+    instance_click = trigger == 'instance-select'
 
     ### Update the instance first.
     update_instance_keys = False
     if not instance_keys:
-        #  instance_keys = get_config('meerschaum', 'web_instance')
         instance_keys = str(get_api_connector())
         update_instance_keys = True
     instance_alerts = []
@@ -516,20 +511,23 @@ def update_keys_options(
     if location_keys:
         num_filter += 1
 
-    _ck_alone, _mk_alone, _lk_alone = False, False, False
-    _ck_filter, _mk_filter, _lk_filter = connector_keys, metric_keys, location_keys
-
-    _ck_alone = connector_keys and num_filter == 1
-    _mk_alone = metric_keys and num_filter == 1
-    _lk_alone = location_keys and num_filter == 1
+    _ck_filter = connector_keys
+    _mk_filter = metric_keys
+    _lk_filter = location_keys
+    _ck_alone = (connector_keys and num_filter == 1) or instance_click
+    _mk_alone = (metric_keys and num_filter == 1) or instance_click
+    _lk_alone = (location_keys and num_filter == 1) or instance_click
 
     from meerschaum.utils import fetch_pipes_keys
 
     try:
         _all_keys = fetch_pipes_keys('registered', get_web_connector(ctx.states))
         _keys = fetch_pipes_keys(
-            'registered', get_web_connector(ctx.states),
-            connector_keys=_ck_filter, metric_keys=_mk_filter, location_keys=_lk_filter
+            'registered',
+            get_web_connector(ctx.states),
+            connector_keys = _ck_filter,
+            metric_keys = _mk_filter,
+            location_keys = _lk_filter,
         )
     except Exception as e:
         instance_alerts += [alert_from_success_tuple((False, str(e)))]
@@ -545,15 +543,39 @@ def update_keys_options(
             k = locals()[key_type]
             if k not in _seen_keys[key_type]:
                 _k = 'None' if k in (None, '[None]', 'None', 'null') else k
-                options.append({'label' : _k, 'value' : _k})
+                options.append({'label': _k, 'value': _k})
                 _seen_keys[key_type].add(k)
 
     add_options(_connectors_options, _all_keys if _ck_alone else _keys, 'ck')
     add_options(_metrics_options, _all_keys if _mk_alone else _keys, 'mk')
     add_options(_locations_options, _all_keys if _lk_alone else _keys, 'lk')
-    connector_keys = [ck for ck in connector_keys if ck in [_ck['value'] for _ck in _connectors_options]]
-    metric_keys = [mk for mk in metric_keys if mk in [_mk['value'] for _mk in _metrics_options]]
-    location_keys = [lk for lk in location_keys if lk in [_lk['value'] for _lk in _locations_options]]
+    _connectors_options.sort(key=lambda x: str(x).lower())
+    _metrics_options.sort(key=lambda x: str(x).lower())
+    _locations_options.sort(key=lambda x: str(x).lower())
+    connector_keys = [
+        ck
+        for ck in connector_keys
+        if ck in [
+            _ck['value']
+            for _ck in _connectors_options
+        ]
+    ]
+    metric_keys = [
+        mk
+        for mk in metric_keys
+        if mk in [
+            _mk['value']
+            for _mk in _metrics_options
+        ]
+    ]
+    location_keys = [
+        lk
+        for lk in location_keys
+        if lk in [
+            _lk['value']
+            for _lk in _locations_options
+        ]
+    ]
     _connectors_datalist = [html.Option(value=o['value']) for o in _connectors_options]
     _metrics_datalist = [html.Option(value=o['value']) for o in _metrics_options]
     _locations_datalist = [html.Option(value=o['value']) for o in _locations_options]
@@ -680,6 +702,9 @@ dash_app.clientside_callback(
     Input({'type': 'pipe-download-csv-button', 'index': ALL}, 'n_clicks'),
 )
 def download_pipe_csv(n_clicks):
+    """
+    Download the most recent chunk as a CSV file.
+    """
     if not n_clicks:
         raise PreventUpdate
     ctx = dash.callback_context.triggered
@@ -688,11 +713,11 @@ def download_pipe_csv(n_clicks):
     pipe = pipe_from_ctx(ctx, 'n_clicks')
     if pipe is None:
         raise PreventUpdate
-    filename = str(pipe.target) + '.csv'
     bounds = pipe.get_chunk_bounds(bounded=True, debug=debug)
-    begin, _ = bounds[-1]
+    begin, end = bounds[-1]
+    filename = str(pipe.target) + f" {begin} - {end}.csv"
     try:
-        df = pipe.get_data(begin=begin, end=None, debug=debug)
+        df = pipe.get_data(begin=begin, end=end, debug=debug)
     except Exception as e:
         df = None
     if df is not None:
@@ -818,6 +843,51 @@ def sync_documents_click(n_clicks, sync_editor_text):
     return alert_from_success_tuple((success, msg))
 
 
+dash_app.clientside_callback(
+    """
+    function(n_clicks_arr, url){
+        display_block = {"display": "block"};
+
+        var clicked = false;
+        for (var i = 0; i < n_clicks_arr.length; i++){
+            if (n_clicks_arr[i]){
+                clicked = true;
+                break;
+            }
+        }
+        if (!clicked){ return dash_clientside.no_update; }
+
+        const triggered_id = dash_clientside.callback_context.triggered_id;
+        const action = triggered_id["action"];
+        const pipe_meta = JSON.parse(triggered_id["index"]);
+
+        iframe = document.getElementById('webterm-iframe');
+        if (!iframe){ return dash_clientside.no_update; }
+        var location = pipe_meta.location;
+        if (!pipe_meta.location){
+            location = "None";
+        }
+
+        iframe.contentWindow.postMessage(
+            {
+                action: action,
+                subaction: "pipes",
+                connector_keys: [pipe_meta.connector],
+                metric_keys: [pipe_meta.metric],
+                location_keys: [location],
+                instance: pipe_meta.instance,
+            },
+            url
+        );
+        dash_clientside.set_props("webterm-div", {"style": display_block});
+        return [];
+    }
+    """,
+    Output('content-div-right', 'children'),
+    Input({'type': 'manage-pipe-button', 'index': ALL, 'action': ALL}, 'n_clicks'),
+    State('location', 'href'),
+)
+
 @dash_app.callback(
     Output("navbar-collapse", "is_open"),
     [Input("navbar-toggler", "n_clicks")],