meerschaum 2.7.5__py3-none-any.whl → 2.7.7__py3-none-any.whl

meerschaum/_internal/shell/Shell.py

@@ -742,8 +742,7 @@ class Shell(cmd.Cmd):
         """
         from meerschaum import get_connector
         from meerschaum.connectors.parse import parse_instance_keys
-        from meerschaum.utils.warnings import warn, info
-        from meerschaum.utils.misc import remove_ansi
+        from meerschaum.utils.warnings import info
 
         if action is None:
             action = []
@@ -829,7 +828,7 @@ class Shell(cmd.Cmd):
         """
         from meerschaum import get_connector
         from meerschaum.connectors.parse import parse_repo_keys
-        from meerschaum.utils.warnings import warn, info
+        from meerschaum.utils.warnings import info
 
         if action is None:
             action = []
@@ -878,7 +877,6 @@ class Shell(cmd.Cmd):
         
         Note that executors are API instances.
         """
-        from meerschaum import get_connector
         from meerschaum.connectors.parse import parse_executor_keys
         from meerschaum.utils.warnings import warn, info
         from meerschaum.jobs import get_executor_keys_from_context
@@ -894,7 +892,7 @@ class Shell(cmd.Cmd):
             executor_keys = get_executor_keys_from_context()
 
         if executor_keys == 'systemd' and get_executor_keys_from_context() != 'systemd':
-            warn(f"Cannot execute via `systemd`, falling back to `local`...", stack=False)
+            warn("Cannot execute via `systemd`, falling back to `local`...", stack=False)
             executor_keys = 'local'
         
         conn = parse_executor_keys(executor_keys, debug=debug)
@@ -935,7 +933,7 @@ class Shell(cmd.Cmd):
         if args['action'][0] not in shell_attrs['_actions']:
             try:
                 print(textwrap.dedent(getattr(self, f"do_{args['action'][0]}").__doc__))
-            except Exception as e:
+            except Exception:
                 print(f"No help on '{args['action'][0]}'.")
             return ""
         parse_help(args)

meerschaum/_internal/shell/ShellCompleter.py

@@ -7,13 +7,15 @@ Implement the prompt_toolkit Completer base class.
 """
 
 from __future__ import annotations
-from prompt_toolkit.completion import Completer, Completion
-from meerschaum.utils.typing import Optional
-from meerschaum.actions import get_shell, get_completer, get_main_action_name, get_action
+from meerschaum.actions import get_shell, get_main_action_name, get_action
 from meerschaum._internal.arguments import parse_line
 
 from meerschaum.utils.packages import attempt_import, ensure_readline
-prompt_toolkit = attempt_import('prompt_toolkit', lazy=False, install=True)
+
+prompt_toolkit_completion = attempt_import('prompt_toolkit.completion', lazy=False, install=True)
+Completer = prompt_toolkit_completion.Completer
+Completion = prompt_toolkit_completion.Completion
+
 
 class ShellCompleter(Completer):
     """
@@ -30,7 +32,6 @@ class ShellCompleter(Completer):
         yielded = []
         ensure_readline()
         parts = document.text.split('-')
-        ends_with_space = parts[0].endswith(' ')
         last_action_line = parts[0].split('+')[-1]
         part_0_subbed_spaces = last_action_line.replace(' ', '_')
         parsed_text = (part_0_subbed_spaces + '-'.join(parts[1:]))

meerschaum/actions/clear.py

@@ -130,12 +130,15 @@ def _ask_with_rowcounts(
                     )
 
 
-    pipes_rowcounts = {p: p.get_rowcount(begin=begin, end=end, debug=debug) for p in pipes} 
+    pipes_rowcounts = {
+        pipe: pipe.get_rowcount(begin=begin, end=end, debug=debug)
+        for pipe in pipes
+    }
     print_options(
-        [str(p) + f'\n{rc}\n' for p, rc in pipes_rowcounts.items()],
+        [str(pipe) + f'\n{rowcount:,}\n' for pipe, rowcount in pipes_rowcounts.items()],
         header='Number of Rows to be Deleted'
     )
-    total_num_rows = sum([rc for p, rc in pipes_rowcounts.items()])
+    total_num_rows = sum([rowcount for rowcount in pipes_rowcounts.values()])
     question = (
         f"Are you sure you want to delete {total_num_rows:,} rows across {len(pipes)} pipe"
         + ('s' if len(pipes) != 1 else '')

meerschaum/actions/copy.py

@@ -7,7 +7,7 @@ Functions for copying elements.
 """
 
 from __future__ import annotations
-from meerschaum.utils.typing import Union, Any, Sequence, SuccessTuple, Optional, Tuple, List
+from meerschaum.utils.typing import Any, SuccessTuple, Optional, List
 
 def copy(
     action: Optional[List[str]] = None,
@@ -69,44 +69,53 @@ def _copy_pipes(
     Copy pipes' attributes and make new pipes.
     """
     from meerschaum import get_pipes, Pipe
-    from meerschaum.utils.prompt import prompt, yes_no
+    from meerschaum.utils.prompt import prompt, yes_no, get_connectors_completer
     from meerschaum.utils.warnings import warn
     from meerschaum.utils.formatting import print_tuple
     from meerschaum.utils.formatting._shell import clear_screen
     pipes = get_pipes(as_list=True, **kw)
     successes = 0
-    for p in pipes:
-        ck = prompt(f"Connector keys for copy of {p}:", default=p.connector_keys)
-        mk = prompt(f"Metric key for copy of {p}:", default=p.metric_key)
-        lk = prompt(f"Location key for copy of {p} ('None' to omit):", default=str(p.location_key))
+    for pipe in pipes:
+        ck = prompt(
+            f"Connector keys for copy of {pipe}:",
+            default=pipe.connector_keys,
+            completer=get_connectors_completer(),
+        )
+        mk = prompt(f"Metric key for copy of {pipe}:", default=pipe.metric_key)
+        lk = prompt(
+            f"Location key for copy of {pipe} ('None' to omit):",
+            default=str(pipe.location_key),
+        )
         if lk in ('', 'None', '[None]'):
             lk = None
-        _new_pipe = Pipe(
-            ck, mk, lk,
-            parameters=p.parameters.copy(),
-        )
+
         instance_keys = prompt(
-            f"Meerschaum instance to store the new {_new_pipe}:",
-            default=p.instance_keys
+            f"Meerschaum instance for copy of {pipe}:",
+            default=pipe.instance_keys
         )
-        _new_pipe.instance_keys = instance_keys
-        if _new_pipe.get_id(debug=debug) is not None:
-            warn(f"New {_new_pipe} already exists. Skipping...", stack=False)
+        new_pipe = Pipe(
+            ck, mk, lk,
+            instance=instance_keys,
+            parameters=pipe.parameters.copy(),
+        )
+
+        if new_pipe.get_id(debug=debug) is not None:
+            warn(f"{new_pipe} already exists. Skipping...", stack=False)
             continue
-        _register_success_tuple = _new_pipe.register(debug=debug)
+        _register_success_tuple = new_pipe.register(debug=debug)
         if not _register_success_tuple[0]:
-            warn(f"Failed to register new {_new_pipe}.", stack=False)
+            warn(f"Failed to register new {new_pipe}.", stack=False)
             continue
 
         clear_screen(debug=debug)
         successes += 1
         print_tuple(
-            (True, f"Successfully copied attributes of {p} " + f" into {_new_pipe}.")
+            (True, f"Successfully copied attributes of {pipe} " + f" into {new_pipe}.")
         )
         if (
             force or yes_no(
                 (
-                    f"Do you want to copy data from {p} into {_new_pipe}?\n\n"
+                    f"Do you want to copy data from {pipe} into {new_pipe}?\n\n"
                     + "If you specified `--begin`, `--end` or `--params`, data will be filtered."
                 ),
                     noask=noask,
@@ -114,8 +123,8 @@ def _copy_pipes(
                     default='n',
                 )
         ):
-            _new_pipe.sync(
-                p.get_data(
+            new_pipe.sync(
+                pipe.get_data(
                     debug=debug,
                     as_iterator=True,
                     **kw
@@ -142,17 +151,14 @@ def _copy_connectors(
 ) -> SuccessTuple:
     """
     Create a new connector from an existing one.
-
     """
-    import os, pathlib
     from meerschaum.utils.prompt import yes_no, prompt
     from meerschaum.connectors.parse import parse_connector_keys
     from meerschaum.config import _config, get_config
-    from meerschaum.config._edit import write_config
-    from meerschaum.utils.warnings import info, warn
+    from meerschaum.utils.warnings import info
     from meerschaum.utils.formatting import pprint
     from meerschaum.actions import get_action
-    cf = _config()
+    _ = _config()
     if action is None:
         action = []
     if connector_keys is None:
@@ -170,7 +176,7 @@ def _copy_connectors(
 
     try:
         conn = parse_connector_keys(ck)
-    except Exception as e:
+    except Exception:
         return False, f"Unable to parse connector '{ck}'."
 
     if len(_keys) == 2:

meerschaum/actions/drop.py

@@ -6,52 +6,65 @@ Functions for dropping elements
 """
 
 from __future__ import annotations
-from meerschaum.utils.typing import SuccessTuple, Union, Any, Optional, Sequence, List
+from meerschaum.utils.typing import SuccessTuple, Any, Optional, List
+
 
 def drop(
-        action: Optional[List[str]] = None,
-        **kw: Any
-    ) -> SuccessTuple:
+    action: Optional[List[str]] = None,
+    **kw: Any
+) -> SuccessTuple:
     """
     Drop pipe data (maintaining registration) or tables.
     """
     from meerschaum.actions import choose_subaction
     options = {
         'pipes'  : _drop_pipes,
+        'indices': _drop_indices,
+        'index': _drop_indices,
+        'indexes': _drop_indices,
     }
     return choose_subaction(action, options, **kw)
 
 
 def _drop_pipes(
-        action: Optional[List[str]] = None,
-        yes: bool = False,
-        force: bool = False,
-        noask: bool = False,
-        debug: bool = False,
-        **kw: Any
-    ) -> SuccessTuple:
+    action: Optional[List[str]] = None,
+    yes: bool = False,
+    force: bool = False,
+    noask: bool = False,
+    debug: bool = False,
+    **kw: Any
+) -> SuccessTuple:
     """
     Drop pipes' tables but keep pipe metadata registration.
     """
-    from meerschaum.utils.prompt import prompt, yes_no
+    from meerschaum.utils.prompt import yes_no
     from meerschaum import get_pipes
     from meerschaum.utils.warnings import warn
     from meerschaum.utils.debug import dprint
     from meerschaum.utils.formatting import pprint
+
     pipes = get_pipes(as_list=True, debug=debug, **kw)
     if len(pipes) == 0:
         return False, "No pipes to drop."
+
     question = (
         "Are you sure you want to drop these tables?\n    "
-        + "Data will be lost and will need to be resynced.\n\n"
+        "Data will be lost and will need to be resynced.\n\n"
     )
-    for p in pipes:
-        question += f"    - {p.target}" + "\n"
+    seen_targets = set()
+    for pipe in pipes:
+        target = pipe.target
+        if target in seen_targets:
+            continue
+        question += f"    - {target}" + "\n"
+        seen_targets.add(target)
+
     question += '\n'
     if force:
         answer = True
     else:
         answer = yes_no(question, default='n', noask=noask, yes=yes)
+
     if not answer:
         return False, "No pipes were dropped."
 
@@ -59,22 +72,87 @@ def _drop_pipes(
     successes, fails = 0, 0
     msg = ""
 
-    for p in pipes:
-        success_tup = p.drop(debug=debug)
-        success_dict[p] = success_tup[1]
-        if success_tup[0]:
+    for pipe in pipes:
+        drop_success, drop_msg = pipe.drop(debug=debug)
+        success_dict[pipe] = drop_msg
+        if drop_success:
             successes += 1
         else:
             fails += 1
-            warn(success_tup[1], stack=False)
+            warn(drop_msg, stack=False)
     
     if debug:
         dprint("Results for dropping pipes.")
         pprint(success_dict)
 
     msg = (
-        f"Finished dropping {len(pipes)} pipes\n" + 
-        f"    ({successes} succeeded, {fails} failed)."
+        f"Finished dropping {len(pipes)} pipes"
+        + ('s' if len(pipes) != 1 else '')
+        + f"\n    ({successes} succeeded, {fails} failed)."
+    )
+    return successes > 0, msg
+
+
+def _drop_indices(
+    action: Optional[List[str]] = None,
+    yes: bool = False,
+    force: bool = False,
+    noask: bool = False,
+    debug: bool = False,
+    **kw: Any
+) -> SuccessTuple:
+    """
+    Drop pipes' tables but keep pipe metadata registration.
+    """
+    from meerschaum.utils.prompt import yes_no
+    from meerschaum import get_pipes
+    from meerschaum.utils.warnings import warn
+    from meerschaum.utils.debug import dprint
+    from meerschaum.utils.formatting import pprint
+
+    pipes = get_pipes(as_list=True, debug=debug, **kw)
+    if len(pipes) == 0:
+        return False, "No pipes to drop."
+
+    question = "Are you sure you want to drop these indices?\n"
+    for pipe in pipes:
+        indices = pipe.get_indices()
+        if not indices:
+            continue
+        question += f"\n{pipe}\n"
+        for ix_key, index_name in indices.items():
+            question += f"    - {index_name}\n"
+
+    question += '\n'
+    if force:
+        answer = True
+    else:
+        answer = yes_no(question, default='n', noask=noask, yes=yes)
+
+    if not answer:
+        return False, "No pipes were dropped."
+
+    success_dict = {}
+    successes, fails = 0, 0
+    msg = ""
+
+    for pipe in pipes:
+        drop_success, drop_msg = pipe.drop_indices(debug=debug)
+        success_dict[pipe] = drop_msg
+        if drop_success:
+            successes += 1
+        else:
+            fails += 1
+            warn(drop_msg, stack=False)
+    
+    if debug:
+        dprint("Results for dropping indices.")
+        pprint(success_dict)
+
+    msg = (
+        f"Finished dropping indices for {len(pipes)} pipe"
+        + ('s' if len(pipes) != 1 else '')
+        + f"\n    ({successes} succeeded, {fails} failed)."
     )
     return successes > 0, msg
 

meerschaum/actions/index.py

@@ -0,0 +1,71 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+# vim:fenc=utf-8
+
+"""
+Functions for indexing tables.
+"""
+
+from __future__ import annotations
+from meerschaum.utils.typing import SuccessTuple, Any, Optional, List
+
+
+def index(
+    action: Optional[List[str]] = None,
+    **kw: Any
+) -> SuccessTuple:
+    """
+    Create pipes' indices.
+    """
+    from meerschaum.actions import choose_subaction
+    options = {
+        'pipes'  : _index_pipes,
+    }
+    return choose_subaction(action, options, **kw)
+
+
+def _index_pipes(
+    action: Optional[List[str]] = None,
+    yes: bool = False,
+    force: bool = False,
+    noask: bool = False,
+    debug: bool = False,
+    **kw: Any
+) -> SuccessTuple:
+    """
+    Create pipes' indices.
+    """
+    from meerschaum import get_pipes
+    from meerschaum.utils.warnings import warn, info
+
+    pipes = get_pipes(as_list=True, debug=debug, **kw)
+    if len(pipes) == 0:
+        return False, "No pipes to index."
+
+    success_dict = {}
+    successes, fails = 0, 0
+    msg = ""
+
+    for pipe in pipes:
+        info(f"Creating indices for {pipe}...")
+        index_success, index_msg = pipe.create_indices(debug=debug)
+        success_dict[pipe] = index_msg
+        if index_success:
+            successes += 1
+        else:
+            fails += 1
+            warn(index_msg, stack=False)
+    
+    msg = (
+        f"Finished indexing {len(pipes)} pipes"
+        + ('s' if len(pipes) != 1 else '')
+        + f"\n    ({successes} succeeded, {fails} failed)."
+    )
+    return successes > 0, msg
+
+
+### NOTE: This must be the final statement of the module.
+###       Any subactions added below these lines will not
+###       be added to the `help` docstring.
+from meerschaum.actions import choices_docstring as _choices_docstring
+index.__doc__ += _choices_docstring('index')

meerschaum/actions/register.py

@@ -10,6 +10,7 @@ from __future__ import annotations
 import meerschaum as mrsm
 from meerschaum.utils.typing import SuccessTuple, Any, List, Optional, Dict
 
+
 def register(
     action: Optional[List[str]] = None,
     **kw: Any
@@ -70,10 +71,9 @@ def _register_pipes(
         `connector_keys` and `metric_keys`.
         If `location_keys` is empty, assume [`None`].
     """
-    from meerschaum import get_pipes, get_connector
+    from meerschaum import get_pipes
     from meerschaum.utils.debug import dprint
-    from meerschaum.utils.warnings import warn, info
-    from meerschaum.utils.misc import items_str
+    from meerschaum.utils.warnings import warn
     from meerschaum.config._patch import apply_patch_to_config
 
     if connector_keys is None:
@@ -164,10 +164,8 @@ def _register_plugins(
     from meerschaum.utils.debug import dprint
     from meerschaum.plugins import reload_plugins, get_plugins_names
     from meerschaum.connectors.parse import parse_repo_keys
-    from meerschaum.config import get_config
-    from meerschaum.utils.warnings import warn, error, info
+    from meerschaum.utils.warnings import warn, info
     from meerschaum.core import Plugin
-    from meerschaum import get_connector
     from meerschaum.utils.formatting import print_tuple
     from meerschaum.utils.prompt import prompt, yes_no
 
@@ -179,7 +177,7 @@ def _register_plugins(
     repo_connector = parse_repo_keys(repository)
     if repo_connector.__dict__.get('type', None) != 'api':
         return False, (
-            f"Can only upload plugins to the Meerschaum API." +
+            "Can only upload plugins to the Meerschaum API." +
             f"Connector '{repo_connector}' is of type " +
             f"'{repo_connector.get('type', type(repo_connector))}'."
         )
@@ -283,13 +281,11 @@ def _register_users(
     """
     from meerschaum.config import get_config
     from meerschaum.config.static import STATIC_CONFIG
-    from meerschaum import get_connector
     from meerschaum.connectors.parse import parse_instance_keys
-    from meerschaum.utils.debug import dprint
-    from meerschaum.utils.warnings import warn, error, info
+    from meerschaum.utils.warnings import warn, info
     from meerschaum.core import User
     from meerschaum.utils.formatting import print_tuple
-    from meerschaum.utils.prompt import prompt, get_password, get_email
+    from meerschaum.utils.prompt import get_password, get_email
     if mrsm_instance is None:
         mrsm_instance = get_config('meerschaum', 'instance')
     instance_connector = parse_instance_keys(mrsm_instance)
@@ -325,7 +321,7 @@ def _register_users(
                 minimum_length = STATIC_CONFIG['users']['min_password_length']
             )
             email = get_email(username, allow_omit=True)
-        except Exception as e:
+        except Exception:
             return False, (
                 "Aborted registering users " +
                 ', '.join(

meerschaum/actions/sql.py

@@ -127,7 +127,7 @@ def sql(
     
     from meerschaum.utils.packages import attempt_import
     from meerschaum.utils.formatting import print_tuple, pprint
-    _ = attempt_import('sqlalchemy.engine.result')
+    _ = attempt_import('sqlalchemy.engine.result', lazy=False)
     if 'sqlalchemy' in str(type(result)):
         if not nopretty:
             print_tuple((True, f"Successfully executed query:\n\n{query}"))

meerschaum/actions/sync.py

@@ -275,12 +275,14 @@ def _sync_pipes(
     import time
     import os
     import contextlib
+    from datetime import timedelta
 
     from meerschaum.utils.warnings import warn, info
     from meerschaum.utils.formatting._shell import progress
     from meerschaum.utils.formatting._shell import clear_screen
     from meerschaum.utils.formatting import print_pipes_results
     from meerschaum.config.static import STATIC_CONFIG
+    from meerschaum.utils.misc import interval_str
 
     noninteractive_val = os.environ.get(STATIC_CONFIG['environment']['noninteractive'], None)
     noninteractive = str(noninteractive_val).lower() in ('1', 'true', 'yes')
@@ -321,7 +323,7 @@ def _sync_pipes(
                     for pipe, (_success, _msg) in results_dict.items()
                     if not _success
                 ]
-        except Exception as e:
+        except Exception:
             import traceback
             traceback.print_exc()
             warn(
@@ -351,9 +353,9 @@ def _sync_pipes(
         success_msg = (
             "Successfully spawned threads for pipes:"
             if unblock
-            else f"Successfully synced pipes:"
+            else "Successfully synced pipes:"
         )
-        fail_msg = f"Failed to sync pipes:"
+        fail_msg = "Failed to sync pipes:"
         if results_dict:
             print_pipes_results(
                 results_dict,
@@ -362,8 +364,10 @@ def _sync_pipes(
                 nopretty = nopretty,
             )
 
+        lap_duration_text = interval_str(timedelta(seconds=(lap_end - lap_begin)))
+
         msg = (
-            f"It took {round(lap_end - lap_begin, 2)} seconds to sync " +
+            f"It took {lap_duration_text} to sync " +
             f"{len(success_pipes) + len(failure_pipes)} pipe" +
                 ("s" if (len(success_pipes) + len(failure_pipes)) != 1 else "") + "\n" +
             f"    ({len(success_pipes)} succeeded, {len(failure_pipes)} failed)."
@@ -385,26 +389,26 @@ def _sync_pipes(
 
 
 def _wrap_pipe(
-        pipe,
-        unblock: bool = False,
-        force: bool = False,
-        debug: bool = False,
-        min_seconds: int = 1,
-        workers = None,
-        verify: bool = False,
-        deduplicate: bool = False,
-        bounded: Optional[bool] = None,
-        chunk_interval: Union[timedelta, int, None] = None,
-        **kw
-    ):
+    pipe,
+    unblock: bool = False,
+    force: bool = False,
+    debug: bool = False,
+    min_seconds: int = 1,
+    workers = None,
+    verify: bool = False,
+    deduplicate: bool = False,
+    bounded: Optional[bool] = None,
+    chunk_interval: Union[timedelta, int, None] = None,
+    **kw
+):
     """
     Wrapper function for handling exceptions.
     """
     import time
     import traceback
-    from datetime import datetime, timedelta, timezone
+    from datetime import datetime, timezone
     import meerschaum as mrsm
-    from meerschaum.utils.typing import is_success_tuple, SuccessTuple
+    from meerschaum.utils.typing import is_success_tuple
     from meerschaum.connectors import get_connector_plugin
     from meerschaum.utils.venv import Venv
     from meerschaum.plugins import _pre_sync_hooks, _post_sync_hooks

meerschaum/api/dash/pipes.py

@@ -12,7 +12,6 @@ import shlex
 from textwrap import dedent
 from urllib.parse import urlencode
 
-from dash.dependencies import Input, Output, State
 from meerschaum.utils.typing import List, Optional, Dict, Any, Tuple, Union
 from meerschaum.utils.misc import string_to_dict
 from meerschaum.utils.packages import attempt_import, import_dcc, import_html, import_pandas
@@ -76,7 +75,7 @@ def keys_from_state(
         try:
             #  params = string_to_dict(state['params-textarea.value'])
             params = string_to_dict(state['search-parameters-editor.value'])
-        except Exception as e:
+        except Exception:
             params = None
     else:
         params = None
@@ -506,7 +505,7 @@ def accordion_items_from_pipe(
 
         stats_rows = []
         if rowcount is not None:
-            stats_rows.append(html.Tr([html.Td("Row-count"), html.Td(f"{rowcount}")]))
+            stats_rows.append(html.Tr([html.Td("Row Count"), html.Td(f"{rowcount:,}")]))
         if interval is not None:
             stats_rows.append(
                 html.Tr([html.Td("Timespan"), html.Td(humanfriendly.format_timespan(interval))])

meerschaum/api/routes/_pipes.py

@@ -688,3 +688,21 @@ def get_pipe_columns_indices(
     Return a dictionary of column names and related indices.
     """
     return get_pipe(connector_keys, metric_key, location_key).get_columns_indices()
+
+
+@app.get(
+    pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/indices/names',
+    tags=['Pipes']
+)
+def get_pipe_index_names(
+    connector_keys: str,
+    metric_key: str,
+    location_key: str,
+    curr_user=(
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> Dict[str, List[Dict[str, str]]]:
+    """
+    Return a dictionary of index keys and index names.
+    """
+    return get_pipe(connector_keys, metric_key, location_key).get_indices()